common.py 27 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887
  1. #!/usr/bin/env python
  2. # Copyright Rene Rivera 2016
  3. #
  4. # Distributed under the Boost Software License, Version 1.0.
  5. # (See accompanying file LICENSE_1_0.txt or copy at
  6. # http://www.boost.org/LICENSE_1_0.txt)
  7. import sys
  8. import inspect
  9. import optparse
  10. import os.path
  11. import string
  12. import time
  13. import subprocess
  14. import codecs
  15. import shutil
  16. import threading
  17. toolset_info = {
  18. 'clang-3.4' : {
  19. 'ppa' : ["ppa:h-rayflood/llvm"],
  20. 'package' : 'clang-3.4',
  21. 'command' : 'clang++-3.4',
  22. 'toolset' : 'clang',
  23. 'version' : ''
  24. },
  25. 'clang-3.5' : {
  26. 'ppa' : ["ppa:h-rayflood/llvm"],
  27. 'package' : 'clang-3.5',
  28. 'command' : 'clang++-3.5',
  29. 'toolset' : 'clang',
  30. 'version' : ''
  31. },
  32. 'clang-3.6' : {
  33. 'ppa' : ["ppa:h-rayflood/llvm"],
  34. 'package' : 'clang-3.6',
  35. 'command' : 'clang++-3.6',
  36. 'toolset' : 'clang',
  37. 'version' : ''
  38. },
  39. 'clang-3.7' : {
  40. 'deb' : ["http://apt.llvm.org/trusty/","llvm-toolchain-trusty-3.7","main"],
  41. 'apt-key' : ['http://apt.llvm.org/llvm-snapshot.gpg.key'],
  42. 'package' : 'clang-3.7',
  43. 'command' : 'clang++-3.7',
  44. 'toolset' : 'clang',
  45. 'version' : ''
  46. },
  47. 'clang-3.8' : {
  48. 'deb' : ["http://apt.llvm.org/trusty/","llvm-toolchain-trusty-3.8","main"],
  49. 'apt-key' : ['http://apt.llvm.org/llvm-snapshot.gpg.key'],
  50. 'package' : 'clang-3.8',
  51. 'command' : 'clang++-3.8',
  52. 'toolset' : 'clang',
  53. 'version' : ''
  54. },
  55. 'clang-3.9' : {
  56. 'deb' : ["http://apt.llvm.org/trusty/","llvm-toolchain-trusty-3.9","main"],
  57. 'apt-key' : ['http://apt.llvm.org/llvm-snapshot.gpg.key'],
  58. 'package' : 'clang-3.9',
  59. 'command' : 'clang++-3.9',
  60. 'toolset' : 'clang',
  61. 'version' : ''
  62. },
  63. 'clang-4.0' : {
  64. 'deb' : ["http://apt.llvm.org/trusty/","llvm-toolchain-trusty-4.0","main"],
  65. 'apt-key' : ['http://apt.llvm.org/llvm-snapshot.gpg.key'],
  66. 'package' : 'clang-4.0',
  67. 'command' : 'clang++-4.0',
  68. 'toolset' : 'clang',
  69. 'version' : ''
  70. },
  71. 'clang-5.0' : {
  72. 'deb' : ["http://apt.llvm.org/trusty/","llvm-toolchain-trusty-5.0","main"],
  73. 'apt-key' : ['http://apt.llvm.org/llvm-snapshot.gpg.key'],
  74. 'package' : 'clang-5.0',
  75. 'command' : 'clang++-5.0',
  76. 'toolset' : 'clang',
  77. 'version' : ''
  78. },
  79. 'clang-6.0' : {
  80. 'deb' : ["http://apt.llvm.org/trusty/","llvm-toolchain-trusty-6.0","main"],
  81. 'apt-key' : ['http://apt.llvm.org/llvm-snapshot.gpg.key'],
  82. 'package' : 'clang-6.0',
  83. 'command' : 'clang++-6.0',
  84. 'toolset' : 'clang',
  85. 'version' : ''
  86. },
  87. 'gcc-4.7' : {
  88. 'ppa' : ["ppa:ubuntu-toolchain-r/test"],
  89. 'package' : 'g++-4.7',
  90. 'command' : 'g++-4.7',
  91. 'toolset' : 'gcc',
  92. 'version' : ''
  93. },
  94. 'gcc-4.8' : {
  95. 'bin' : 'gcc-4.8',
  96. 'ppa' : ["ppa:ubuntu-toolchain-r/test"],
  97. 'package' : 'g++-4.8',
  98. 'command' : 'g++-4.8',
  99. 'toolset' : 'gcc',
  100. 'version' : ''
  101. },
  102. 'gcc-4.9' : {
  103. 'ppa' : ["ppa:ubuntu-toolchain-r/test"],
  104. 'package' : 'g++-4.9',
  105. 'command' : 'g++-4.9',
  106. 'toolset' : 'gcc',
  107. 'version' : ''
  108. },
  109. 'gcc-5.1' : {
  110. 'ppa' : ["ppa:ubuntu-toolchain-r/test"],
  111. 'package' : 'g++-5',
  112. 'command' : 'g++-5',
  113. 'toolset' : 'gcc',
  114. 'version' : ''
  115. },
  116. 'gcc-5' : {
  117. 'ppa' : ["ppa:ubuntu-toolchain-r/test"],
  118. 'package' : 'g++-5',
  119. 'command' : 'g++-5',
  120. 'toolset' : 'gcc',
  121. 'version' : ''
  122. },
  123. 'gcc-6' : {
  124. 'ppa' : ["ppa:ubuntu-toolchain-r/test"],
  125. 'package' : 'g++-6',
  126. 'command' : 'g++-6',
  127. 'toolset' : 'gcc',
  128. 'version' : ''
  129. },
  130. 'gcc-7' : {
  131. 'ppa' : ["ppa:ubuntu-toolchain-r/test"],
  132. 'package' : 'g++-7',
  133. 'command' : 'g++-7',
  134. 'toolset' : 'gcc',
  135. 'version' : ''
  136. },
  137. 'gcc-8' : {
  138. 'ppa' : ["ppa:ubuntu-toolchain-r/test"],
  139. 'package' : 'g++-8',
  140. 'command' : 'g++-8',
  141. 'toolset' : 'gcc',
  142. 'version' : ''
  143. },
  144. 'mingw-5' : {
  145. 'toolset' : 'gcc',
  146. 'command' : 'C:\\\\MinGW\\\\bin\\\\g++.exe',
  147. 'version' : ''
  148. },
  149. 'mingw64-6' : {
  150. 'toolset' : 'gcc',
  151. 'command' : 'C:\\\\mingw-w64\\\\x86_64-6.3.0-posix-seh-rt_v5-rev1\\\\mingw64\\\\bin\\\\g++.exe',
  152. 'version' : ''
  153. },
  154. 'vs-2008' : {
  155. 'toolset' : 'msvc',
  156. 'command' : '',
  157. 'version' : '9.0'
  158. },
  159. 'vs-2010' : {
  160. 'toolset' : 'msvc',
  161. 'command' : '',
  162. 'version' : '10.0'
  163. },
  164. 'vs-2012' : {
  165. 'toolset' : 'msvc',
  166. 'command' : '',
  167. 'version' : '11.0'
  168. },
  169. 'vs-2013' : {
  170. 'toolset' : 'msvc',
  171. 'command' : '',
  172. 'version' : '12.0'
  173. },
  174. 'vs-2015' : {
  175. 'toolset' : 'msvc',
  176. 'command' : '',
  177. 'version' : '14.0'
  178. },
  179. 'vs-2017' : {
  180. 'toolset' : 'msvc',
  181. 'command' : '',
  182. 'version' : '14.1'
  183. },
  184. 'xcode-6.1' : {
  185. 'command' : 'clang++',
  186. 'toolset' : 'clang',
  187. 'version' : ''
  188. },
  189. 'xcode-6.2' : {
  190. 'command' : 'clang++',
  191. 'toolset' : 'clang',
  192. 'version' : ''
  193. },
  194. 'xcode-6.3' : {
  195. 'command' : 'clang++',
  196. 'toolset' : 'clang',
  197. 'version' : ''
  198. },
  199. 'xcode-6.4' : {
  200. 'command' : 'clang++',
  201. 'toolset' : 'clang',
  202. 'version' : ''
  203. },
  204. 'xcode-7.0' : {
  205. 'command' : 'clang++',
  206. 'toolset' : 'clang',
  207. 'version' : ''
  208. },
  209. 'xcode-7.1' : {
  210. 'command' : 'clang++',
  211. 'toolset' : 'clang',
  212. 'version' : ''
  213. },
  214. 'xcode-7.2' : {
  215. 'command' : 'clang++',
  216. 'toolset' : 'clang',
  217. 'version' : ''
  218. },
  219. 'xcode-7.3' : {
  220. 'command' : 'clang++',
  221. 'toolset' : 'clang',
  222. 'version' : ''
  223. },
  224. 'xcode-8.0' : {
  225. 'command' : 'clang++',
  226. 'toolset' : 'clang',
  227. 'version' : ''
  228. },
  229. 'xcode-8.1' : {
  230. 'command' : 'clang++',
  231. 'toolset' : 'clang',
  232. 'version' : ''
  233. },
  234. 'xcode-8.2' : {
  235. 'command' : 'clang++',
  236. 'toolset' : 'clang',
  237. 'version' : ''
  238. },
  239. 'xcode-8.3' : {
  240. 'command' : 'clang++',
  241. 'toolset' : 'clang',
  242. 'version' : ''
  243. },
  244. 'xcode-9.0' : {
  245. 'command' : 'clang++',
  246. 'toolset' : 'clang',
  247. 'version' : ''
  248. },
  249. 'xcode-9.1' : {
  250. 'command' : 'clang++',
  251. 'toolset' : 'clang',
  252. 'version' : ''
  253. },
  254. 'xcode-9.2' : {
  255. 'command' : 'clang++',
  256. 'toolset' : 'clang',
  257. 'version' : ''
  258. },
  259. 'xcode-9.3' : {
  260. 'command' : 'clang++',
  261. 'toolset' : 'clang',
  262. 'version' : ''
  263. },
  264. 'xcode-9.4' : {
  265. 'command' : 'clang++',
  266. 'toolset' : 'clang',
  267. 'version' : ''
  268. },
  269. 'xcode-10.0' : {
  270. 'command' : 'clang++',
  271. 'toolset' : 'clang',
  272. 'version' : ''
  273. },
  274. }
  275. class SystemCallError(Exception):
  276. def __init__(self, command, result):
  277. self.command = command
  278. self.result = result
  279. def __str__(self, *args, **kwargs):
  280. return "'%s' ==> %s"%("' '".join(self.command), self.result)
  281. class utils:
  282. call_stats = []
  283. @staticmethod
  284. def call(*command, **kargs):
  285. utils.log( "%s> '%s'"%(os.getcwd(), "' '".join(command)) )
  286. t = time.time()
  287. result = subprocess.call(command, **kargs)
  288. t = time.time()-t
  289. if result != 0:
  290. print "Failed: '%s' ERROR = %s"%("' '".join(command), result)
  291. utils.call_stats.append((t,os.getcwd(),command,result))
  292. utils.log( "%s> '%s' execution time %s seconds"%(os.getcwd(), "' '".join(command), t) )
  293. return result
  294. @staticmethod
  295. def print_call_stats():
  296. utils.log("================================================================================")
  297. for j in sorted(utils.call_stats, reverse=True):
  298. utils.log("{:>12.4f}\t{}> {} ==> {}".format(*j))
  299. utils.log("================================================================================")
  300. @staticmethod
  301. def check_call(*command, **kargs):
  302. cwd = os.getcwd()
  303. result = utils.call(*command, **kargs)
  304. if result != 0:
  305. raise(SystemCallError([cwd].extend(command), result))
  306. @staticmethod
  307. def makedirs( path ):
  308. if not os.path.exists( path ):
  309. os.makedirs( path )
  310. @staticmethod
  311. def log_level():
  312. frames = inspect.stack()
  313. level = 0
  314. for i in frames[ 3: ]:
  315. if i[0].f_locals.has_key( '__log__' ):
  316. level = level + i[0].f_locals[ '__log__' ]
  317. return level
  318. @staticmethod
  319. def log( message ):
  320. sys.stdout.flush()
  321. sys.stderr.flush()
  322. sys.stderr.write( '# ' + ' ' * utils.log_level() + message + '\n' )
  323. sys.stderr.flush()
  324. @staticmethod
  325. def rmtree(path):
  326. if os.path.exists( path ):
  327. #~ shutil.rmtree( unicode( path ) )
  328. if sys.platform == 'win32':
  329. os.system( 'del /f /s /q "%s" >nul 2>&1' % path )
  330. shutil.rmtree( unicode( path ) )
  331. else:
  332. os.system( 'rm -f -r "%s"' % path )
  333. @staticmethod
  334. def retry( f, max_attempts=5, sleep_secs=10 ):
  335. for attempts in range( max_attempts, -1, -1 ):
  336. try:
  337. return f()
  338. except Exception, msg:
  339. utils.log( '%s failed with message "%s"' % ( f.__name__, msg ) )
  340. if attempts == 0:
  341. utils.log( 'Giving up.' )
  342. raise
  343. utils.log( 'Retrying (%d more attempts).' % attempts )
  344. time.sleep( sleep_secs )
  345. @staticmethod
  346. def web_get( source_url, destination_file, proxy = None ):
  347. import urllib
  348. proxies = None
  349. if proxy is not None:
  350. proxies = {
  351. 'https' : proxy,
  352. 'http' : proxy
  353. }
  354. src = urllib.urlopen( source_url, proxies = proxies )
  355. f = open( destination_file, 'wb' )
  356. while True:
  357. data = src.read( 16*1024 )
  358. if len( data ) == 0: break
  359. f.write( data )
  360. f.close()
  361. src.close()
  362. @staticmethod
  363. def unpack_archive( archive_path ):
  364. utils.log( 'Unpacking archive ("%s")...' % archive_path )
  365. archive_name = os.path.basename( archive_path )
  366. extension = archive_name[ archive_name.find( '.' ) : ]
  367. if extension in ( ".tar.gz", ".tar.bz2" ):
  368. import tarfile
  369. import stat
  370. mode = os.path.splitext( extension )[1][1:]
  371. tar = tarfile.open( archive_path, 'r:%s' % mode )
  372. for tarinfo in tar:
  373. tar.extract( tarinfo )
  374. if sys.platform == 'win32' and not tarinfo.isdir():
  375. # workaround what appears to be a Win32-specific bug in 'tarfile'
  376. # (modification times for extracted files are not set properly)
  377. f = os.path.join( os.curdir, tarinfo.name )
  378. os.chmod( f, stat.S_IWRITE )
  379. os.utime( f, ( tarinfo.mtime, tarinfo.mtime ) )
  380. tar.close()
  381. elif extension in ( ".zip" ):
  382. import zipfile
  383. z = zipfile.ZipFile( archive_path, 'r', zipfile.ZIP_DEFLATED )
  384. for f in z.infolist():
  385. destination_file_path = os.path.join( os.curdir, f.filename )
  386. if destination_file_path[-1] == "/": # directory
  387. if not os.path.exists( destination_file_path ):
  388. os.makedirs( destination_file_path )
  389. else: # file
  390. result = open( destination_file_path, 'wb' )
  391. result.write( z.read( f.filename ) )
  392. result.close()
  393. z.close()
  394. else:
  395. raise 'Do not know how to unpack archives with extension \"%s\"' % extension
  396. @staticmethod
  397. def make_file(filename, *text):
  398. text = string.join( text, '\n' )
  399. with codecs.open( filename, 'w', 'utf-8' ) as f:
  400. f.write( text )
  401. @staticmethod
  402. def append_file(filename, *text):
  403. with codecs.open( filename, 'a', 'utf-8' ) as f:
  404. f.write( string.join( text, '\n' ) )
  405. @staticmethod
  406. def mem_info():
  407. if sys.platform == "darwin":
  408. utils.call("top","-l","1","-s","0","-n","0")
  409. elif sys.platform.startswith("linux"):
  410. utils.call("free","-m","-l")
  411. @staticmethod
  412. def query_boost_version(boost_root):
  413. '''
  414. Read in the Boost version from a given boost_root.
  415. '''
  416. boost_version = None
  417. if os.path.exists(os.path.join(boost_root,'Jamroot')):
  418. with codecs.open(os.path.join(boost_root,'Jamroot'), 'r', 'utf-8') as f:
  419. for line in f.readlines():
  420. parts = line.split()
  421. if len(parts) >= 5 and parts[1] == 'BOOST_VERSION':
  422. boost_version = parts[3]
  423. break
  424. if not boost_version:
  425. boost_version = 'default'
  426. return boost_version
  427. @staticmethod
  428. def git_clone(owner, repo, branch, commit = None, repo_dir = None, submodules = False, url_format = "https://github.com/%(owner)s/%(repo)s.git"):
  429. '''
  430. This clone mimicks the way Travis-CI clones a project's repo. So far
  431. Travis-CI is the most limiting in the sense of only fetching partial
  432. history of the repo.
  433. '''
  434. if not repo_dir:
  435. repo_dir = os.path.join(os.getcwd(), owner+','+repo)
  436. utils.makedirs(os.path.dirname(repo_dir))
  437. if not os.path.exists(os.path.join(repo_dir,'.git')):
  438. utils.check_call("git","clone",
  439. "--depth=1",
  440. "--branch=%s"%(branch),
  441. url_format%{'owner':owner,'repo':repo},
  442. repo_dir)
  443. os.chdir(repo_dir)
  444. else:
  445. os.chdir(repo_dir)
  446. utils.check_call("git","pull",
  447. # "--depth=1", # Can't do depth as we get merge errors.
  448. "--quiet","--no-recurse-submodules")
  449. if commit:
  450. utils.check_call("git","checkout","-qf",commit)
  451. if os.path.exists(os.path.join('.git','modules')):
  452. if sys.platform == 'win32':
  453. utils.check_call('dir',os.path.join('.git','modules'))
  454. else:
  455. utils.check_call('ls','-la',os.path.join('.git','modules'))
  456. if submodules:
  457. utils.check_call("git","submodule","--quiet","update",
  458. "--quiet","--init","--recursive",
  459. )
  460. utils.check_call("git","submodule","--quiet","foreach","git","fetch")
  461. return repo_dir
  462. class parallel_call(threading.Thread):
  463. '''
  464. Runs a synchronous command in a thread waiting for it to complete.
  465. '''
  466. def __init__(self, *command, **kargs):
  467. super(parallel_call,self).__init__()
  468. self.command = command
  469. self.command_kargs = kargs
  470. self.start()
  471. def run(self):
  472. self.result = utils.call(*self.command, **self.command_kargs)
  473. def join(self):
  474. super(parallel_call,self).join()
  475. if self.result != 0:
  476. raise(SystemCallError(self.command, self.result))
  477. def set_arg(args, k, v = None):
  478. if not args.get(k):
  479. args[k] = v
  480. return args[k]
  481. class script_common(object):
  482. '''
  483. Main script to run continuous integration.
  484. '''
  485. def __init__(self, ci_klass, **kargs):
  486. self.ci = ci_klass(self)
  487. opt = optparse.OptionParser(
  488. usage="%prog [options] [commands]")
  489. #~ Debug Options:
  490. opt.add_option( '--debug-level',
  491. help="debugging level; controls the amount of debugging output printed",
  492. type='int' )
  493. opt.add_option( '-j',
  494. help="maximum number of parallel jobs to use for building with b2",
  495. type='int', dest='jobs')
  496. opt.add_option('--branch')
  497. opt.add_option('--commit')
  498. kargs = self.init(opt,kargs)
  499. kargs = self.ci.init(opt, kargs)
  500. set_arg(kargs,'debug_level',0)
  501. set_arg(kargs,'jobs',2)
  502. set_arg(kargs,'branch',None)
  503. set_arg(kargs,'commit',None)
  504. set_arg(kargs,'repo',None)
  505. set_arg(kargs,'repo_dir',None)
  506. set_arg(kargs,'actions',None)
  507. set_arg(kargs,'pull_request', None)
  508. #~ Defaults
  509. for (k,v) in kargs.iteritems():
  510. setattr(self,k,v)
  511. ( _opt_, self.actions ) = opt.parse_args(None,self)
  512. if not self.actions or self.actions == []:
  513. self.actions = kargs.get('actions',None)
  514. if not self.actions or self.actions == []:
  515. self.actions = [ 'info' ]
  516. if not self.repo_dir:
  517. self.repo_dir = os.getcwd()
  518. self.build_dir = os.path.join(os.path.dirname(self.repo_dir), "build")
  519. # API keys.
  520. self.bintray_key = os.getenv('BINTRAY_KEY')
  521. try:
  522. self.start()
  523. self.command_info()
  524. self.main()
  525. utils.print_call_stats()
  526. except:
  527. utils.print_call_stats()
  528. raise
  529. def init(self, opt, kargs):
  530. return kargs
  531. def start(self):
  532. pass
  533. def main(self):
  534. for action in self.actions:
  535. action_m = "command_"+action.replace('-','_')
  536. ci_command = getattr(self.ci, action_m, None)
  537. ci_script = getattr(self, action_m, None)
  538. if ci_command or ci_script:
  539. utils.log( "### %s.."%(action) )
  540. if os.path.exists(self.repo_dir):
  541. os.chdir(self.repo_dir)
  542. if ci_command:
  543. ci_command()
  544. elif ci_script:
  545. ci_script()
  546. def b2( self, *args, **kargs ):
  547. cmd = ['b2','--debug-configuration', '-j%s'%(self.jobs)]
  548. cmd.extend(args)
  549. if 'toolset' in kargs:
  550. cmd.append('toolset=' + kargs['toolset'])
  551. if 'parallel' in kargs:
  552. return parallel_call(*cmd)
  553. else:
  554. return utils.check_call(*cmd)
  555. # Common test commands in the order they should be executed..
  556. def command_info(self):
  557. pass
  558. def command_install(self):
  559. utils.makedirs(self.build_dir)
  560. os.chdir(self.build_dir)
  561. def command_install_toolset(self, toolset):
  562. if self.ci and hasattr(self.ci,'install_toolset'):
  563. self.ci.install_toolset(toolset)
  564. def command_before_build(self):
  565. pass
  566. def command_build(self):
  567. pass
  568. def command_before_cache(self):
  569. pass
  570. def command_after_success(self):
  571. pass
  572. class ci_cli(object):
  573. '''
  574. This version of the script provides a way to do manual building. It sets up
  575. additional environment and adds fetching of the git repos that would
  576. normally be done by the CI system.
  577. The common way to use this variant is to invoke something like:
  578. mkdir ci
  579. cd ci
  580. python path-to/library_test.py --branch=develop [--repo=mylib] ...
  581. Status: In working order.
  582. '''
  583. def __init__(self,script):
  584. if sys.platform == 'darwin':
  585. # Requirements for running on OSX:
  586. # https://www.stack.nl/~dimitri/doxygen/download.html#srcbin
  587. # https://tug.org/mactex/morepackages.html
  588. doxygen_path = "/Applications/Doxygen.app/Contents/Resources"
  589. if os.path.isdir(doxygen_path):
  590. os.environ["PATH"] = doxygen_path+':'+os.environ['PATH']
  591. self.script = script
  592. self.repo_dir = os.getcwd()
  593. self.exit_result = 0
  594. def init(self, opt, kargs):
  595. kargs['actions'] = [
  596. # 'clone',
  597. 'install',
  598. 'before_build',
  599. 'build',
  600. 'before_cache',
  601. 'finish'
  602. ]
  603. return kargs
  604. def finish(self, result):
  605. self.exit_result = result
  606. def command_finish(self):
  607. exit(self.exit_result)
  608. class ci_travis(object):
  609. '''
  610. This variant build releases in the context of the Travis-CI service.
  611. '''
  612. def __init__(self,script):
  613. self.script = script
  614. def init(self, opt, kargs):
  615. set_arg(kargs,'repo_dir', os.getenv("TRAVIS_BUILD_DIR"))
  616. set_arg(kargs,'branch', os.getenv("TRAVIS_BRANCH"))
  617. set_arg(kargs,'commit', os.getenv("TRAVIS_COMMIT"))
  618. set_arg(kargs,'repo', os.getenv("TRAVIS_REPO_SLUG").split("/")[1])
  619. set_arg(kargs,'pull_request',
  620. os.getenv('TRAVIS_PULL_REQUEST') \
  621. if os.getenv('TRAVIS_PULL_REQUEST') != 'false' else None)
  622. return kargs
  623. def finish(self, result):
  624. exit(result)
  625. def install_toolset(self, toolset):
  626. '''
  627. Installs specific toolset on CI system.
  628. '''
  629. info = toolset_info[toolset]
  630. if sys.platform.startswith('linux'):
  631. os.chdir(self.script.build_dir)
  632. if 'ppa' in info:
  633. for ppa in info['ppa']:
  634. utils.check_call(
  635. 'sudo','add-apt-repository','--yes',ppa)
  636. if 'deb' in info:
  637. utils.make_file('sources.list',
  638. "deb %s"%(' '.join(info['deb'])),
  639. "deb-src %s"%(' '.join(info['deb'])))
  640. utils.check_call('sudo','bash','-c','cat sources.list >> /etc/apt/sources.list')
  641. if 'apt-key' in info:
  642. for key in info['apt-key']:
  643. utils.check_call('wget',key,'-O','apt.key')
  644. utils.check_call('sudo','apt-key','add','apt.key')
  645. utils.check_call(
  646. 'sudo','apt-get','update','-qq')
  647. utils.check_call(
  648. 'sudo','apt-get','install','-qq',info['package'])
  649. if 'debugpackage' in info and info['debugpackage']:
  650. utils.check_call(
  651. 'sudo','apt-get','install','-qq',info['debugpackage'])
  652. # Travis-CI commands in the order they are executed. We need
  653. # these to forward to our common commands, if they are different.
  654. def command_before_install(self):
  655. pass
  656. def command_install(self):
  657. self.script.command_install()
  658. def command_before_script(self):
  659. self.script.command_before_build()
  660. def command_script(self):
  661. self.script.command_build()
  662. def command_before_cache(self):
  663. self.script.command_before_cache()
  664. def command_after_success(self):
  665. self.script.command_after_success()
  666. def command_after_failure(self):
  667. pass
  668. def command_before_deploy(self):
  669. pass
  670. def command_after_deploy(self):
  671. pass
  672. def command_after_script(self):
  673. pass
  674. class ci_circleci(object):
  675. '''
  676. This variant build releases in the context of the CircleCI service.
  677. '''
  678. def __init__(self,script):
  679. self.script = script
  680. def init(self, opt, kargs):
  681. set_arg(kargs,'repo_dir', os.path.join(os.getenv("HOME"),os.getenv("CIRCLE_PROJECT_REPONAME")))
  682. set_arg(kargs,'branch', os.getenv("CIRCLE_BRANCH"))
  683. set_arg(kargs,'commit', os.getenv("CIRCLE_SHA1"))
  684. set_arg(kargs,'repo', os.getenv("CIRCLE_PROJECT_REPONAME").split("/")[1])
  685. set_arg(kargs,'pull_request', os.getenv('CIRCLE_PR_NUMBER'))
  686. return kargs
  687. def finish(self, result):
  688. exit(result)
  689. def command_machine_post(self):
  690. # Apt update for the pckages installs we'll do later.
  691. utils.check_call('sudo','apt-get','-qq','update')
  692. # Need PyYAML to read Travis yaml in a later step.
  693. utils.check_call("pip","install","--user","PyYAML")
  694. def command_checkout_post(self):
  695. os.chdir(self.script.repo_dir)
  696. utils.check_call("git","submodule","update","--quiet","--init","--recursive")
  697. def command_dependencies_pre(self):
  698. # Read in .travis.yml for list of packages to install
  699. # as CircleCI doesn't have a convenient apt install method.
  700. import yaml
  701. utils.check_call('sudo','-E','apt-get','-yqq','update')
  702. utils.check_call('sudo','apt-get','-yqq','purge','texlive*')
  703. with open(os.path.join(self.script.repo_dir,'.travis.yml')) as yml:
  704. travis_yml = yaml.load(yml)
  705. utils.check_call('sudo','apt-get','-yqq',
  706. '--no-install-suggests','--no-install-recommends','--force-yes','install',
  707. *travis_yml['addons']['apt']['packages'])
  708. def command_dependencies_override(self):
  709. self.script.command_install()
  710. def command_dependencies_post(self):
  711. pass
  712. def command_database_pre(self):
  713. pass
  714. def command_database_override(self):
  715. pass
  716. def command_database_post(self):
  717. pass
  718. def command_test_pre(self):
  719. self.script.command_install()
  720. self.script.command_before_build()
  721. def command_test_override(self):
  722. # CircleCI runs all the test subsets. So in order to avoid
  723. # running the after_success we do it here as the build step
  724. # will halt accordingly.
  725. self.script.command_build()
  726. self.script.command_before_cache()
  727. self.script.command_after_success()
  728. def command_test_post(self):
  729. pass
  730. class ci_appveyor(object):
  731. def __init__(self,script):
  732. self.script = script
  733. def init(self, opt, kargs):
  734. set_arg(kargs,'repo_dir',os.getenv("APPVEYOR_BUILD_FOLDER"))
  735. set_arg(kargs,'branch',os.getenv("APPVEYOR_REPO_BRANCH"))
  736. set_arg(kargs,'commit',os.getenv("APPVEYOR_REPO_COMMIT"))
  737. set_arg(kargs,'repo',os.getenv("APPVEYOR_REPO_NAME").split("/")[1])
  738. set_arg(kargs,'address_model',os.getenv("PLATFORM",None))
  739. set_arg(kargs,'variant',os.getenv("CONFIGURATION","debug"))
  740. set_arg(kargs,'pull_request', os.getenv('APPVEYOR_PULL_REQUEST_NUMBER'))
  741. return kargs
  742. def finish(self, result):
  743. exit(result)
  744. # Appveyor commands in the order they are executed. We need
  745. # these to forward to our common commands, if they are different.
  746. def command_install(self):
  747. self.script.command_install()
  748. def command_before_build(self):
  749. os.chdir(self.script.repo_dir)
  750. utils.check_call("git","submodule","update","--quiet","--init","--recursive")
  751. self.script.command_before_build()
  752. def command_build_script(self):
  753. self.script.command_build()
  754. def command_after_build(self):
  755. self.script.command_before_cache()
  756. def command_before_test(self):
  757. pass
  758. def command_test_script(self):
  759. pass
  760. def command_after_test(self):
  761. pass
  762. def command_on_success(self):
  763. self.script.command_after_success()
  764. def command_on_failure(self):
  765. pass
  766. def command_on_finish(self):
  767. pass
  768. def main(script_klass):
  769. if os.getenv('TRAVIS', False):
  770. script_klass(ci_travis)
  771. elif os.getenv('CIRCLECI', False):
  772. script_klass(ci_circleci)
  773. elif os.getenv('APPVEYOR', False):
  774. script_klass(ci_appveyor)
  775. else:
  776. script_klass(ci_cli)