matrix/matrix.py

   1 # Copyright (C) 2006-2008 Movial Oy
   2 # Authors: Timo Savola <tsavola@movial.fi>
   3 #          Toni Timonen
   4 #          Kalle Vahlman <kalle.vahlman@movial.fi>
   5 #          Tuomas Kulve <tuomas.kulve@movial.fi>
   6
   7 import glob
   8 import os
   9 import re
  10 import sys
  11 import tarfile
  12 from sets import Set as set
  13
  14 import config
  15 import git
  16 from rootfs import RootFS
  17
  18 Error = RuntimeError
  19
  20 def main():
  21         parse_config('config.local')
  22
  23         command, params, targets = parse_args(sys.argv)
  24
  25         parse_config('config')
  26         parse_config('boards')
  27         parse_config('components')
  28
  29         if 'global-cache' in config.flags:
  30                 config.cache_dir = config.global_cache_dir
  31
  32         if not os.path.exists(config.cache_dir):
  33                 os.makedirs(config.cache_dir)
  34
  35         all_targets = update_components()
  36
  37         for name in targets:
  38                 if name not in config.components:
  39                         raise Error('Component "%s" does not exist' % name)
  40
  41         if not targets:
  42                 targets = all_targets
  43
  44         if command == 'install':
  45                 build_components(targets, **params)
  46         elif command == 'clone':
  47                 clone_components(targets)
  48         elif command == 'clean':
  49                 clean_components(targets)
  50         elif command == 'pull':
  51                 pull_components(targets)
  52         elif command == 'source-dist':
  53                 source_dist_components(targets)
  54         elif command == 'rootfs':
  55                 build_rootfs(**params)
  56         else:
  57                 raise Error('Invalid command: ' + command)
  58
  59 def help(file, args):
  60         print >>file, '''
  61 Copyright (C) 2006-2008 Movial Oy
  62
  63 Usage: %(progname)s [<options>] <command> [<params>] [<components>]
  64
  65 If no components are specified, all of them will be targeted.  All component
  66 metadata will be downloaded regardless of the specified command and components.
  67
  68 Options:
  69         -v              Verbose output.
  70         -d              Debug output.
  71         -r URL          Specify the root for component git repos to clone from.
  72                         If this option is not given, the roots specified in the
  73                         components file will be used.  This option may be
  74                         specified multiple times.
  75         -f              Build components with dirty files.
  76         -h, --help      Print this help text.
  77
  78 Commands and parameters:
  79         clone           Download the components' git repositories.
  80         install         Download, build and install the components.
  81                 -j N            Execute N build jobs in parallel.
  82                 -mj N           Run component builds with make -j N.
  83         clean           Remove all non-tracked files from the component git
  84                         repository directories.
  85         pull            Run git pull for the components.
  86         source-dist     Download and package the component sources.
  87         rootfs          Build a rootfs from the current target installation.
  88                 -n              Do not clean env.faked and stripped directory.
  89                 -r              Only generate a stripped rootfs.
  90                 -j              Only generate a jffs2 image
  91                 -d              Only generate a rootstrap (development rootfs).
  92 ''' % {'progname': args[0]}
  93
  94 def parse_args(args):
  95         def parse_jobs(arg):
  96                 jobs = int(arg)
  97                 if jobs <= 0:
  98                         raise Error('Please specify a valid number of jobs')
  99                 return jobs
 100
 101         command = None
 102         params = {}
 103         targets = []
 104
 105         i = 1
 106         while i < len(args):
 107                 if args[i].startswith('-'):
 108                         if not command:
 109                                 if args[i] == '-v':
 110                                         config.verbose = True
 111                                 elif args[i] == '-d':
 112                                         config.debug = True
 113                                 elif args[i] == '-r':
 114                                         i += 1
 115                                         config.roots.append(args[i])
 116                                 elif args[i] == '-f':
 117                                         config.force = True
 118                                 elif args[i] in ('-h', '--help'):
 119                                         help(sys.stdout, args)
 120                                         sys.exit(0)
 121                                 else:
 122                                         raise Error('Bad option: ' + args[i])
 123                         elif command == 'install':
 124                                 if args[i] == '-j':
 125                                         i += 1
 126                                         params['build_jobs'] = parse_jobs(args[i])
 127                                 elif args[i] == '-mj':
 128                                         i += 1
 129                                         params['make_jobs'] = parse_jobs(args[i])
 130                                 else:
 131                                         raise Error('Bad parameter: ' + args[i])
 132                         elif command == 'rootfs':
 133                                 if args[i] == '-n':
 134                                         i += 1
 135                                         params['clean'] = False
 136                                 elif args[i] == '-r':
 137                                         i += 1
 138                                         params['rootfs_only'] = True
 139                                 elif args[i] == '-j':
 140                                         i += 1
 141                                         params['jffs2_only'] = True
 142                                 elif args[i] == '-d':
 143                                         i += 1
 144                                         params['devrootfs_only'] = True
 145                         else:
 146                                 raise Error('Command takes no parameters')
 147                 elif not command:
 148                         command = args[i]
 149                 else:
 150                         targets.append(args[i])
 151                 i += 1
 152
 153         if not command:
 154                 help(sys.stderr, args)
 155                 sys.exit(1)
 156
 157         return command, params, targets
 158
 159 def parse_config(path):
 160         if os.path.isabs(path):
 161                 path = os.path.join(config.top_dir, path)
 162
 163         if os.path.exists(path):
 164                 print 'Reading', path
 165                 execfile(path, config.__dict__, config.__dict__)
 166
 167 def update_components():
 168         targets = []
 169         packages = {}
 170
 171         for c in config.components.itervalues():
 172                 update_component_url(c)
 173                 clone_metadata(c)
 174                 update_component_packages(c, targets, packages)
 175                 c.active_depends = []
 176
 177         update_component_depends(packages)
 178
 179         return targets
 180
 181 def update_component_url(c):
 182         update_repository_url(c.repo)
 183         update_repository_url(c.meta)
 184
 185 def update_repository_url(repo):
 186         if git.contains_database(repo.path):
 187                 url = git.getvar(repo.path, 'remote.origin.url')
 188                 if url:
 189                         repo.active_url = url
 190                         if config.debug:
 191                                 print 'Using', repo.active_url
 192                         return
 193
 194         for root in config.roots:
 195                 for suffix in ('.git', ''):
 196                         url = '%s/%s%s' % (root, repo.name, suffix)
 197
 198                         if config.debug:
 199                                 print 'Trying', url
 200
 201                         if not git.peek_remote(url, quiet=True):
 202                                 continue
 203
 204                         repo.active_url = url
 205                         if config.debug:
 206                                 print 'Found', repo.active_url
 207                         return
 208
 209         raise Error('Failed to locate repository: ' + repo.name)
 210
 211 def update_component_depends(packages):
 212         for pkg in packages.itervalues():
 213                 for spec in pkg.depends.split():
 214                         depname = Depend(spec).name
 215                         deppkg = packages.get(depname)
 216
 217                         if not deppkg:
 218                                 print >>sys.stderr, 'Package', pkg.name, \
 219                                         'depends on non-existent package', depname
 220                                 continue
 221
 222                         if deppkg.component == pkg.component:
 223                                 continue
 224
 225                         pkg.component.active_depends.append(deppkg.component)
 226
 227         fail = False
 228         for pkg in packages.itervalues():
 229                 for spec in pkg.depends.split():
 230                         if not Depend(spec).check(packages):
 231                                 fail = True
 232                                 print >>sys.stderr, 'Dependency', spec, \
 233                                         'failed for', pkg.name
 234
 235                 for spec in pkg.conflicts.split():
 236                         if Depend(spec).check(packages):
 237                                 fail = True
 238                                 print >>sys.stderr, pkg.name, \
 239                                         'conflicts with', spec
 240
 241         if fail:
 242                 raise Error('Invalid component tree')
 243
 244 class Depend(object):
 245         regex = re.compile(r'([@]?)([^\s:]+)[:]?([<>=]*)([^\s:]*)[:]?(.*)')
 246
 247         def __init__(self, spec):
 248                 match = self.regex.match(spec)
 249                 if not match:
 250                         raise Error('Bad dependency specification: ' + spec)
 251
 252                 self.build, self.name, self.tag_op, self.tag, flags \
 253                         = match.groups()
 254                 self.flags = flags.split()
 255
 256         def check(self, packages):
 257                 # TODO: check version and flags
 258                 return self.name in packages
 259
 260 def execute(args):
 261         if config.debug:
 262                 print 'Executing:', ' '.join(args)
 263
 264         if os.spawnvp(os.P_WAIT, args[0], args) != 0:
 265                 raise Error('Failed: ' + ' '.join(args))
 266
 267 def remove_tree(path):
 268         execute(['rm', '-rf', path])
 269
 270 def clone_components(targets):
 271         if not targets:
 272                 targets = config.components.keys()
 273
 274         for name in targets:
 275                 c = config.components[name]
 276                 clone_component(c)
 277
 278 def clone_component(c, overwrite=False):
 279         have_repo = git.contains_database(c.repo.path)
 280         have_meta = git.contains_database(c.meta.path)
 281
 282         if not overwrite and have_repo and have_meta:
 283                 return
 284
 285         if overwrite and os.path.exists(c.repo.path):
 286                 print 'Removing', c.repo.path
 287
 288                 remove_tree(c.repo.path)
 289                 have_repo = False
 290                 have_meta = False
 291
 292         if not have_repo:
 293                 clone_repository(c.repo)
 294                 git.exclude(c.repo.path, 'meta')
 295
 296         if not have_meta:
 297                 clone_repository(c.meta)
 298
 299 def clone_metadata(c):
 300         if not git.contains_database(c.meta.path):
 301                 clone_repository(c.meta)
 302
 303 def clone_repository(repo):
 304         print 'Cloning', repo.path
 305
 306         if os.path.exists(repo.path):
 307                 tmp = os.path.join(repo.path, 'tmp')
 308                 git.clone(tmp, repo.active_url, checkout=False)
 309                 try:
 310                         tmpdb = os.path.join(tmp, '.git')
 311                         repodb = os.path.join(repo.path, '.git')
 312                         if config.debug:
 313                                 print 'Renaming "%s" as "%s"' % (tmpdb, repodb)
 314                         os.rename(tmpdb, repodb)
 315                 finally:
 316                         os.rmdir(tmp)
 317                 git.checkout(repo.path)
 318         else:
 319                 git.clone(repo.path, repo.active_url, checkout=True)
 320
 321 def update_component_packages(c, targets, packages):
 322         c.active_packages = {}
 323
 324         for path in glob.glob(os.path.join(c.meta.path, '*.package')):
 325                 name = os.path.basename(path)[:-8]
 326
 327                 pkg = parse_package(name, c, path)
 328                 if not pkg:
 329                         continue
 330
 331                 c.active_packages[name] = pkg
 332                 packages[name] = pkg
 333
 334                 if config.debug:
 335                         print 'Component', c.name, 'provides', name
 336
 337         if c.active_packages:
 338                 targets.append(c.name)
 339         elif config.debug:
 340                 print 'Component', c.name, 'does not provide any packages'
 341
 342 class Package(object):
 343         def __init__(self, name, component):
 344                 self.name = name
 345                 self.component = component
 346
 347                 self.depends = []
 348                 self.conflicts = []
 349                 self.architectures = None
 350
 351 def parse_package(name, component, path):
 352         pkg = Package(name, component)
 353         execfile(path, pkg.__dict__, pkg.__dict__)
 354
 355         if pkg.architectures:
 356                 arch = config.boards[config.board].arch
 357                 if arch not in pkg.architectures:
 358                         return None
 359
 360         return pkg
 361
 362 def build_components(targets, build_jobs=1, make_jobs=1):
 363         selected = set([config.components[i] for i in targets])
 364
 365         depends = None
 366         while depends is None or depends:
 367                 depends = set()
 368
 369                 for c in selected:
 370                         for dep in c.active_depends:
 371                                 if dep not in selected:
 372                                         depends.add(dep)
 373
 374                 selected |= depends
 375
 376         components = list(selected)
 377         components.sort()
 378
 379         if config.debug:
 380                 print 'Building components:'
 381                 for c in components:
 382                         print '\t' + c.name
 383
 384         jobs = {}
 385
 386         while True:
 387                 found = False
 388                 for c in components:
 389                         if len(jobs) >= build_jobs:
 390                                 break
 391
 392                         if component_buildable(c):
 393                                 found = True
 394
 395                                 if not component_cached(c):
 396                                         print 'Starting to build', c.name
 397                                         start_job(c, jobs, make_jobs)
 398                                 else:
 399                                         print c.name, 'found from cache'
 400                                         c.active_state = 'built'
 401
 402                 if not found and not jobs:
 403                         for c in components:
 404                                 if c.active_state != 'built':
 405                                         raise Error('Internal error')
 406                         break
 407
 408                 wait_for_job(jobs)
 409
 410 def component_buildable(c):
 411         if c.active_state:
 412                 if config.debug:
 413                         print c.name, 'is in state:', c.active_state
 414                 return False
 415
 416         for dep in c.active_depends:
 417                 if config.components[dep.name].active_state != 'built':
 418                         if config.debug:
 419                                 print c.name, 'depends on unbuilt', dep.name
 420                         return False
 421
 422         return True
 423
 424 def component_cached(c):
 425         if c.from_platform:
 426                 return True
 427
 428         for repo in (c.repo, c.meta):
 429                 if git.ls_files(repo.path, ['-m', '-d']):
 430                         if config.force:
 431                                 return False
 432                         else:
 433                                 raise Error('Dirty files in ' + repo.path)
 434
 435         path = os.path.join(config.cache_dir, c.name)
 436         if not os.path.exists(path):
 437                 return False
 438
 439         file = open(path, 'r')
 440         line = file.readline()
 441         file.close()
 442
 443         match = re.match(r'([^\s]+)[\s]?([^\s]*)', line)
 444         if not match:
 445                 return False
 446
 447         hash, flagstring = match.groups()
 448
 449         if hash != get_component_hash(c):
 450                 print 'Component has been changed:', c.name
 451                 return False
 452
 453         if flagstring:
 454                 flags = flagstring.split(',')
 455         else:
 456                 flags = []
 457
 458         if set(flags) != set(c.flags):
 459                 print 'Component flags have been changed:', c.name
 460                 return False
 461
 462         return True
 463
 464 def update_cache(c):
 465         path = os.path.join(config.cache_dir, c.name)
 466
 467         dir = os.path.dirname(path)
 468         if not os.path.exists(dir):
 469                 os.makedirs(dir)
 470
 471         file = open(path, 'w')
 472         done = False
 473         try:
 474                 print >>file, get_component_hash(c), ','.join(c.flags)
 475                 done = True
 476         finally:
 477                 file.close()
 478                 if not done:
 479                         os.remove(path)
 480
 481 def get_component_hash(c):
 482         return '%s+%s' % (get_repository_hash(c.repo), get_repository_hash(c.meta))
 483
 484 def get_repository_hash(repo):
 485         if repo.active_hash is None:
 486                 repo.active_hash = git.rev_parse(repo.path, 'HEAD')
 487         return repo.active_hash
 488
 489 def start_job(c, jobs, make_jobs):
 490         board = config.boards[config.board]
 491
 492         makefile = os.path.join(config.script_dir, 'matrix.mak')
 493
 494         workdir = os.path.join(config.top_dir, 'src', c.name)
 495         args = ['make', '--no-print-directory', '-f', makefile, '-C', workdir,
 496                 '-j', str(make_jobs), 'build_matrix_component',
 497                 'MATRIX_TOPDIR='    + os.path.abspath(config.top_dir),
 498                 'MATRIX_SCRIPTDIR=' + os.path.abspath(config.script_dir),
 499                 'MATRIX_COMPONENT=' + c.name,
 500                 'MATRIX_ARCH='      + board.arch,
 501                 'MATRIX_GCC_MARCH=' + board.gcc_march,
 502                 'MATRIX_GCC_MCPU='  + board.gcc_mcpu,
 503                 'MATRIX_GCC_MFPU='  + board.gcc_mfpu,
 504                 'MATRIX_GCC_OPTIONS=' + board.gcc_options,
 505                 'MATRIX_GNU_HOST='  + board.gnu_host,
 506                 'MATRIX_LIBC='      + config.libc]
 507
 508         for flag in config.flags:
 509                 args.append(flag + '=1')
 510
 511         if config.verbose:
 512                 args.append('MATRIX_VERBOSE=1')
 513
 514         if config.debug:
 515                 print 'Executing:', ' '.join(args)
 516
 517         pid = os.spawnvp(os.P_NOWAIT, args[0], args)
 518
 519         jobs[pid] = c
 520         c.active_state = 'running'
 521
 522 def wait_for_job(jobs):
 523         if not jobs:
 524                 return
 525
 526         pid, status = os.wait()
 527
 528         c = jobs.get(pid)
 529         if not c:
 530                 return
 531
 532         del jobs[pid]
 533
 534         if status == 0:
 535                 c.active_state = 'built'
 536                 update_cache(c)
 537                 print c.name, 'completed'
 538         else:
 539                 c.active_state = 'error'
 540                 print >>sys.stderr, c.name, 'failed with status:', status
 541
 542                 wait_for_job(jobs)
 543                 raise Error('Build failed')
 544
 545 def clean_components(targets):
 546         if not targets:
 547                 targets = config.components.keys()
 548
 549         for name in targets:
 550                 c = config.components[name]
 551                 print 'Cleaning', c.repo.path
 552
 553                 files = git.ls_files(c.repo.path, ['-o'])
 554                 paths = [os.path.join(c.repo.path, i) for i in files]
 555                 paths.sort()
 556                 paths.reverse()
 557
 558                 cache = os.path.join(config.cache_dir, name)
 559                 if os.path.exists(cache):
 560                         paths.append(cache)
 561
 562                 for path in paths:
 563                         if git.contains_database(path):
 564                                 continue
 565
 566                         if config.debug:
 567                                 print 'Removing', path
 568
 569                         if os.path.islink(path) or not os.path.isdir(path):
 570                                 os.remove(path)
 571                         else:
 572                                 remove_tree(path)
 573
 574                 for repo in (c.repo, c.meta):
 575                         files = git.ls_files(repo.path, ['-m', '-d'])
 576                         if files:
 577                                 print len(files), 'dirty files left in', repo.path
 578
 579 def pull_components(targets):
 580         if not targets:
 581                 targets = config.components.keys()
 582
 583         for name in targets:
 584                 print 'Pulling component:', name
 585
 586                 c = config.components[name]
 587                 git.pull(c.repo.path)
 588                 git.pull(c.meta.path)
 589
 590 def source_dist_components(targets):
 591         if not targets:
 592                 targets = config.components.keys()
 593
 594         for name in targets:
 595                 c = config.components[name]
 596                 generate_component_changes(c, 'dist')
 597                 package_component_sources(c, 'dist')
 598
 599 def generate_component_changes(c, location):
 600         print 'Generating change log for', c.repo.path
 601
 602         path = os.path.join(location, c.name) + '.changes'
 603
 604         pathdir = os.path.dirname(path)
 605         if not os.path.exists(pathdir):
 606                 os.makedirs(pathdir)
 607
 608         fd = os.open(path, os.O_WRONLY | os.O_CREAT, 0644)
 609         git.log(c.repo.path, [c.get_active_tag()], fd=fd)
 610         os.close(fd)
 611
 612 def package_component_sources(c, location):
 613         print 'Archiving', c.repo.path
 614
 615         rev = git.describe(c.repo.path)
 616         if rev:
 617                 rev = '_' + rev
 618         else:
 619                 rev = ''
 620
 621         path = os.path.join(location, c.name) + rev + '.tar.bz2'
 622         if os.path.exists(path):
 623                 os.remove(path)
 624
 625         pathdir = os.path.dirname(path)
 626         if not os.path.exists(pathdir):
 627                 os.makedirs(pathdir)
 628
 629         git.archive(c.repo.path, path,
 630                     prefix=os.path.basename(c.name) + '/',
 631                     branch=c.get_active_tag())
 632
 633 def build_rootfs(clean=True, rootfs_only=False, jffs2_only=False, devrootfs_only=False):
 634         parse_config('rootfs')
 635
 636         f = os.popen('sb-conf cu')
 637         target = f.read()
 638         f.close()
 639
 640         rootfs = RootFS(target.strip())
 641         rootfs.include_paths(config.include_paths)
 642         rootfs.include_files(config.include_files)
 643         rootfs.filter_paths(config.exclude_paths)
 644         rootfs.filter_files(config.exclude_files)
 645         rootfs.filter_expressions(config.exclude_expressions)
 646         rootfs.add_paths(config.created_paths)
 647         rootfs.set_devices(config.devices)
 648         rootfs.set_change_owner(config.change_owner)
 649         rootfs.set_erase_size(config.boards[config.board].flash_erase_size)
 650         rootfs.set_pad_size(config.boards[config.board].flash_pad_size)
 651
 652         if rootfs_only:
 653                 rootfs.generate(clean, build_target="rootfs")
 654         elif jffs2_only:
 655                 rootfs.generate(clean, build_target="jffs2")
 656         elif devrootfs_only:
 657                 rootfs.generate(clean, build_target="devrootfs")
 658         else:
 659                 rootfs.generate(clean, build_target="all")