1 # Copyright (C) 2006-2008 Movial Oy
2 # Authors: Timo Savola <tsavola@movial.fi>
4 # Kalle Vahlman <kalle.vahlman@movial.fi>
5 # Tuomas Kulve <tuomas.kulve@movial.fi>
12 from sets import Set as set
16 from rootfs import RootFS
21 parse_config('config.local')
23 command, params, targets = parse_args(sys.argv)
25 parse_config('config')
26 parse_config('boards')
27 parse_config('components')
29 if 'global-cache' in config.flags:
30 config.cache_dir = config.global_cache_dir
32 if not os.path.exists(config.cache_dir):
33 os.makedirs(config.cache_dir)
35 all_targets = update_components()
38 if name not in config.components:
39 raise Error('Component "%s" does not exist' % name)
44 if command == 'install':
45 build_components(targets, **params)
46 elif command == 'clone':
47 clone_components(targets)
48 elif command == 'clean':
49 clean_components(targets)
50 elif command == 'pull':
51 pull_components(targets)
52 elif command == 'source-dist':
53 source_dist_components(targets)
54 elif command == 'rootfs':
55 build_rootfs(**params)
57 raise Error('Invalid command: ' + command)
61 Copyright (C) 2006-2008 Movial Oy
63 Usage: %(progname)s [<options>] <command> [<params>] [<components>]
65 If no components are specified, all of them will be targeted. All component
66 metadata will be downloaded regardless of the specified command and components.
71 -r URL Specify the root for component git repos to clone from.
72 If this option is not given, the roots specified in the
73 components file will be used. This option may be
74 specified multiple times.
75 -f Build components with dirty files.
76 -h, --help Print this help text.
78 Commands and parameters:
79 clone Download the components' git repositories.
80 install Download, build and install the components.
81 -j N Execute N build jobs in parallel.
82 -mj N Run component builds with make -j N.
83 clean Remove all non-tracked files from the component git
84 repository directories.
85 pull Run git pull for the components.
86 source-dist Download and package the component sources.
87 rootfs Build a rootfs from the current target installation.
88 -n Do not clean env.faked and stripped directory.
89 -r Only generate a stripped rootfs.
90 -j Only generate a jffs2 image
91 -d Only generate a rootstrap (development rootfs).
92 ''' % {'progname': args[0]}
98 raise Error('Please specify a valid number of jobs')
107 if args[i].startswith('-'):
110 config.verbose = True
111 elif args[i] == '-d':
113 elif args[i] == '-r':
115 config.roots.append(args[i])
116 elif args[i] == '-f':
118 elif args[i] in ('-h', '--help'):
119 help(sys.stdout, args)
122 raise Error('Bad option: ' + args[i])
123 elif command == 'install':
126 params['build_jobs'] = parse_jobs(args[i])
127 elif args[i] == '-mj':
129 params['make_jobs'] = parse_jobs(args[i])
131 raise Error('Bad parameter: ' + args[i])
132 elif command == 'rootfs':
135 params['clean'] = False
136 elif args[i] == '-r':
138 params['rootfs_only'] = True
139 elif args[i] == '-j':
141 params['jffs2_only'] = True
142 elif args[i] == '-d':
144 params['devrootfs_only'] = True
146 raise Error('Command takes no parameters')
150 targets.append(args[i])
154 help(sys.stderr, args)
157 return command, params, targets
159 def parse_config(path):
160 if os.path.isabs(path):
161 path = os.path.join(config.top_dir, path)
163 if os.path.exists(path):
164 print 'Reading', path
165 execfile(path, config.__dict__, config.__dict__)
167 def update_components():
171 for c in config.components.itervalues():
172 update_component_url(c)
174 update_component_packages(c, targets, packages)
175 c.active_depends = []
177 update_component_depends(packages)
181 def update_component_url(c):
182 update_repository_url(c.repo)
183 update_repository_url(c.meta)
185 def update_repository_url(repo):
186 if git.contains_database(repo.path):
187 url = git.getvar(repo.path, 'remote.origin.url')
189 repo.active_url = url
191 print 'Using', repo.active_url
194 for root in config.roots:
195 for suffix in ('.git', ''):
196 url = '%s/%s%s' % (root, repo.name, suffix)
201 if not git.peek_remote(url, quiet=True):
204 repo.active_url = url
206 print 'Found', repo.active_url
209 raise Error('Failed to locate repository: ' + repo.name)
211 def update_component_depends(packages):
212 for pkg in packages.itervalues():
213 for spec in pkg.depends.split():
214 depname = Depend(spec).name
215 deppkg = packages.get(depname)
218 print >>sys.stderr, 'Package', pkg.name, \
219 'depends on non-existent package', depname
222 if deppkg.component == pkg.component:
225 pkg.component.active_depends.append(deppkg.component)
228 for pkg in packages.itervalues():
229 for spec in pkg.depends.split():
230 if not Depend(spec).check(packages):
232 print >>sys.stderr, 'Dependency', spec, \
233 'failed for', pkg.name
235 for spec in pkg.conflicts.split():
236 if Depend(spec).check(packages):
238 print >>sys.stderr, pkg.name, \
239 'conflicts with', spec
242 raise Error('Invalid component tree')
244 class Depend(object):
245 regex = re.compile(r'([@]?)([^\s:]+)[:]?([<>=]*)([^\s:]*)[:]?(.*)')
247 def __init__(self, spec):
248 match = self.regex.match(spec)
250 raise Error('Bad dependency specification: ' + spec)
252 self.build, self.name, self.tag_op, self.tag, flags \
254 self.flags = flags.split()
256 def check(self, packages):
257 # TODO: check version and flags
258 return self.name in packages
262 print 'Executing:', ' '.join(args)
264 if os.spawnvp(os.P_WAIT, args[0], args) != 0:
265 raise Error('Failed: ' + ' '.join(args))
267 def remove_tree(path):
268 execute(['rm', '-rf', path])
270 def clone_components(targets):
272 targets = config.components.keys()
275 c = config.components[name]
278 def clone_component(c, overwrite=False):
279 have_repo = git.contains_database(c.repo.path)
280 have_meta = git.contains_database(c.meta.path)
282 if not overwrite and have_repo and have_meta:
285 if overwrite and os.path.exists(c.repo.path):
286 print 'Removing', c.repo.path
288 remove_tree(c.repo.path)
293 clone_repository(c.repo)
294 git.exclude(c.repo.path, 'meta')
297 clone_repository(c.meta)
299 def clone_metadata(c):
300 if not git.contains_database(c.meta.path):
301 clone_repository(c.meta)
303 def clone_repository(repo):
304 print 'Cloning', repo.path
306 if os.path.exists(repo.path):
307 tmp = os.path.join(repo.path, 'tmp')
308 git.clone(tmp, repo.active_url, checkout=False)
310 tmpdb = os.path.join(tmp, '.git')
311 repodb = os.path.join(repo.path, '.git')
313 print 'Renaming "%s" as "%s"' % (tmpdb, repodb)
314 os.rename(tmpdb, repodb)
317 git.checkout(repo.path)
319 git.clone(repo.path, repo.active_url, checkout=True)
321 def update_component_packages(c, targets, packages):
322 c.active_packages = {}
324 for path in glob.glob(os.path.join(c.meta.path, '*.package')):
325 name = os.path.basename(path)[:-8]
327 pkg = parse_package(name, c, path)
331 c.active_packages[name] = pkg
335 print 'Component', c.name, 'provides', name
337 if c.active_packages:
338 targets.append(c.name)
340 print 'Component', c.name, 'does not provide any packages'
342 class Package(object):
343 def __init__(self, name, component):
345 self.component = component
349 self.architectures = None
351 def parse_package(name, component, path):
352 pkg = Package(name, component)
353 execfile(path, pkg.__dict__, pkg.__dict__)
355 if pkg.architectures:
356 arch = config.boards[config.board].arch
357 if arch not in pkg.architectures:
362 def build_components(targets, build_jobs=1, make_jobs=1):
363 selected = set([config.components[i] for i in targets])
366 while depends is None or depends:
370 for dep in c.active_depends:
371 if dep not in selected:
376 components = list(selected)
380 print 'Building components:'
389 if len(jobs) >= build_jobs:
392 if component_buildable(c):
395 if not component_cached(c):
396 print 'Starting to build', c.name
397 start_job(c, jobs, make_jobs)
399 print c.name, 'found from cache'
400 c.active_state = 'built'
402 if not found and not jobs:
404 if c.active_state != 'built':
405 raise Error('Internal error')
410 def component_buildable(c):
413 print c.name, 'is in state:', c.active_state
416 for dep in c.active_depends:
417 if config.components[dep.name].active_state != 'built':
419 print c.name, 'depends on unbuilt', dep.name
424 def component_cached(c):
428 for repo in (c.repo, c.meta):
429 if git.ls_files(repo.path, ['-m', '-d']):
433 raise Error('Dirty files in ' + repo.path)
435 path = os.path.join(config.cache_dir, c.name)
436 if not os.path.exists(path):
439 file = open(path, 'r')
440 line = file.readline()
443 match = re.match(r'([^\s]+)[\s]?([^\s]*)', line)
447 hash, flagstring = match.groups()
449 if hash != get_component_hash(c):
450 print 'Component has been changed:', c.name
454 flags = flagstring.split(',')
458 if set(flags) != set(c.flags):
459 print 'Component flags have been changed:', c.name
465 path = os.path.join(config.cache_dir, c.name)
467 dir = os.path.dirname(path)
468 if not os.path.exists(dir):
471 file = open(path, 'w')
474 print >>file, get_component_hash(c), ','.join(c.flags)
481 def get_component_hash(c):
482 return '%s+%s' % (get_repository_hash(c.repo), get_repository_hash(c.meta))
484 def get_repository_hash(repo):
485 if repo.active_hash is None:
486 repo.active_hash = git.rev_parse(repo.path, 'HEAD')
487 return repo.active_hash
489 def start_job(c, jobs, make_jobs):
490 board = config.boards[config.board]
492 makefile = os.path.join(config.script_dir, 'matrix.mak')
494 workdir = os.path.join(config.top_dir, 'src', c.name)
495 args = ['make', '--no-print-directory', '-f', makefile, '-C', workdir,
496 '-j', str(make_jobs), 'build_matrix_component',
497 'MATRIX_TOPDIR=' + os.path.abspath(config.top_dir),
498 'MATRIX_SCRIPTDIR=' + os.path.abspath(config.script_dir),
499 'MATRIX_COMPONENT=' + c.name,
500 'MATRIX_ARCH=' + board.arch,
501 'MATRIX_GCC_MARCH=' + board.gcc_march,
502 'MATRIX_GCC_MCPU=' + board.gcc_mcpu,
503 'MATRIX_GCC_MFPU=' + board.gcc_mfpu,
504 'MATRIX_GCC_OPTIONS=' + board.gcc_options,
505 'MATRIX_GNU_HOST=' + board.gnu_host,
506 'MATRIX_LIBC=' + config.libc]
508 for flag in config.flags:
509 args.append(flag + '=1')
512 args.append('MATRIX_VERBOSE=1')
515 print 'Executing:', ' '.join(args)
517 pid = os.spawnvp(os.P_NOWAIT, args[0], args)
520 c.active_state = 'running'
522 def wait_for_job(jobs):
526 pid, status = os.wait()
535 c.active_state = 'built'
537 print c.name, 'completed'
539 c.active_state = 'error'
540 print >>sys.stderr, c.name, 'failed with status:', status
543 raise Error('Build failed')
545 def clean_components(targets):
547 targets = config.components.keys()
550 c = config.components[name]
551 print 'Cleaning', c.repo.path
553 files = git.ls_files(c.repo.path, ['-o'])
554 paths = [os.path.join(c.repo.path, i) for i in files]
558 cache = os.path.join(config.cache_dir, name)
559 if os.path.exists(cache):
563 if git.contains_database(path):
567 print 'Removing', path
569 if os.path.islink(path) or not os.path.isdir(path):
574 for repo in (c.repo, c.meta):
575 files = git.ls_files(repo.path, ['-m', '-d'])
577 print len(files), 'dirty files left in', repo.path
579 def pull_components(targets):
581 targets = config.components.keys()
584 print 'Pulling component:', name
586 c = config.components[name]
587 git.pull(c.repo.path)
588 git.pull(c.meta.path)
590 def source_dist_components(targets):
592 targets = config.components.keys()
595 c = config.components[name]
596 generate_component_changes(c, 'dist')
597 package_component_sources(c, 'dist')
599 def generate_component_changes(c, location):
600 print 'Generating change log for', c.repo.path
602 path = os.path.join(location, c.name) + '.changes'
604 pathdir = os.path.dirname(path)
605 if not os.path.exists(pathdir):
608 fd = os.open(path, os.O_WRONLY | os.O_CREAT, 0644)
609 git.log(c.repo.path, [c.get_active_tag()], fd=fd)
612 def package_component_sources(c, location):
613 print 'Archiving', c.repo.path
615 rev = git.describe(c.repo.path)
621 path = os.path.join(location, c.name) + rev + '.tar.bz2'
622 if os.path.exists(path):
625 pathdir = os.path.dirname(path)
626 if not os.path.exists(pathdir):
629 git.archive(c.repo.path, path,
630 prefix=os.path.basename(c.name) + '/',
631 branch=c.get_active_tag())
633 def build_rootfs(clean=True, rootfs_only=False, jffs2_only=False, devrootfs_only=False):
634 parse_config('rootfs')
636 f = os.popen('sb-conf cu')
640 rootfs = RootFS(target.strip())
641 rootfs.include_paths(config.include_paths)
642 rootfs.include_files(config.include_files)
643 rootfs.filter_paths(config.exclude_paths)
644 rootfs.filter_files(config.exclude_files)
645 rootfs.filter_expressions(config.exclude_expressions)
646 rootfs.add_paths(config.created_paths)
647 rootfs.set_devices(config.devices)
648 rootfs.set_change_owner(config.change_owner)
649 rootfs.set_erase_size(config.boards[config.board].flash_erase_size)
650 rootfs.set_pad_size(config.boards[config.board].flash_pad_size)
653 rootfs.generate(clean, build_target="rootfs")
655 rootfs.generate(clean, build_target="jffs2")
657 rootfs.generate(clean, build_target="devrootfs")
659 rootfs.generate(clean, build_target="all")