return an error code when build fails
[matrix.git] / matrix / matrix.py
1 # Copyright (C) 2006-2008 Movial Oy
2 # Authors: Timo Savola <tsavola@movial.fi>
3 #          Toni Timonen
4 #          Kalle Vahlman <kalle.vahlman@movial.fi>
5 #          Tuomas Kulve <tuomas.kulve@movial.fi>
6
7 import glob
8 import os
9 import re
10 import signal
11 import sys
12 import tarfile
13 from sets import Set as set
14
15 import cache
16 import config
17 import git
18 from rootfs import RootFS
19
20 Error = RuntimeError
21
22 error_log = []
23
24 def log_error(msg):
25         error_log.append(msg)
26
27 def main():
28         try:
29                 try:
30                         logged_main()
31                 except:
32                         if not error_log:
33                                 print >>sys.stderr
34                         raise
35         finally:
36                 if error_log:
37                         print >>sys.stderr
38                         for msg in error_log:
39                                 print >>sys.stderr, msg
40
41 def logged_main():
42         parse_config('config.local')
43
44         command, params, targets = parse_args(sys.argv)
45
46         parse_config('config')
47         parse_config('boards')
48         parse_config('components')
49
50         if 'global-cache' in config.flags:
51                 config.cache_dir = config.global_cache_dir
52
53         if not os.path.exists(config.cache_dir):
54                 os.makedirs(config.cache_dir)
55
56         all_targets = update_components()
57
58         for name in targets:
59                 if name not in config.components:
60                         raise Error('Component "%s" does not exist' % name)
61
62         if not targets:
63                 targets = all_targets
64
65         if command == 'install':
66                 build_components(targets, **params)
67         elif command == 'clone':
68                 clone_components(targets)
69         elif command == 'clean':
70                 clean_components(targets)
71         elif command == 'rebase':
72                 rebase_components(targets)
73         elif command == 'pull':
74                 pull_components(targets)
75         elif command == 'source-dist':
76                 source_dist_components(targets)
77         elif command == 'rootfs':
78                 build_rootfs(**params)
79         else:
80                 raise Error('Invalid command: ' + command)
81
82 def help(file, args):
83         print >>file, '''
84 Copyright (C) 2006-2008 Movial Oy
85
86 Usage: %(progname)s [<options>] <command> [<params>] [<components>]
87
88 If no components are specified, all of them will be targeted.  All component
89 metadata will be downloaded regardless of the specified command and components.
90
91 Options:
92         -v              Verbose output.
93         -d              Debug output.
94         -r URL          Specify the root for component git repos to clone from.
95                         If this option is not given, the roots specified in the
96                         components file will be used.  This option may be
97                         specified multiple times.
98         -f              Build components with dirty files.
99         -h, --help      Print this help text.
100
101 Commands and parameters:
102         clone           Download the components' git repositories.
103         install         Download, build and install the components.
104                 -j N            Execute N build jobs in parallel.
105                 -mj N           Run component builds with make -j N.
106         clean           Remove all non-tracked files from the component git
107                         repository directories.
108         rebase          Update repositories from server by rebasing.
109         pull            Update repositories from server by merging.
110         source-dist     Download and package the component sources.
111         rootfs          Build a rootfs from the current target installation.
112                 -n              Do not clean env.faked and stripped directory.
113                 -r              Only generate a stripped rootfs.
114                 -j              Only generate a jffs2 image
115                 -d              Only generate a rootstrap (development rootfs).
116 ''' % {'progname': args[0]}
117
118 def parse_args(args):
119         def parse_jobs(arg):
120                 jobs = int(arg)
121                 if jobs <= 0:
122                         raise Error('Please specify a valid number of jobs')
123                 return jobs
124
125         command = None
126         params = {}
127         targets = []
128
129         i = 1
130         while i < len(args):
131                 if args[i].startswith('-'):
132                         if not command:
133                                 if args[i] == '-v':
134                                         config.verbose = True
135                                 elif args[i] == '-d':
136                                         config.debug = True
137                                 elif args[i] == '-r':
138                                         i += 1
139                                         config.roots.append(args[i])
140                                 elif args[i] == '-f':
141                                         config.force = True
142                                 elif args[i] in ('-h', '--help'):
143                                         help(sys.stdout, args)
144                                         sys.exit(0)
145                                 else:
146                                         raise Error('Bad option: ' + args[i])
147                         elif command == 'install':
148                                 if args[i] == '-j':
149                                         i += 1
150                                         params['build_jobs'] = parse_jobs(args[i])
151                                 elif args[i] == '-mj':
152                                         i += 1
153                                         params['make_jobs'] = parse_jobs(args[i])
154                                 else:
155                                         raise Error('Bad parameter: ' + args[i])
156                         elif command == 'rootfs':
157                                 if args[i] == '-n':
158                                         i += 1
159                                         params['clean'] = False
160                                 elif args[i] == '-r':
161                                         i += 1
162                                         params['rootfs_only'] = True
163                                 elif args[i] == '-j':
164                                         i += 1
165                                         params['jffs2_only'] = True
166                                 elif args[i] == '-d':
167                                         i += 1
168                                         params['devrootfs_only'] = True
169                         else:
170                                 raise Error('Command takes no parameters')
171                 elif not command:
172                         command = args[i]
173                 else:
174                         targets.append(args[i])
175                 i += 1
176
177         if not command:
178                 help(sys.stderr, args)
179                 sys.exit(1)
180
181         return command, params, targets
182
183 def parse_config(path):
184         if os.path.isabs(path):
185                 path = os.path.join(config.top_dir, path)
186
187         if os.path.exists(path):
188                 print 'Reading', path
189                 execfile(path, config.__dict__, config.__dict__)
190
191 def update_components():
192         targets = []
193         packages = {}
194
195         for c in config.components.itervalues():
196                 update_component_url(c)
197                 clone_metadata(c)
198                 update_component_packages(c, targets, packages)
199                 c.active_depends = []
200
201         update_component_depends(packages)
202
203         return targets
204
205 def update_component_url(c):
206         update_repository_url(c.repo)
207         update_repository_url(c.meta)
208
209 def update_repository_url(repo):
210         if git.contains_database(repo.path):
211                 url = git.getvar(repo.path, 'remote.origin.url')
212                 if url:
213                         repo.active_url = url
214                         if config.debug:
215                                 print 'Using', repo.active_url
216                         return
217
218         for root in config.roots:
219                 for suffix in ('.git', ''):
220                         url = '%s/%s%s' % (root, repo.name, suffix)
221
222                         if config.debug:
223                                 print 'Trying', url
224
225                         if not git.peek_remote(url, quiet=True):
226                                 continue
227
228                         repo.active_url = url
229                         if config.debug:
230                                 print 'Found', repo.active_url
231                         return
232
233         raise Error('Failed to locate repository: ' + repo.name)
234
235 def update_component_depends(packages):
236         for pkg in packages.itervalues():
237                 for spec in pkg.depends.split():
238                         depname = Depend(spec).name
239                         deppkg = packages.get(depname)
240
241                         if not deppkg:
242                                 log_error('Package %s depends on ' \
243                                           'non-existent package %s', \
244                                           pkg.name, depname)
245                                 continue
246
247                         if deppkg.component == pkg.component:
248                                 continue
249
250                         pkg.component.active_depends.append(deppkg.component)
251
252         fail = False
253         for pkg in packages.itervalues():
254                 for spec in pkg.depends.split():
255                         if not Depend(spec).check(packages):
256                                 fail = True
257                                 log_error('Dependency %s failed for %s' % \
258                                           (spec, pkg.name))
259
260                 for spec in pkg.conflicts.split():
261                         if Depend(spec).check(packages):
262                                 fail = True
263                                 log_error('Package %s conflicts with %s' % \
264                                           (pkg.name, spec))
265
266         if fail:
267                 raise Error('Invalid component tree')
268
269 class Depend(object):
270         regex = re.compile(r'([@]?)([^\s:]+)[:]?([<>=]*)([^\s:]*)[:]?(.*)')
271
272         def __init__(self, spec):
273                 match = self.regex.match(spec)
274                 if not match:
275                         raise Error('Bad dependency specification: ' + spec)
276
277                 self.build, self.name, self.tag_op, self.tag, flags \
278                         = match.groups()
279                 self.flags = flags.split()
280
281         def check(self, packages):
282                 # TODO: check version and flags
283                 return self.name in packages
284
285 def execute(args):
286         if config.debug:
287                 print 'Executing:', ' '.join(args)
288
289         if os.spawnvp(os.P_WAIT, args[0], args) != 0:
290                 raise Error('Failed: ' + ' '.join(args))
291
292 def remove_tree(path):
293         execute(['rm', '-rf', path])
294
295 def clone_components(targets):
296         if not targets:
297                 targets = config.components.keys()
298
299         for name in targets:
300                 c = config.components[name]
301                 clone_component(c)
302
303 def clone_component(c, overwrite=False):
304         have_repo = git.contains_database(c.repo.path)
305         have_meta = git.contains_database(c.meta.path)
306
307         if not overwrite and have_repo and have_meta:
308                 return
309
310         if overwrite and os.path.exists(c.repo.path):
311                 print 'Removing', c.repo.path
312
313                 remove_tree(c.repo.path)
314                 have_repo = False
315                 have_meta = False
316
317         if not have_repo:
318                 clone_repository(c.repo)
319                 git.exclude(c.repo.path, 'meta')
320
321         if not have_meta:
322                 clone_repository(c.meta)
323
324 def clone_metadata(c):
325         if not git.contains_database(c.meta.path):
326                 clone_repository(c.meta)
327
328 def clone_repository(repo):
329         print 'Cloning', repo.path
330
331         if os.path.exists(repo.path):
332                 tmp = os.path.join(repo.path, 'tmp')
333                 git.clone(tmp, repo.active_url, checkout=False)
334                 try:
335                         tmpdb = os.path.join(tmp, '.git')
336                         repodb = os.path.join(repo.path, '.git')
337                         if config.debug:
338                                 print 'Renaming "%s" as "%s"' % (tmpdb, repodb)
339                         os.rename(tmpdb, repodb)
340                 finally:
341                         os.rmdir(tmp)
342                 git.checkout(repo.path)
343         else:
344                 git.clone(repo.path, repo.active_url, checkout=True)
345
346 def update_component_packages(c, targets, packages):
347         c.active_packages = {}
348
349         for path in glob.glob(os.path.join(c.meta.path, '*.package')):
350                 name = os.path.basename(path)[:-8]
351
352                 pkg = parse_package(name, c, path)
353                 if not pkg:
354                         continue
355
356                 c.active_packages[name] = pkg
357                 packages[name] = pkg
358
359                 if config.debug:
360                         print 'Component', c.name, 'provides', name
361
362         if c.active_packages:
363                 targets.append(c.name)
364         elif config.debug:
365                 print 'Component', c.name, 'does not provide any packages'
366
367 class Package(object):
368         def __init__(self, name, component):
369                 self.name = name
370                 self.component = component
371
372                 self.depends = []
373                 self.conflicts = []
374                 self.architectures = None
375
376 def parse_package(name, component, path):
377         pkg = Package(name, component)
378         execfile(path, pkg.__dict__, pkg.__dict__)
379
380         if pkg.architectures:
381                 arch = config.boards[config.board].arch
382                 if arch not in pkg.architectures:
383                         return None
384
385         return pkg
386
387 def build_components(targets, build_jobs=1, make_jobs=1):
388         selected = set([config.components[i] for i in targets])
389
390         depends = None
391         while depends is None or depends:
392                 depends = set()
393
394                 for c in selected:
395                         for dep in c.active_depends:
396                                 if dep not in selected:
397                                         depends.add(dep)
398
399                 selected |= depends
400
401         components = list(selected)
402         components.sort()
403
404         if config.debug:
405                 print 'Building components:'
406                 for c in components:
407                         print '\t' + c.name
408
409         builder = Builder(components, build_jobs, make_jobs)
410         builder.run()
411
412 class Builder(object):
413         def __init__(self, components, max_jobs, max_make_jobs):
414                 self.max_jobs = max_jobs
415                 self.max_make_jobs = max_make_jobs
416
417                 self.jobs = {}
418
419                 self.wait_build = set(components)
420                 self.wait_install = []
421                 self.in_install = None
422
423                 self.progress_now = 0
424                 self.progress_total = len(components) * 2
425
426                 self.error = False
427
428         def run(self):
429                 try:
430                         while self.run_iteration():
431                                 pass
432                 except:
433                         for pid in self.jobs:
434                                 try:
435                                         os.kill(pid, signal.SIGTERM)
436                                 except OSError, e:
437                                         log_error('kill: %s' % e)
438
439                                 c = self.jobs[pid]
440                                 c.active_state = 'killed'
441
442                         while self.jobs:
443                                 self.wait()
444
445                         raise
446
447                 if self.error:
448                         raise Error()
449
450         def run_iteration(self):
451                 if self.can_start_install():
452                         c = self.wait_install.pop(0)
453                         self.start_install(c)
454                         self.in_install = c
455
456                 while self.can_start_build():
457                         found = False
458
459                         for c in self.wait_build:
460                                 if self.is_buildable(c):
461                                         found = True
462
463                                         if not self.try_cache(c):
464                                                 self.start_build(c)
465
466                                         self.wait_build.remove(c)
467                                         break
468
469                         if not found:
470                                 break
471
472                 if self.jobs:
473                         self.wait()
474                         return True
475                 else:
476                         if self.wait_build and not self.error:
477                                 raise Error('Circular dependencies?')
478                         return False
479
480         def progress(self, increment=1):
481                 self.progress_now += increment
482
483         def message(self, arg1, arg2=None):
484                 if arg2 is None:
485                         message = arg1
486                 else:
487                         message = '%-10s %s' % (arg1, arg2)
488
489                 progress = '%3d%%' % \
490                            (self.progress_now *100 / self.progress_total)
491
492                 print progress, message
493
494         def is_buildable(self, c):
495                 if c.active_state:
496                         if config.debug:
497                                 self.message('Component %s is in state: %s' % \
498                                         (c.name, c.active_state))
499                         return False
500
501                 for dep in c.active_depends:
502                         if config.components[dep.name].active_state != 'done':
503                                 if config.debug:
504                                         self.message('Component %s depends ' \
505                                                 'on uninstalled %s' % \
506                                                 (c.name, dep.name))
507                                 return False
508
509                 return True
510
511         def try_cache(self, c):
512                 if not cache.contains(c):
513                         return False
514
515                 c.active_state = 'done'
516                 self.progress(2)
517
518                 if config.debug:
519                         self.message('Cached', c.repo.path)
520
521                 return True
522
523         def can_start_build(self):
524                 return self.can_start_job()
525
526         def can_start_install(self):
527                 return not self.in_install and self.wait_install and \
528                        self.can_start_job()
529
530         def can_start_job(self):
531                 return not self.error and len(self.jobs) < self.max_jobs
532
533         def start_build(self, c):
534                 self.message('Building', c.repo.path)
535                 self.start_job(c, 'build_matrix_component', 'in-build')
536
537         def start_install(self, c):
538                 self.message('Installing', c.repo.path)
539                 self.start_job(c, 'install_matrix_component', 'in-install')
540
541         def start_job(self, c, make_target, state):
542                 pid = spawn_make(c, self.max_make_jobs, make_target)
543                 self.jobs[pid] = c
544                 c.active_state = state
545
546         def wait(self):
547                 pid, status = os.wait()
548
549                 if status:
550                         self.error = True
551
552                 c = self.jobs[pid]
553                 del self.jobs[pid]
554
555                 if c.active_state == 'in-build':
556                         if status:
557                                 log_error('Failed to build %s' % c.name)
558                                 return
559
560                         self.progress()
561                         if config.debug:
562                                 self.message('Built', c.repo.path)
563
564                         c.active_state = 'wait-install'
565                         self.wait_install.append(c)
566
567                 elif c.active_state == 'in-install':
568                         assert c == self.in_install
569
570                         if status:
571                                 log_error('Failed to install %s' % c.name)
572                                 return
573
574                         self.progress()
575                         self.message('Finished', c.repo.path)
576
577                         c.active_state = 'done'
578                         self.in_install = None
579                         cache.update(c)
580
581                 elif c.active_state != 'killed':
582                         raise Error('Unexpected state: %s' % c.active_state)
583
584 def spawn_make(c, jobs, target):
585         board = config.boards[config.board]
586
587         make = os.getenv('MAKE', 'make')
588         makefile = os.path.join(config.script_dir, 'matrix.mak')
589         workdir = os.path.join(config.top_dir, 'src', c.name)
590         args = [make, '--no-print-directory', '-f', makefile, '-C', workdir,
591                 '-j', str(jobs), target,
592                 'MATRIX_TOPDIR='    + os.path.abspath(config.top_dir),
593                 'MATRIX_SCRIPTDIR=' + os.path.abspath(config.script_dir),
594                 'MATRIX_COMPONENT=' + c.name,
595                 'MATRIX_ARCH='      + board.arch,
596                 'MATRIX_GCC_MARCH=' + board.gcc_march,
597                 'MATRIX_GCC_MCPU='  + board.gcc_mcpu,
598                 'MATRIX_GCC_MFPU='  + board.gcc_mfpu,
599                 'MATRIX_GCC_OPTIONS=' + board.gcc_options,
600                 'MATRIX_GNU_HOST='  + board.gnu_host,
601                 'MATRIX_LIBC='      + config.libc]
602
603         for flag in config.flags:
604                 args.append(flag + '=1')
605
606         if config.verbose:
607                 args.append('MATRIX_VERBOSE=1')
608
609         if config.debug:
610                 print 'Executing:', ' '.join(args)
611
612         return os.spawnvp(os.P_NOWAIT, args[0], args)
613
614 def clean_components(targets):
615         if not targets:
616                 targets = config.components.keys()
617
618         for name in targets:
619                 c = config.components[name]
620                 print 'Cleaning', c.repo.path
621
622                 cache.remove(c)
623
624                 files = git.ls_files(c.repo.path, ['-o'])
625                 paths = [os.path.join(c.repo.path, i) for i in files]
626                 paths.sort()
627                 paths.reverse()
628
629                 for path in paths:
630                         if git.contains_database(path):
631                                 continue
632
633                         if config.debug:
634                                 print 'Removing', path
635
636                         if os.path.islink(path) or not os.path.isdir(path):
637                                 os.remove(path)
638                         else:
639                                 remove_tree(path)
640
641                 for repo in (c.repo, c.meta):
642                         files = git.ls_files(repo.path, ['-m', '-d'])
643                         if files:
644                                 log_error('Dirty files left in %s' % repo.path)
645
646 def for_each_repository(func, targets=None):
647         if not targets:
648                 targets = config.components.keys()
649
650         for name in targets:
651                 c = config.components[name]
652                 if git.contains_database(c.repo.path):
653                         func(c.repo)
654                 func(c.meta)
655
656 def rebase_components(targets):
657         for_each_repository(rebase_repository, targets)
658
659 def pull_components(targets):
660         for_each_repository(pull_repository, targets)
661
662 def rebase_repository(repo):
663         print 'Rebasing', repo.path
664         git.remote_update(repo.path)
665         git.rebase(repo.path)
666
667 def pull_repository(repo):
668         print 'Pulling', repo.path
669         git.pull(repo.path)
670
671 def source_dist_components(targets):
672         if not targets:
673                 targets = config.components.keys()
674
675         for name in targets:
676                 c = config.components[name]
677                 generate_component_changes(c, 'dist')
678                 package_component_sources(c, 'dist')
679
680 def generate_component_changes(c, location):
681         print 'Generating change log for', c.repo.path
682
683         path = os.path.join(location, c.name) + '.changes'
684
685         pathdir = os.path.dirname(path)
686         if not os.path.exists(pathdir):
687                 os.makedirs(pathdir)
688
689         fd = os.open(path, os.O_WRONLY | os.O_CREAT, 0644)
690         git.log(c.repo.path, [c.get_active_tag()], fd=fd)
691         os.close(fd)
692
693 def package_component_sources(c, location):
694         print 'Archiving', c.repo.path
695
696         rev = git.describe(c.repo.path)
697         if rev:
698                 rev = '_' + rev
699         else:
700                 rev = ''
701
702         path = os.path.join(location, c.name) + rev + '.tar.bz2'
703         if os.path.exists(path):
704                 os.remove(path)
705
706         pathdir = os.path.dirname(path)
707         if not os.path.exists(pathdir):
708                 os.makedirs(pathdir)
709
710         git.archive(c.repo.path, path,
711                     prefix=os.path.basename(c.name) + '/',
712                     branch=c.get_active_tag())
713
714 def build_rootfs(clean=True, rootfs_only=False, jffs2_only=False, devrootfs_only=False):
715         parse_config('rootfs')
716
717         f = os.popen('sb-conf cu')
718         target = f.read()
719         f.close()
720
721         rootfs = RootFS(target.strip())
722         rootfs.include_paths(config.include_paths)
723         rootfs.include_files(config.include_files)
724         rootfs.filter_paths(config.exclude_paths)
725         rootfs.filter_files(config.exclude_files)
726         rootfs.filter_expressions(config.exclude_expressions)
727         rootfs.add_paths(config.created_paths)
728         rootfs.set_devices(config.devices)
729         rootfs.set_change_owner(config.change_owner)
730         rootfs.set_erase_size(config.boards[config.board].flash_erase_size)
731         rootfs.set_pad_size(config.boards[config.board].flash_pad_size)
732
733         if rootfs_only:
734                 rootfs.generate(clean, build_target="rootfs")
735         elif jffs2_only:
736                 rootfs.generate(clean, build_target="jffs2")
737         elif devrootfs_only:
738                 rootfs.generate(clean, build_target="devrootfs")
739         else:
740                 rootfs.generate(clean, build_target="all")