133ae36982100398bd477723e6b3db0694f856cf
[matrix.git] / matrix / matrix.py
1 # Copyright (C) 2006-2008 Movial Oy
2 # Authors: Timo Savola <tsavola@movial.fi>
3 #          Toni Timonen
4 #          Kalle Vahlman <kalle.vahlman@movial.fi>
5 #          Tuomas Kulve <tuomas.kulve@movial.fi>
6
7 import glob
8 import os
9 import re
10 import signal
11 import sys
12 import tarfile
13 from sets import Set as set
14
15 import config
16 import git
17 from rootfs import RootFS
18
19 Error = RuntimeError
20
21 error_log = []
22
23 def log_error(msg):
24         error_log.append(msg)
25
26 def main():
27         try:
28                 try:
29                         logged_main()
30                 except:
31                         if not error_log:
32                                 print >>sys.stderr
33                         raise
34         finally:
35                 if error_log:
36                         print >>sys.stderr
37                         for msg in error_log:
38                                 print >>sys.stderr, msg
39
40 def logged_main():
41         parse_config('config.local')
42
43         command, params, targets = parse_args(sys.argv)
44
45         parse_config('config')
46         parse_config('boards')
47         parse_config('components')
48
49         if 'global-cache' in config.flags:
50                 config.cache_dir = config.global_cache_dir
51
52         if not os.path.exists(config.cache_dir):
53                 os.makedirs(config.cache_dir)
54
55         all_targets = update_components()
56
57         for name in targets:
58                 if name not in config.components:
59                         raise Error('Component "%s" does not exist' % name)
60
61         if not targets:
62                 targets = all_targets
63
64         if command == 'install':
65                 build_components(targets, **params)
66         elif command == 'clone':
67                 clone_components(targets)
68         elif command == 'clean':
69                 clean_components(targets)
70         elif command == 'rebase':
71                 rebase_components(targets)
72         elif command == 'pull':
73                 pull_components(targets)
74         elif command == 'source-dist':
75                 source_dist_components(targets)
76         elif command == 'rootfs':
77                 build_rootfs(**params)
78         else:
79                 raise Error('Invalid command: ' + command)
80
81 def help(file, args):
82         print >>file, '''
83 Copyright (C) 2006-2008 Movial Oy
84
85 Usage: %(progname)s [<options>] <command> [<params>] [<components>]
86
87 If no components are specified, all of them will be targeted.  All component
88 metadata will be downloaded regardless of the specified command and components.
89
90 Options:
91         -v              Verbose output.
92         -d              Debug output.
93         -r URL          Specify the root for component git repos to clone from.
94                         If this option is not given, the roots specified in the
95                         components file will be used.  This option may be
96                         specified multiple times.
97         -f              Build components with dirty files.
98         -h, --help      Print this help text.
99
100 Commands and parameters:
101         clone           Download the components' git repositories.
102         install         Download, build and install the components.
103                 -j N            Execute N build jobs in parallel.
104                 -mj N           Run component builds with make -j N.
105         clean           Remove all non-tracked files from the component git
106                         repository directories.
107         rebase          Update repositories from server by rebasing.
108         pull            Update repositories from server by merging.
109         source-dist     Download and package the component sources.
110         rootfs          Build a rootfs from the current target installation.
111                 -n              Do not clean env.faked and stripped directory.
112                 -r              Only generate a stripped rootfs.
113                 -j              Only generate a jffs2 image
114                 -d              Only generate a rootstrap (development rootfs).
115 ''' % {'progname': args[0]}
116
117 def parse_args(args):
118         def parse_jobs(arg):
119                 jobs = int(arg)
120                 if jobs <= 0:
121                         raise Error('Please specify a valid number of jobs')
122                 return jobs
123
124         command = None
125         params = {}
126         targets = []
127
128         i = 1
129         while i < len(args):
130                 if args[i].startswith('-'):
131                         if not command:
132                                 if args[i] == '-v':
133                                         config.verbose = True
134                                 elif args[i] == '-d':
135                                         config.debug = True
136                                 elif args[i] == '-r':
137                                         i += 1
138                                         config.roots.append(args[i])
139                                 elif args[i] == '-f':
140                                         config.force = True
141                                 elif args[i] in ('-h', '--help'):
142                                         help(sys.stdout, args)
143                                         sys.exit(0)
144                                 else:
145                                         raise Error('Bad option: ' + args[i])
146                         elif command == 'install':
147                                 if args[i] == '-j':
148                                         i += 1
149                                         params['build_jobs'] = parse_jobs(args[i])
150                                 elif args[i] == '-mj':
151                                         i += 1
152                                         params['make_jobs'] = parse_jobs(args[i])
153                                 else:
154                                         raise Error('Bad parameter: ' + args[i])
155                         elif command == 'rootfs':
156                                 if args[i] == '-n':
157                                         i += 1
158                                         params['clean'] = False
159                                 elif args[i] == '-r':
160                                         i += 1
161                                         params['rootfs_only'] = True
162                                 elif args[i] == '-j':
163                                         i += 1
164                                         params['jffs2_only'] = True
165                                 elif args[i] == '-d':
166                                         i += 1
167                                         params['devrootfs_only'] = True
168                         else:
169                                 raise Error('Command takes no parameters')
170                 elif not command:
171                         command = args[i]
172                 else:
173                         targets.append(args[i])
174                 i += 1
175
176         if not command:
177                 help(sys.stderr, args)
178                 sys.exit(1)
179
180         return command, params, targets
181
182 def parse_config(path):
183         if os.path.isabs(path):
184                 path = os.path.join(config.top_dir, path)
185
186         if os.path.exists(path):
187                 print 'Reading', path
188                 execfile(path, config.__dict__, config.__dict__)
189
190 def update_components():
191         targets = []
192         packages = {}
193
194         for c in config.components.itervalues():
195                 update_component_url(c)
196                 clone_metadata(c)
197                 update_component_packages(c, targets, packages)
198                 c.active_depends = []
199
200         update_component_depends(packages)
201
202         return targets
203
204 def update_component_url(c):
205         update_repository_url(c.repo)
206         update_repository_url(c.meta)
207
208 def update_repository_url(repo):
209         if git.contains_database(repo.path):
210                 url = git.getvar(repo.path, 'remote.origin.url')
211                 if url:
212                         repo.active_url = url
213                         if config.debug:
214                                 print 'Using', repo.active_url
215                         return
216
217         for root in config.roots:
218                 for suffix in ('.git', ''):
219                         url = '%s/%s%s' % (root, repo.name, suffix)
220
221                         if config.debug:
222                                 print 'Trying', url
223
224                         if not git.peek_remote(url, quiet=True):
225                                 continue
226
227                         repo.active_url = url
228                         if config.debug:
229                                 print 'Found', repo.active_url
230                         return
231
232         raise Error('Failed to locate repository: ' + repo.name)
233
234 def update_component_depends(packages):
235         for pkg in packages.itervalues():
236                 for spec in pkg.depends.split():
237                         depname = Depend(spec).name
238                         deppkg = packages.get(depname)
239
240                         if not deppkg:
241                                 log_error('Package %s depends on ' \
242                                           'non-existent package %s', \
243                                           pkg.name, depname)
244                                 continue
245
246                         if deppkg.component == pkg.component:
247                                 continue
248
249                         pkg.component.active_depends.append(deppkg.component)
250
251         fail = False
252         for pkg in packages.itervalues():
253                 for spec in pkg.depends.split():
254                         if not Depend(spec).check(packages):
255                                 fail = True
256                                 log_error('Dependency %s failed for %s' % \
257                                           (spec, pkg.name))
258
259                 for spec in pkg.conflicts.split():
260                         if Depend(spec).check(packages):
261                                 fail = True
262                                 log_error('Package %s conflicts with %s' % \
263                                           (pkg.name, spec))
264
265         if fail:
266                 raise Error('Invalid component tree')
267
268 class Depend(object):
269         regex = re.compile(r'([@]?)([^\s:]+)[:]?([<>=]*)([^\s:]*)[:]?(.*)')
270
271         def __init__(self, spec):
272                 match = self.regex.match(spec)
273                 if not match:
274                         raise Error('Bad dependency specification: ' + spec)
275
276                 self.build, self.name, self.tag_op, self.tag, flags \
277                         = match.groups()
278                 self.flags = flags.split()
279
280         def check(self, packages):
281                 # TODO: check version and flags
282                 return self.name in packages
283
284 def execute(args):
285         if config.debug:
286                 print 'Executing:', ' '.join(args)
287
288         if os.spawnvp(os.P_WAIT, args[0], args) != 0:
289                 raise Error('Failed: ' + ' '.join(args))
290
291 def remove_tree(path):
292         execute(['rm', '-rf', path])
293
294 def clone_components(targets):
295         if not targets:
296                 targets = config.components.keys()
297
298         for name in targets:
299                 c = config.components[name]
300                 clone_component(c)
301
302 def clone_component(c, overwrite=False):
303         have_repo = git.contains_database(c.repo.path)
304         have_meta = git.contains_database(c.meta.path)
305
306         if not overwrite and have_repo and have_meta:
307                 return
308
309         if overwrite and os.path.exists(c.repo.path):
310                 print 'Removing', c.repo.path
311
312                 remove_tree(c.repo.path)
313                 have_repo = False
314                 have_meta = False
315
316         if not have_repo:
317                 clone_repository(c.repo)
318                 git.exclude(c.repo.path, 'meta')
319
320         if not have_meta:
321                 clone_repository(c.meta)
322
323 def clone_metadata(c):
324         if not git.contains_database(c.meta.path):
325                 clone_repository(c.meta)
326
327 def clone_repository(repo):
328         print 'Cloning', repo.path
329
330         if os.path.exists(repo.path):
331                 tmp = os.path.join(repo.path, 'tmp')
332                 git.clone(tmp, repo.active_url, checkout=False)
333                 try:
334                         tmpdb = os.path.join(tmp, '.git')
335                         repodb = os.path.join(repo.path, '.git')
336                         if config.debug:
337                                 print 'Renaming "%s" as "%s"' % (tmpdb, repodb)
338                         os.rename(tmpdb, repodb)
339                 finally:
340                         os.rmdir(tmp)
341                 git.checkout(repo.path)
342         else:
343                 git.clone(repo.path, repo.active_url, checkout=True)
344
345 def update_component_packages(c, targets, packages):
346         c.active_packages = {}
347
348         for path in glob.glob(os.path.join(c.meta.path, '*.package')):
349                 name = os.path.basename(path)[:-8]
350
351                 pkg = parse_package(name, c, path)
352                 if not pkg:
353                         continue
354
355                 c.active_packages[name] = pkg
356                 packages[name] = pkg
357
358                 if config.debug:
359                         print 'Component', c.name, 'provides', name
360
361         if c.active_packages:
362                 targets.append(c.name)
363         elif config.debug:
364                 print 'Component', c.name, 'does not provide any packages'
365
366 class Package(object):
367         def __init__(self, name, component):
368                 self.name = name
369                 self.component = component
370
371                 self.depends = []
372                 self.conflicts = []
373                 self.architectures = None
374
375 def parse_package(name, component, path):
376         pkg = Package(name, component)
377         execfile(path, pkg.__dict__, pkg.__dict__)
378
379         if pkg.architectures:
380                 arch = config.boards[config.board].arch
381                 if arch not in pkg.architectures:
382                         return None
383
384         return pkg
385
386 def build_components(targets, build_jobs=1, make_jobs=1):
387         selected = set([config.components[i] for i in targets])
388
389         depends = None
390         while depends is None or depends:
391                 depends = set()
392
393                 for c in selected:
394                         for dep in c.active_depends:
395                                 if dep not in selected:
396                                         depends.add(dep)
397
398                 selected |= depends
399
400         components = list(selected)
401         components.sort()
402
403         if config.debug:
404                 print 'Building components:'
405                 for c in components:
406                         print '\t' + c.name
407
408         builder = Builder(components, build_jobs, make_jobs)
409         builder.run()
410
411 class Builder(object):
412         def __init__(self, components, max_jobs, max_make_jobs):
413                 self.max_jobs = max_jobs
414                 self.max_make_jobs = max_make_jobs
415
416                 self.jobs = {}
417
418                 self.wait_build = set(components)
419                 self.wait_install = []
420                 self.in_install = None
421
422                 self.progress_now = 0
423                 self.progress_total = len(components) * 2
424
425                 self.error = False
426
427         def run(self):
428                 try:
429                         while self.run_iteration():
430                                 pass
431                 except:
432                         for pid in self.jobs:
433                                 try:
434                                         os.kill(pid, signal.SIGTERM)
435                                 except OSError, e:
436                                         log_error('kill: %s' % e)
437
438                                 c = self.jobs[pid]
439                                 c.active_state = 'killed'
440
441                         while self.jobs:
442                                 self.wait()
443
444                         raise
445
446         def run_iteration(self):
447                 if self.can_start_install():
448                         c = self.wait_install.pop(0)
449                         self.start_install(c)
450                         self.in_install = c
451
452                 while self.can_start_build():
453                         found = False
454
455                         for c in self.wait_build:
456                                 if self.is_buildable(c):
457                                         found = True
458
459                                         if not self.try_cache(c):
460                                                 self.start_build(c)
461
462                                         self.wait_build.remove(c)
463                                         break
464
465                         if not found:
466                                 break
467
468                 if self.jobs:
469                         self.wait()
470                         return True
471                 else:
472                         if self.wait_build and not self.error:
473                                 raise Error('Circular dependencies?')
474                         return False
475
476         def progress(self, increment=1):
477                 self.progress_now += increment
478
479         def message(self, arg1, arg2=None):
480                 if arg2 is None:
481                         message = arg1
482                 else:
483                         message = '%-10s %s' % (arg1, arg2)
484
485                 progress = '%3d%%' % \
486                            (self.progress_now *100 / self.progress_total)
487
488                 print progress, message
489
490         def is_buildable(self, c):
491                 if c.active_state:
492                         if config.debug:
493                                 self.message('Component %s is in state: %s' % \
494                                         (c.name, c.active_state))
495                         return False
496
497                 for dep in c.active_depends:
498                         if config.components[dep.name].active_state != 'done':
499                                 if config.debug:
500                                         self.message('Component %s depends ' \
501                                                 'on uninstalled %s' % \
502                                                 (c.name, dep.name))
503                                 return False
504
505                 return True
506
507         def try_cache(self, c):
508                 if not component_cached(c):
509                         return False
510
511                 c.active_state = 'done'
512                 self.progress(2)
513
514                 if config.debug:
515                         self.message('Cached', c.repo.path)
516
517                 return True
518
519         def can_start_build(self):
520                 return self.can_start_job()
521
522         def can_start_install(self):
523                 return not self.in_install and self.wait_install and \
524                        self.can_start_job()
525
526         def can_start_job(self):
527                 return not self.error and len(self.jobs) < self.max_jobs
528
529         def start_build(self, c):
530                 self.message('Building', c.repo.path)
531                 self.start_job(c, 'build_matrix_component', 'in-build')
532
533         def start_install(self, c):
534                 self.message('Installing', c.repo.path)
535                 self.start_job(c, 'install_matrix_component', 'in-install')
536
537         def start_job(self, c, make_target, state):
538                 pid = spawn_make(c, self.max_make_jobs, make_target)
539                 self.jobs[pid] = c
540                 c.active_state = state
541
542         def wait(self):
543                 pid, status = os.wait()
544
545                 if status:
546                         self.error = True
547
548                 c = self.jobs[pid]
549                 del self.jobs[pid]
550
551                 if c.active_state == 'in-build':
552                         if status:
553                                 log_error('Failed to build %s' % c.name)
554                                 return
555
556                         self.progress()
557                         if config.debug:
558                                 self.message('Built', c.repo.path)
559
560                         c.active_state = 'wait-install'
561                         self.wait_install.append(c)
562
563                 elif c.active_state == 'in-install':
564                         assert c == self.in_install
565
566                         if status:
567                                 log_error('Failed to install %s' % c.name)
568                                 return
569
570                         self.progress()
571                         self.message('Finished', c.repo.path)
572
573                         c.active_state = 'done'
574                         self.in_install = None
575                         update_cache(c)
576
577                 elif c.active_state != 'killed':
578                         raise Error('Unexpected state: %s' % c.active_state)
579
580 def component_cached(c):
581         if c.from_platform:
582                 return True
583
584         for repo in (c.repo, c.meta):
585                 if git.ls_files(repo.path, ['-m', '-d']):
586                         if config.force:
587                                 return False
588                         else:
589                                 raise Error('Dirty files in ' + repo.path)
590
591         path = os.path.join(config.cache_dir, c.name)
592         if not os.path.exists(path):
593                 return False
594
595         file = open(path, 'r')
596         line = file.readline()
597         file.close()
598
599         match = re.match(r'([^\s]+)[\s]?([^\s]*)', line)
600         if not match:
601                 return False
602
603         hash, flagstring = match.groups()
604
605         if hash != get_component_hash(c):
606                 if config.debug:
607                         print 'Component has been changed:', c.name
608                 return False
609
610         if flagstring:
611                 flags = flagstring.split(',')
612         else:
613                 flags = []
614
615         if set(flags) != set(c.flags):
616                 if config.debug:
617                         print 'Component flags have been changed:', c.name
618                 return False
619
620         return True
621
622 def update_cache(c):
623         path = os.path.join(config.cache_dir, c.name)
624
625         dir = os.path.dirname(path)
626         if not os.path.exists(dir):
627                 os.makedirs(dir)
628
629         file = open(path, 'w')
630         done = False
631         try:
632                 print >>file, get_component_hash(c), ','.join(c.flags)
633                 done = True
634         finally:
635                 file.close()
636                 if not done:
637                         os.remove(path)
638
639 def get_component_hash(c):
640         return '%s+%s' % (get_repository_hash(c.repo), get_repository_hash(c.meta))
641
642 def get_repository_hash(repo):
643         if repo.active_hash is None:
644                 repo.active_hash = git.rev_parse(repo.path, 'HEAD')
645         return repo.active_hash
646
647 def spawn_make(c, jobs, target):
648         board = config.boards[config.board]
649
650         make = os.getenv('MAKE', 'make')
651         makefile = os.path.join(config.script_dir, 'matrix.mak')
652         workdir = os.path.join(config.top_dir, 'src', c.name)
653         args = [make, '--no-print-directory', '-f', makefile, '-C', workdir,
654                 '-j', str(jobs), target,
655                 'MATRIX_TOPDIR='    + os.path.abspath(config.top_dir),
656                 'MATRIX_SCRIPTDIR=' + os.path.abspath(config.script_dir),
657                 'MATRIX_COMPONENT=' + c.name,
658                 'MATRIX_ARCH='      + board.arch,
659                 'MATRIX_GCC_MARCH=' + board.gcc_march,
660                 'MATRIX_GCC_MCPU='  + board.gcc_mcpu,
661                 'MATRIX_GCC_MFPU='  + board.gcc_mfpu,
662                 'MATRIX_GCC_OPTIONS=' + board.gcc_options,
663                 'MATRIX_GNU_HOST='  + board.gnu_host,
664                 'MATRIX_LIBC='      + config.libc]
665
666         for flag in config.flags:
667                 args.append(flag + '=1')
668
669         if config.verbose:
670                 args.append('MATRIX_VERBOSE=1')
671
672         if config.debug:
673                 print 'Executing:', ' '.join(args)
674
675         return os.spawnvp(os.P_NOWAIT, args[0], args)
676
677 def clean_components(targets):
678         if not targets:
679                 targets = config.components.keys()
680
681         for name in targets:
682                 c = config.components[name]
683                 print 'Cleaning', c.repo.path
684
685                 files = git.ls_files(c.repo.path, ['-o'])
686                 paths = [os.path.join(c.repo.path, i) for i in files]
687                 paths.sort()
688                 paths.reverse()
689
690                 cache = os.path.join(config.cache_dir, name)
691                 if os.path.exists(cache):
692                         paths.append(cache)
693
694                 for path in paths:
695                         if git.contains_database(path):
696                                 continue
697
698                         if config.debug:
699                                 print 'Removing', path
700
701                         if os.path.islink(path) or not os.path.isdir(path):
702                                 os.remove(path)
703                         else:
704                                 remove_tree(path)
705
706                 for repo in (c.repo, c.meta):
707                         files = git.ls_files(repo.path, ['-m', '-d'])
708                         if files:
709                                 log_error('Dirty files left in %s' % repo.path)
710
711 def for_each_repository(func, targets=None):
712         if not targets:
713                 targets = config.components.keys()
714
715         for name in targets:
716                 c = config.components[name]
717                 if git.contains_database(c.repo.path):
718                         func(c.repo)
719                 func(c.meta)
720
721 def rebase_components(targets):
722         for_each_repository(rebase_repository, targets)
723
724 def pull_components(targets):
725         for_each_repository(pull_repository, targets)
726
727 def rebase_repository(repo):
728         print 'Rebasing', repo.path
729         git.remote_update(repo.path)
730         git.rebase(repo.path)
731
732 def pull_repository(repo):
733         print 'Pulling', repo.path
734         git.pull(repo.path)
735
736 def source_dist_components(targets):
737         if not targets:
738                 targets = config.components.keys()
739
740         for name in targets:
741                 c = config.components[name]
742                 generate_component_changes(c, 'dist')
743                 package_component_sources(c, 'dist')
744
745 def generate_component_changes(c, location):
746         print 'Generating change log for', c.repo.path
747
748         path = os.path.join(location, c.name) + '.changes'
749
750         pathdir = os.path.dirname(path)
751         if not os.path.exists(pathdir):
752                 os.makedirs(pathdir)
753
754         fd = os.open(path, os.O_WRONLY | os.O_CREAT, 0644)
755         git.log(c.repo.path, [c.get_active_tag()], fd=fd)
756         os.close(fd)
757
758 def package_component_sources(c, location):
759         print 'Archiving', c.repo.path
760
761         rev = git.describe(c.repo.path)
762         if rev:
763                 rev = '_' + rev
764         else:
765                 rev = ''
766
767         path = os.path.join(location, c.name) + rev + '.tar.bz2'
768         if os.path.exists(path):
769                 os.remove(path)
770
771         pathdir = os.path.dirname(path)
772         if not os.path.exists(pathdir):
773                 os.makedirs(pathdir)
774
775         git.archive(c.repo.path, path,
776                     prefix=os.path.basename(c.name) + '/',
777                     branch=c.get_active_tag())
778
779 def build_rootfs(clean=True, rootfs_only=False, jffs2_only=False, devrootfs_only=False):
780         parse_config('rootfs')
781
782         f = os.popen('sb-conf cu')
783         target = f.read()
784         f.close()
785
786         rootfs = RootFS(target.strip())
787         rootfs.include_paths(config.include_paths)
788         rootfs.include_files(config.include_files)
789         rootfs.filter_paths(config.exclude_paths)
790         rootfs.filter_files(config.exclude_files)
791         rootfs.filter_expressions(config.exclude_expressions)
792         rootfs.add_paths(config.created_paths)
793         rootfs.set_devices(config.devices)
794         rootfs.set_change_owner(config.change_owner)
795         rootfs.set_erase_size(config.boards[config.board].flash_erase_size)
796         rootfs.set_pad_size(config.boards[config.board].flash_pad_size)
797
798         if rootfs_only:
799                 rootfs.generate(clean, build_target="rootfs")
800         elif jffs2_only:
801                 rootfs.generate(clean, build_target="jffs2")
802         elif devrootfs_only:
803                 rootfs.generate(clean, build_target="devrootfs")
804         else:
805                 rootfs.generate(clean, build_target="all")