log module
[matrix.git] / matrix / matrix.py
1 # Copyright (C) 2006-2008 Movial Oy
2 # Authors: Timo Savola <tsavola@movial.fi>
3 #          Toni Timonen
4 #          Kalle Vahlman <kalle.vahlman@movial.fi>
5 #          Tuomas Kulve <tuomas.kulve@movial.fi>
6
7 import glob
8 import os
9 import re
10 import signal
11 import sys
12 import tarfile
13 from sets import Set as set
14
15 import cache
16 import config
17 import git
18 import log
19 from rootfs import RootFS
20
21 Error = RuntimeError
22
23 def main():
24         parse_config('config.local')
25
26         command, params, targets = parse_args(sys.argv)
27
28         parse_config('config')
29         parse_config('boards')
30         parse_config('components')
31
32         if 'global-cache' in config.flags:
33                 config.cache_dir = config.global_cache_dir
34
35         if not os.path.exists(config.cache_dir):
36                 os.makedirs(config.cache_dir)
37
38         all_targets = update_components()
39
40         for name in targets:
41                 if name not in config.components:
42                         raise Error('Component "%s" does not exist' % name)
43
44         if not targets:
45                 targets = all_targets
46
47         if command == 'install':
48                 build_components(targets, **params)
49         elif command == 'clone':
50                 clone_components(targets)
51         elif command == 'clean':
52                 clean_components(targets)
53         elif command == 'rebase':
54                 rebase_components(targets)
55         elif command == 'pull':
56                 pull_components(targets)
57         elif command == 'source-dist':
58                 source_dist_components(targets)
59         elif command == 'rootfs':
60                 build_rootfs(**params)
61         else:
62                 raise Error('Invalid command: ' + command)
63
64 def help(file, args):
65         print >>file, '''
66 Copyright (C) 2006-2008 Movial Oy
67
68 Usage: %(progname)s [<options>] <command> [<params>] [<components>]
69
70 If no components are specified, all of them will be targeted.  All component
71 metadata will be downloaded regardless of the specified command and components.
72
73 Options:
74         -v              Verbose output.
75         -d              Debug output.
76         -r URL          Specify the root for component git repos to clone from.
77                         If this option is not given, the roots specified in the
78                         components file will be used.  This option may be
79                         specified multiple times.
80         -f              Build components with dirty files.
81         -h, --help      Print this help text.
82
83 Commands and parameters:
84         clone           Download the components' git repositories.
85         install         Download, build and install the components.
86                 -j N            Execute N build jobs in parallel.
87                 -mj N           Run component builds with make -j N.
88         clean           Remove all non-tracked files from the component git
89                         repository directories.
90         rebase          Update repositories from server by rebasing.
91         pull            Update repositories from server by merging.
92         source-dist     Download and package the component sources.
93         rootfs          Build a rootfs from the current target installation.
94                 -n              Do not clean env.faked and stripped directory.
95                 -r              Only generate a stripped rootfs.
96                 -j              Only generate a jffs2 image
97                 -d              Only generate a rootstrap (development rootfs).
98 ''' % {'progname': args[0]}
99
100 def parse_args(args):
101         def parse_jobs(arg):
102                 jobs = int(arg)
103                 if jobs <= 0:
104                         raise Error('Please specify a valid number of jobs')
105                 return jobs
106
107         command = None
108         params = {}
109         targets = []
110
111         i = 1
112         while i < len(args):
113                 if args[i].startswith('-'):
114                         if not command:
115                                 if args[i] == '-v':
116                                         config.verbose = True
117                                 elif args[i] == '-d':
118                                         config.debug = True
119                                 elif args[i] == '-r':
120                                         i += 1
121                                         config.roots.append(args[i])
122                                 elif args[i] == '-f':
123                                         config.force = True
124                                 elif args[i] in ('-h', '--help'):
125                                         help(sys.stdout, args)
126                                         sys.exit(0)
127                                 else:
128                                         raise Error('Bad option: ' + args[i])
129                         elif command == 'install':
130                                 if args[i] == '-j':
131                                         i += 1
132                                         params['build_jobs'] = parse_jobs(args[i])
133                                 elif args[i] == '-mj':
134                                         i += 1
135                                         params['make_jobs'] = parse_jobs(args[i])
136                                 else:
137                                         raise Error('Bad parameter: ' + args[i])
138                         elif command == 'rootfs':
139                                 if args[i] == '-n':
140                                         i += 1
141                                         params['clean'] = False
142                                 elif args[i] == '-r':
143                                         i += 1
144                                         params['rootfs_only'] = True
145                                 elif args[i] == '-j':
146                                         i += 1
147                                         params['jffs2_only'] = True
148                                 elif args[i] == '-d':
149                                         i += 1
150                                         params['devrootfs_only'] = True
151                         else:
152                                 raise Error('Command takes no parameters')
153                 elif not command:
154                         command = args[i]
155                 else:
156                         targets.append(args[i])
157                 i += 1
158
159         if not command:
160                 help(sys.stderr, args)
161                 sys.exit(1)
162
163         return command, params, targets
164
165 def parse_config(path):
166         if os.path.isabs(path):
167                 path = os.path.join(config.top_dir, path)
168
169         if os.path.exists(path):
170                 print 'Reading', path
171                 execfile(path, config.__dict__, config.__dict__)
172
173 def update_components():
174         targets = []
175         packages = {}
176
177         for c in config.components.itervalues():
178                 update_component_url(c)
179                 clone_metadata(c)
180                 update_component_packages(c, targets, packages)
181                 c.active_depends = []
182
183         update_component_depends(packages)
184
185         return targets
186
187 def update_component_url(c):
188         update_repository_url(c.repo)
189         update_repository_url(c.meta)
190
191 def update_repository_url(repo):
192         if git.contains_database(repo.path):
193                 url = git.getvar(repo.path, 'remote.origin.url')
194                 if url:
195                         repo.active_url = url
196                         if config.debug:
197                                 print 'Using', repo.active_url
198                         return
199
200         for root in config.roots:
201                 for suffix in ('.git', ''):
202                         url = '%s/%s%s' % (root, repo.name, suffix)
203
204                         if config.debug:
205                                 print 'Trying', url
206
207                         if not git.peek_remote(url, quiet=True):
208                                 continue
209
210                         repo.active_url = url
211                         if config.debug:
212                                 print 'Found', repo.active_url
213                         return
214
215         raise Error('Failed to locate repository: ' + repo.name)
216
217 def update_component_depends(packages):
218         for pkg in packages.itervalues():
219                 for spec in pkg.depends.split():
220                         depname = Depend(spec).name
221                         deppkg = packages.get(depname)
222
223                         if not deppkg:
224                                 log.error('Package %s depends on ' \
225                                           'non-existent package %s', \
226                                           pkg.name, depname)
227                                 continue
228
229                         if deppkg.component == pkg.component:
230                                 continue
231
232                         pkg.component.active_depends.append(deppkg.component)
233
234         fail = False
235         for pkg in packages.itervalues():
236                 for spec in pkg.depends.split():
237                         if not Depend(spec).check(packages):
238                                 fail = True
239                                 log.error('Dependency %s failed for %s' % \
240                                           (spec, pkg.name))
241
242                 for spec in pkg.conflicts.split():
243                         if Depend(spec).check(packages):
244                                 fail = True
245                                 log.error('Package %s conflicts with %s' % \
246                                           (pkg.name, spec))
247
248         if fail:
249                 raise Error('Invalid component tree')
250
251 class Depend(object):
252         regex = re.compile(r'([@]?)([^\s:]+)[:]?([<>=]*)([^\s:]*)[:]?(.*)')
253
254         def __init__(self, spec):
255                 match = self.regex.match(spec)
256                 if not match:
257                         raise Error('Bad dependency specification: ' + spec)
258
259                 self.build, self.name, self.tag_op, self.tag, flags \
260                         = match.groups()
261                 self.flags = flags.split()
262
263         def check(self, packages):
264                 # TODO: check version and flags
265                 return self.name in packages
266
267 def execute(args):
268         if config.debug:
269                 print 'Executing:', ' '.join(args)
270
271         if os.spawnvp(os.P_WAIT, args[0], args) != 0:
272                 raise Error('Failed: ' + ' '.join(args))
273
274 def remove_tree(path):
275         execute(['rm', '-rf', path])
276
277 def clone_components(targets):
278         if not targets:
279                 targets = config.components.keys()
280
281         for name in targets:
282                 c = config.components[name]
283                 clone_component(c)
284
285 def clone_component(c, overwrite=False):
286         have_repo = git.contains_database(c.repo.path)
287         have_meta = git.contains_database(c.meta.path)
288
289         if not overwrite and have_repo and have_meta:
290                 return
291
292         if overwrite and os.path.exists(c.repo.path):
293                 print 'Removing', c.repo.path
294
295                 remove_tree(c.repo.path)
296                 have_repo = False
297                 have_meta = False
298
299         if not have_repo:
300                 clone_repository(c.repo)
301                 git.exclude(c.repo.path, 'meta')
302
303         if not have_meta:
304                 clone_repository(c.meta)
305
306 def clone_metadata(c):
307         if not git.contains_database(c.meta.path):
308                 clone_repository(c.meta)
309
310 def clone_repository(repo):
311         print 'Cloning', repo.path
312
313         if os.path.exists(repo.path):
314                 tmp = os.path.join(repo.path, 'tmp')
315                 git.clone(tmp, repo.active_url, checkout=False)
316                 try:
317                         tmpdb = os.path.join(tmp, '.git')
318                         repodb = os.path.join(repo.path, '.git')
319                         if config.debug:
320                                 print 'Renaming "%s" as "%s"' % (tmpdb, repodb)
321                         os.rename(tmpdb, repodb)
322                 finally:
323                         os.rmdir(tmp)
324                 git.checkout(repo.path)
325         else:
326                 git.clone(repo.path, repo.active_url, checkout=True)
327
328 def update_component_packages(c, targets, packages):
329         c.active_packages = {}
330
331         for path in glob.glob(os.path.join(c.meta.path, '*.package')):
332                 name = os.path.basename(path)[:-8]
333
334                 pkg = parse_package(name, c, path)
335                 if not pkg:
336                         continue
337
338                 c.active_packages[name] = pkg
339                 packages[name] = pkg
340
341                 if config.debug:
342                         print 'Component', c.name, 'provides', name
343
344         if c.active_packages:
345                 targets.append(c.name)
346         elif config.debug:
347                 print 'Component', c.name, 'does not provide any packages'
348
349 class Package(object):
350         def __init__(self, name, component):
351                 self.name = name
352                 self.component = component
353
354                 self.depends = []
355                 self.conflicts = []
356                 self.architectures = None
357
358 def parse_package(name, component, path):
359         pkg = Package(name, component)
360         execfile(path, pkg.__dict__, pkg.__dict__)
361
362         if pkg.architectures:
363                 arch = config.boards[config.board].arch
364                 if arch not in pkg.architectures:
365                         return None
366
367         return pkg
368
369 def build_components(targets, build_jobs=1, make_jobs=1):
370         selected = set([config.components[i] for i in targets])
371
372         depends = None
373         while depends is None or depends:
374                 depends = set()
375
376                 for c in selected:
377                         for dep in c.active_depends:
378                                 if dep not in selected:
379                                         depends.add(dep)
380
381                 selected |= depends
382
383         components = list(selected)
384         components.sort()
385
386         if config.debug:
387                 print 'Building components:'
388                 for c in components:
389                         print '\t' + c.name
390
391         builder = Builder(components, build_jobs, make_jobs)
392         builder.run()
393
394 class Builder(object):
395         def __init__(self, components, max_jobs, max_make_jobs):
396                 self.max_jobs = max_jobs
397                 self.max_make_jobs = max_make_jobs
398
399                 self.jobs = {}
400
401                 self.wait_build = set(components)
402                 self.wait_install = []
403                 self.in_install = None
404
405                 self.progress_now = 0
406                 self.progress_total = len(components) * 2
407
408                 self.error = False
409
410         def run(self):
411                 try:
412                         while self.run_iteration():
413                                 pass
414                 except:
415                         for pid in self.jobs:
416                                 try:
417                                         os.kill(pid, signal.SIGTERM)
418                                 except OSError, e:
419                                         log.error('kill: %s' % e)
420
421                                 c = self.jobs[pid]
422                                 c.active_state = 'killed'
423
424                         while self.jobs:
425                                 self.wait()
426
427                         raise
428
429                 if self.error:
430                         raise Error()
431
432         def run_iteration(self):
433                 if self.can_start_install():
434                         c = self.wait_install.pop(0)
435                         self.start_install(c)
436                         self.in_install = c
437
438                 while self.can_start_build():
439                         found = False
440
441                         for c in self.wait_build:
442                                 if self.is_buildable(c):
443                                         found = True
444
445                                         if not self.try_cache(c):
446                                                 self.start_build(c)
447
448                                         self.wait_build.remove(c)
449                                         break
450
451                         if not found:
452                                 break
453
454                 if self.jobs:
455                         self.wait()
456                         return True
457                 else:
458                         if self.wait_build and not self.error:
459                                 raise Error('Circular dependencies?')
460                         return False
461
462         def progress(self, increment=1):
463                 self.progress_now += increment
464
465         def message(self, arg1, arg2=None):
466                 if arg2 is None:
467                         message = arg1
468                 else:
469                         message = '%-10s %s' % (arg1, arg2)
470
471                 progress = '%3d%%' % \
472                            (self.progress_now *100 / self.progress_total)
473
474                 print progress, message
475
476         def is_buildable(self, c):
477                 if c.active_state:
478                         if config.debug:
479                                 self.message('Component %s is in state: %s' % \
480                                         (c.name, c.active_state))
481                         return False
482
483                 for dep in c.active_depends:
484                         if config.components[dep.name].active_state != 'done':
485                                 if config.debug:
486                                         self.message('Component %s depends ' \
487                                                 'on uninstalled %s' % \
488                                                 (c.name, dep.name))
489                                 return False
490
491                 return True
492
493         def try_cache(self, c):
494                 if not cache.contains(c):
495                         return False
496
497                 c.active_state = 'done'
498                 self.progress(2)
499
500                 if config.debug:
501                         self.message('Cached', c.repo.path)
502
503                 return True
504
505         def can_start_build(self):
506                 return self.can_start_job()
507
508         def can_start_install(self):
509                 return not self.in_install and self.wait_install and \
510                        self.can_start_job()
511
512         def can_start_job(self):
513                 return not self.error and len(self.jobs) < self.max_jobs
514
515         def start_build(self, c):
516                 self.message('Building', c.repo.path)
517                 self.start_job(c, 'build_matrix_component', 'in-build')
518
519         def start_install(self, c):
520                 self.message('Installing', c.repo.path)
521                 self.start_job(c, 'install_matrix_component', 'in-install')
522
523         def start_job(self, c, make_target, state):
524                 pid = spawn_make(c, self.max_make_jobs, make_target)
525                 self.jobs[pid] = c
526                 c.active_state = state
527
528         def wait(self):
529                 pid, status = os.wait()
530
531                 if status:
532                         self.error = True
533
534                 c = self.jobs[pid]
535                 del self.jobs[pid]
536
537                 if c.active_state == 'in-build':
538                         if status:
539                                 log.error('Failed to build %s' % c.name)
540                                 return
541
542                         self.progress()
543                         if config.debug:
544                                 self.message('Built', c.repo.path)
545
546                         c.active_state = 'wait-install'
547                         self.wait_install.append(c)
548
549                 elif c.active_state == 'in-install':
550                         assert c == self.in_install
551
552                         if status:
553                                 log.error('Failed to install %s' % c.name)
554                                 return
555
556                         self.progress()
557                         self.message('Finished', c.repo.path)
558
559                         c.active_state = 'done'
560                         self.in_install = None
561                         cache.update(c)
562
563                 elif c.active_state != 'killed':
564                         raise Error('Unexpected state: %s' % c.active_state)
565
566 def spawn_make(c, jobs, target):
567         board = config.boards[config.board]
568
569         make = os.getenv('MAKE', 'make')
570         makefile = os.path.join(config.script_dir, 'matrix.mak')
571         workdir = os.path.join(config.top_dir, 'src', c.name)
572         args = [make, '--no-print-directory', '-f', makefile, '-C', workdir,
573                 '-j', str(jobs), target,
574                 'MATRIX_TOPDIR='    + os.path.abspath(config.top_dir),
575                 'MATRIX_SCRIPTDIR=' + os.path.abspath(config.script_dir),
576                 'MATRIX_COMPONENT=' + c.name,
577                 'MATRIX_ARCH='      + board.arch,
578                 'MATRIX_GCC_MARCH=' + board.gcc_march,
579                 'MATRIX_GCC_MCPU='  + board.gcc_mcpu,
580                 'MATRIX_GCC_MFPU='  + board.gcc_mfpu,
581                 'MATRIX_GCC_OPTIONS=' + board.gcc_options,
582                 'MATRIX_GNU_HOST='  + board.gnu_host,
583                 'MATRIX_LIBC='      + config.libc]
584
585         for flag in config.flags:
586                 args.append(flag + '=1')
587
588         if config.verbose:
589                 args.append('MATRIX_VERBOSE=1')
590
591         if config.debug:
592                 print 'Executing:', ' '.join(args)
593
594         return os.spawnvp(os.P_NOWAIT, args[0], args)
595
596 def clean_components(targets):
597         if not targets:
598                 targets = config.components.keys()
599
600         for name in targets:
601                 c = config.components[name]
602                 print 'Cleaning', c.repo.path
603
604                 cache.remove(c)
605
606                 files = git.ls_files(c.repo.path, ['-o'])
607                 paths = [os.path.join(c.repo.path, i) for i in files]
608                 paths.sort()
609                 paths.reverse()
610
611                 for path in paths:
612                         if git.contains_database(path):
613                                 continue
614
615                         if config.debug:
616                                 print 'Removing', path
617
618                         if os.path.islink(path) or not os.path.isdir(path):
619                                 os.remove(path)
620                         else:
621                                 remove_tree(path)
622
623                 for repo in (c.repo, c.meta):
624                         files = git.ls_files(repo.path, ['-m', '-d'])
625                         if files:
626                                 log.error('Dirty files left in %s' % repo.path)
627
628 def for_each_repository(func, targets=None):
629         if not targets:
630                 targets = config.components.keys()
631
632         for name in targets:
633                 c = config.components[name]
634                 if git.contains_database(c.repo.path):
635                         func(c.repo)
636                 func(c.meta)
637
638 def rebase_components(targets):
639         for_each_repository(rebase_repository, targets)
640
641 def pull_components(targets):
642         for_each_repository(pull_repository, targets)
643
644 def rebase_repository(repo):
645         print 'Rebasing', repo.path
646         git.remote_update(repo.path)
647         git.rebase(repo.path)
648
649 def pull_repository(repo):
650         print 'Pulling', repo.path
651         git.pull(repo.path)
652
653 def source_dist_components(targets):
654         if not targets:
655                 targets = config.components.keys()
656
657         for name in targets:
658                 c = config.components[name]
659                 generate_component_changes(c, 'dist')
660                 package_component_sources(c, 'dist')
661
662 def generate_component_changes(c, location):
663         print 'Generating change log for', c.repo.path
664
665         path = os.path.join(location, c.name) + '.changes'
666
667         pathdir = os.path.dirname(path)
668         if not os.path.exists(pathdir):
669                 os.makedirs(pathdir)
670
671         fd = os.open(path, os.O_WRONLY | os.O_CREAT, 0644)
672         git.log(c.repo.path, [c.get_active_tag()], fd=fd)
673         os.close(fd)
674
675 def package_component_sources(c, location):
676         print 'Archiving', c.repo.path
677
678         rev = git.describe(c.repo.path)
679         if rev:
680                 rev = '_' + rev
681         else:
682                 rev = ''
683
684         path = os.path.join(location, c.name) + rev + '.tar.bz2'
685         if os.path.exists(path):
686                 os.remove(path)
687
688         pathdir = os.path.dirname(path)
689         if not os.path.exists(pathdir):
690                 os.makedirs(pathdir)
691
692         git.archive(c.repo.path, path,
693                     prefix=os.path.basename(c.name) + '/',
694                     branch=c.get_active_tag())
695
696 def build_rootfs(clean=True, rootfs_only=False, jffs2_only=False, devrootfs_only=False):
697         parse_config('rootfs')
698
699         f = os.popen('sb-conf cu')
700         target = f.read()
701         f.close()
702
703         rootfs = RootFS(target.strip())
704         rootfs.include_paths(config.include_paths)
705         rootfs.include_files(config.include_files)
706         rootfs.filter_paths(config.exclude_paths)
707         rootfs.filter_files(config.exclude_files)
708         rootfs.filter_expressions(config.exclude_expressions)
709         rootfs.add_paths(config.created_paths)
710         rootfs.set_devices(config.devices)
711         rootfs.set_change_owner(config.change_owner)
712         rootfs.set_erase_size(config.boards[config.board].flash_erase_size)
713         rootfs.set_pad_size(config.boards[config.board].flash_pad_size)
714
715         if rootfs_only:
716                 rootfs.generate(clean, build_target="rootfs")
717         elif jffs2_only:
718                 rootfs.generate(clean, build_target="jffs2")
719         elif devrootfs_only:
720                 rootfs.generate(clean, build_target="devrootfs")
721         else:
722                 rootfs.generate(clean, build_target="all")