cache module
[matrix.git] / matrix / matrix.py
1 # Copyright (C) 2006-2008 Movial Oy
2 # Authors: Timo Savola <tsavola@movial.fi>
3 #          Toni Timonen
4 #          Kalle Vahlman <kalle.vahlman@movial.fi>
5 #          Tuomas Kulve <tuomas.kulve@movial.fi>
6
7 import glob
8 import os
9 import re
10 import signal
11 import sys
12 import tarfile
13 from sets import Set as set
14
15 import cache
16 import config
17 import git
18 from rootfs import RootFS
19
20 Error = RuntimeError
21
22 error_log = []
23
24 def log_error(msg):
25         error_log.append(msg)
26
27 def main():
28         try:
29                 try:
30                         logged_main()
31                 except:
32                         if not error_log:
33                                 print >>sys.stderr
34                         raise
35         finally:
36                 if error_log:
37                         print >>sys.stderr
38                         for msg in error_log:
39                                 print >>sys.stderr, msg
40
41 def logged_main():
42         parse_config('config.local')
43
44         command, params, targets = parse_args(sys.argv)
45
46         parse_config('config')
47         parse_config('boards')
48         parse_config('components')
49
50         if 'global-cache' in config.flags:
51                 config.cache_dir = config.global_cache_dir
52
53         if not os.path.exists(config.cache_dir):
54                 os.makedirs(config.cache_dir)
55
56         all_targets = update_components()
57
58         for name in targets:
59                 if name not in config.components:
60                         raise Error('Component "%s" does not exist' % name)
61
62         if not targets:
63                 targets = all_targets
64
65         if command == 'install':
66                 build_components(targets, **params)
67         elif command == 'clone':
68                 clone_components(targets)
69         elif command == 'clean':
70                 clean_components(targets)
71         elif command == 'rebase':
72                 rebase_components(targets)
73         elif command == 'pull':
74                 pull_components(targets)
75         elif command == 'source-dist':
76                 source_dist_components(targets)
77         elif command == 'rootfs':
78                 build_rootfs(**params)
79         else:
80                 raise Error('Invalid command: ' + command)
81
82 def help(file, args):
83         print >>file, '''
84 Copyright (C) 2006-2008 Movial Oy
85
86 Usage: %(progname)s [<options>] <command> [<params>] [<components>]
87
88 If no components are specified, all of them will be targeted.  All component
89 metadata will be downloaded regardless of the specified command and components.
90
91 Options:
92         -v              Verbose output.
93         -d              Debug output.
94         -r URL          Specify the root for component git repos to clone from.
95                         If this option is not given, the roots specified in the
96                         components file will be used.  This option may be
97                         specified multiple times.
98         -f              Build components with dirty files.
99         -h, --help      Print this help text.
100
101 Commands and parameters:
102         clone           Download the components' git repositories.
103         install         Download, build and install the components.
104                 -j N            Execute N build jobs in parallel.
105                 -mj N           Run component builds with make -j N.
106         clean           Remove all non-tracked files from the component git
107                         repository directories.
108         rebase          Update repositories from server by rebasing.
109         pull            Update repositories from server by merging.
110         source-dist     Download and package the component sources.
111         rootfs          Build a rootfs from the current target installation.
112                 -n              Do not clean env.faked and stripped directory.
113                 -r              Only generate a stripped rootfs.
114                 -j              Only generate a jffs2 image
115                 -d              Only generate a rootstrap (development rootfs).
116 ''' % {'progname': args[0]}
117
118 def parse_args(args):
119         def parse_jobs(arg):
120                 jobs = int(arg)
121                 if jobs <= 0:
122                         raise Error('Please specify a valid number of jobs')
123                 return jobs
124
125         command = None
126         params = {}
127         targets = []
128
129         i = 1
130         while i < len(args):
131                 if args[i].startswith('-'):
132                         if not command:
133                                 if args[i] == '-v':
134                                         config.verbose = True
135                                 elif args[i] == '-d':
136                                         config.debug = True
137                                 elif args[i] == '-r':
138                                         i += 1
139                                         config.roots.append(args[i])
140                                 elif args[i] == '-f':
141                                         config.force = True
142                                 elif args[i] in ('-h', '--help'):
143                                         help(sys.stdout, args)
144                                         sys.exit(0)
145                                 else:
146                                         raise Error('Bad option: ' + args[i])
147                         elif command == 'install':
148                                 if args[i] == '-j':
149                                         i += 1
150                                         params['build_jobs'] = parse_jobs(args[i])
151                                 elif args[i] == '-mj':
152                                         i += 1
153                                         params['make_jobs'] = parse_jobs(args[i])
154                                 else:
155                                         raise Error('Bad parameter: ' + args[i])
156                         elif command == 'rootfs':
157                                 if args[i] == '-n':
158                                         i += 1
159                                         params['clean'] = False
160                                 elif args[i] == '-r':
161                                         i += 1
162                                         params['rootfs_only'] = True
163                                 elif args[i] == '-j':
164                                         i += 1
165                                         params['jffs2_only'] = True
166                                 elif args[i] == '-d':
167                                         i += 1
168                                         params['devrootfs_only'] = True
169                         else:
170                                 raise Error('Command takes no parameters')
171                 elif not command:
172                         command = args[i]
173                 else:
174                         targets.append(args[i])
175                 i += 1
176
177         if not command:
178                 help(sys.stderr, args)
179                 sys.exit(1)
180
181         return command, params, targets
182
183 def parse_config(path):
184         if os.path.isabs(path):
185                 path = os.path.join(config.top_dir, path)
186
187         if os.path.exists(path):
188                 print 'Reading', path
189                 execfile(path, config.__dict__, config.__dict__)
190
191 def update_components():
192         targets = []
193         packages = {}
194
195         for c in config.components.itervalues():
196                 update_component_url(c)
197                 clone_metadata(c)
198                 update_component_packages(c, targets, packages)
199                 c.active_depends = []
200
201         update_component_depends(packages)
202
203         return targets
204
205 def update_component_url(c):
206         update_repository_url(c.repo)
207         update_repository_url(c.meta)
208
209 def update_repository_url(repo):
210         if git.contains_database(repo.path):
211                 url = git.getvar(repo.path, 'remote.origin.url')
212                 if url:
213                         repo.active_url = url
214                         if config.debug:
215                                 print 'Using', repo.active_url
216                         return
217
218         for root in config.roots:
219                 for suffix in ('.git', ''):
220                         url = '%s/%s%s' % (root, repo.name, suffix)
221
222                         if config.debug:
223                                 print 'Trying', url
224
225                         if not git.peek_remote(url, quiet=True):
226                                 continue
227
228                         repo.active_url = url
229                         if config.debug:
230                                 print 'Found', repo.active_url
231                         return
232
233         raise Error('Failed to locate repository: ' + repo.name)
234
235 def update_component_depends(packages):
236         for pkg in packages.itervalues():
237                 for spec in pkg.depends.split():
238                         depname = Depend(spec).name
239                         deppkg = packages.get(depname)
240
241                         if not deppkg:
242                                 log_error('Package %s depends on ' \
243                                           'non-existent package %s', \
244                                           pkg.name, depname)
245                                 continue
246
247                         if deppkg.component == pkg.component:
248                                 continue
249
250                         pkg.component.active_depends.append(deppkg.component)
251
252         fail = False
253         for pkg in packages.itervalues():
254                 for spec in pkg.depends.split():
255                         if not Depend(spec).check(packages):
256                                 fail = True
257                                 log_error('Dependency %s failed for %s' % \
258                                           (spec, pkg.name))
259
260                 for spec in pkg.conflicts.split():
261                         if Depend(spec).check(packages):
262                                 fail = True
263                                 log_error('Package %s conflicts with %s' % \
264                                           (pkg.name, spec))
265
266         if fail:
267                 raise Error('Invalid component tree')
268
269 class Depend(object):
270         regex = re.compile(r'([@]?)([^\s:]+)[:]?([<>=]*)([^\s:]*)[:]?(.*)')
271
272         def __init__(self, spec):
273                 match = self.regex.match(spec)
274                 if not match:
275                         raise Error('Bad dependency specification: ' + spec)
276
277                 self.build, self.name, self.tag_op, self.tag, flags \
278                         = match.groups()
279                 self.flags = flags.split()
280
281         def check(self, packages):
282                 # TODO: check version and flags
283                 return self.name in packages
284
285 def execute(args):
286         if config.debug:
287                 print 'Executing:', ' '.join(args)
288
289         if os.spawnvp(os.P_WAIT, args[0], args) != 0:
290                 raise Error('Failed: ' + ' '.join(args))
291
292 def remove_tree(path):
293         execute(['rm', '-rf', path])
294
295 def clone_components(targets):
296         if not targets:
297                 targets = config.components.keys()
298
299         for name in targets:
300                 c = config.components[name]
301                 clone_component(c)
302
303 def clone_component(c, overwrite=False):
304         have_repo = git.contains_database(c.repo.path)
305         have_meta = git.contains_database(c.meta.path)
306
307         if not overwrite and have_repo and have_meta:
308                 return
309
310         if overwrite and os.path.exists(c.repo.path):
311                 print 'Removing', c.repo.path
312
313                 remove_tree(c.repo.path)
314                 have_repo = False
315                 have_meta = False
316
317         if not have_repo:
318                 clone_repository(c.repo)
319                 git.exclude(c.repo.path, 'meta')
320
321         if not have_meta:
322                 clone_repository(c.meta)
323
324 def clone_metadata(c):
325         if not git.contains_database(c.meta.path):
326                 clone_repository(c.meta)
327
328 def clone_repository(repo):
329         print 'Cloning', repo.path
330
331         if os.path.exists(repo.path):
332                 tmp = os.path.join(repo.path, 'tmp')
333                 git.clone(tmp, repo.active_url, checkout=False)
334                 try:
335                         tmpdb = os.path.join(tmp, '.git')
336                         repodb = os.path.join(repo.path, '.git')
337                         if config.debug:
338                                 print 'Renaming "%s" as "%s"' % (tmpdb, repodb)
339                         os.rename(tmpdb, repodb)
340                 finally:
341                         os.rmdir(tmp)
342                 git.checkout(repo.path)
343         else:
344                 git.clone(repo.path, repo.active_url, checkout=True)
345
346 def update_component_packages(c, targets, packages):
347         c.active_packages = {}
348
349         for path in glob.glob(os.path.join(c.meta.path, '*.package')):
350                 name = os.path.basename(path)[:-8]
351
352                 pkg = parse_package(name, c, path)
353                 if not pkg:
354                         continue
355
356                 c.active_packages[name] = pkg
357                 packages[name] = pkg
358
359                 if config.debug:
360                         print 'Component', c.name, 'provides', name
361
362         if c.active_packages:
363                 targets.append(c.name)
364         elif config.debug:
365                 print 'Component', c.name, 'does not provide any packages'
366
367 class Package(object):
368         def __init__(self, name, component):
369                 self.name = name
370                 self.component = component
371
372                 self.depends = []
373                 self.conflicts = []
374                 self.architectures = None
375
376 def parse_package(name, component, path):
377         pkg = Package(name, component)
378         execfile(path, pkg.__dict__, pkg.__dict__)
379
380         if pkg.architectures:
381                 arch = config.boards[config.board].arch
382                 if arch not in pkg.architectures:
383                         return None
384
385         return pkg
386
387 def build_components(targets, build_jobs=1, make_jobs=1):
388         selected = set([config.components[i] for i in targets])
389
390         depends = None
391         while depends is None or depends:
392                 depends = set()
393
394                 for c in selected:
395                         for dep in c.active_depends:
396                                 if dep not in selected:
397                                         depends.add(dep)
398
399                 selected |= depends
400
401         components = list(selected)
402         components.sort()
403
404         if config.debug:
405                 print 'Building components:'
406                 for c in components:
407                         print '\t' + c.name
408
409         builder = Builder(components, build_jobs, make_jobs)
410         builder.run()
411
412 class Builder(object):
413         def __init__(self, components, max_jobs, max_make_jobs):
414                 self.max_jobs = max_jobs
415                 self.max_make_jobs = max_make_jobs
416
417                 self.jobs = {}
418
419                 self.wait_build = set(components)
420                 self.wait_install = []
421                 self.in_install = None
422
423                 self.progress_now = 0
424                 self.progress_total = len(components) * 2
425
426                 self.error = False
427
428         def run(self):
429                 try:
430                         while self.run_iteration():
431                                 pass
432                 except:
433                         for pid in self.jobs:
434                                 try:
435                                         os.kill(pid, signal.SIGTERM)
436                                 except OSError, e:
437                                         log_error('kill: %s' % e)
438
439                                 c = self.jobs[pid]
440                                 c.active_state = 'killed'
441
442                         while self.jobs:
443                                 self.wait()
444
445                         raise
446
447         def run_iteration(self):
448                 if self.can_start_install():
449                         c = self.wait_install.pop(0)
450                         self.start_install(c)
451                         self.in_install = c
452
453                 while self.can_start_build():
454                         found = False
455
456                         for c in self.wait_build:
457                                 if self.is_buildable(c):
458                                         found = True
459
460                                         if not self.try_cache(c):
461                                                 self.start_build(c)
462
463                                         self.wait_build.remove(c)
464                                         break
465
466                         if not found:
467                                 break
468
469                 if self.jobs:
470                         self.wait()
471                         return True
472                 else:
473                         if self.wait_build and not self.error:
474                                 raise Error('Circular dependencies?')
475                         return False
476
477         def progress(self, increment=1):
478                 self.progress_now += increment
479
480         def message(self, arg1, arg2=None):
481                 if arg2 is None:
482                         message = arg1
483                 else:
484                         message = '%-10s %s' % (arg1, arg2)
485
486                 progress = '%3d%%' % \
487                            (self.progress_now *100 / self.progress_total)
488
489                 print progress, message
490
491         def is_buildable(self, c):
492                 if c.active_state:
493                         if config.debug:
494                                 self.message('Component %s is in state: %s' % \
495                                         (c.name, c.active_state))
496                         return False
497
498                 for dep in c.active_depends:
499                         if config.components[dep.name].active_state != 'done':
500                                 if config.debug:
501                                         self.message('Component %s depends ' \
502                                                 'on uninstalled %s' % \
503                                                 (c.name, dep.name))
504                                 return False
505
506                 return True
507
508         def try_cache(self, c):
509                 if not cache.contains(c):
510                         return False
511
512                 c.active_state = 'done'
513                 self.progress(2)
514
515                 if config.debug:
516                         self.message('Cached', c.repo.path)
517
518                 return True
519
520         def can_start_build(self):
521                 return self.can_start_job()
522
523         def can_start_install(self):
524                 return not self.in_install and self.wait_install and \
525                        self.can_start_job()
526
527         def can_start_job(self):
528                 return not self.error and len(self.jobs) < self.max_jobs
529
530         def start_build(self, c):
531                 self.message('Building', c.repo.path)
532                 self.start_job(c, 'build_matrix_component', 'in-build')
533
534         def start_install(self, c):
535                 self.message('Installing', c.repo.path)
536                 self.start_job(c, 'install_matrix_component', 'in-install')
537
538         def start_job(self, c, make_target, state):
539                 pid = spawn_make(c, self.max_make_jobs, make_target)
540                 self.jobs[pid] = c
541                 c.active_state = state
542
543         def wait(self):
544                 pid, status = os.wait()
545
546                 if status:
547                         self.error = True
548
549                 c = self.jobs[pid]
550                 del self.jobs[pid]
551
552                 if c.active_state == 'in-build':
553                         if status:
554                                 log_error('Failed to build %s' % c.name)
555                                 return
556
557                         self.progress()
558                         if config.debug:
559                                 self.message('Built', c.repo.path)
560
561                         c.active_state = 'wait-install'
562                         self.wait_install.append(c)
563
564                 elif c.active_state == 'in-install':
565                         assert c == self.in_install
566
567                         if status:
568                                 log_error('Failed to install %s' % c.name)
569                                 return
570
571                         self.progress()
572                         self.message('Finished', c.repo.path)
573
574                         c.active_state = 'done'
575                         self.in_install = None
576                         cache.update(c)
577
578                 elif c.active_state != 'killed':
579                         raise Error('Unexpected state: %s' % c.active_state)
580
581 def spawn_make(c, jobs, target):
582         board = config.boards[config.board]
583
584         make = os.getenv('MAKE', 'make')
585         makefile = os.path.join(config.script_dir, 'matrix.mak')
586         workdir = os.path.join(config.top_dir, 'src', c.name)
587         args = [make, '--no-print-directory', '-f', makefile, '-C', workdir,
588                 '-j', str(jobs), target,
589                 'MATRIX_TOPDIR='    + os.path.abspath(config.top_dir),
590                 'MATRIX_SCRIPTDIR=' + os.path.abspath(config.script_dir),
591                 'MATRIX_COMPONENT=' + c.name,
592                 'MATRIX_ARCH='      + board.arch,
593                 'MATRIX_GCC_MARCH=' + board.gcc_march,
594                 'MATRIX_GCC_MCPU='  + board.gcc_mcpu,
595                 'MATRIX_GCC_MFPU='  + board.gcc_mfpu,
596                 'MATRIX_GCC_OPTIONS=' + board.gcc_options,
597                 'MATRIX_GNU_HOST='  + board.gnu_host,
598                 'MATRIX_LIBC='      + config.libc]
599
600         for flag in config.flags:
601                 args.append(flag + '=1')
602
603         if config.verbose:
604                 args.append('MATRIX_VERBOSE=1')
605
606         if config.debug:
607                 print 'Executing:', ' '.join(args)
608
609         return os.spawnvp(os.P_NOWAIT, args[0], args)
610
611 def clean_components(targets):
612         if not targets:
613                 targets = config.components.keys()
614
615         for name in targets:
616                 c = config.components[name]
617                 print 'Cleaning', c.repo.path
618
619                 cache.remove(c)
620
621                 files = git.ls_files(c.repo.path, ['-o'])
622                 paths = [os.path.join(c.repo.path, i) for i in files]
623                 paths.sort()
624                 paths.reverse()
625
626                 for path in paths:
627                         if git.contains_database(path):
628                                 continue
629
630                         if config.debug:
631                                 print 'Removing', path
632
633                         if os.path.islink(path) or not os.path.isdir(path):
634                                 os.remove(path)
635                         else:
636                                 remove_tree(path)
637
638                 for repo in (c.repo, c.meta):
639                         files = git.ls_files(repo.path, ['-m', '-d'])
640                         if files:
641                                 log_error('Dirty files left in %s' % repo.path)
642
643 def for_each_repository(func, targets=None):
644         if not targets:
645                 targets = config.components.keys()
646
647         for name in targets:
648                 c = config.components[name]
649                 if git.contains_database(c.repo.path):
650                         func(c.repo)
651                 func(c.meta)
652
653 def rebase_components(targets):
654         for_each_repository(rebase_repository, targets)
655
656 def pull_components(targets):
657         for_each_repository(pull_repository, targets)
658
659 def rebase_repository(repo):
660         print 'Rebasing', repo.path
661         git.remote_update(repo.path)
662         git.rebase(repo.path)
663
664 def pull_repository(repo):
665         print 'Pulling', repo.path
666         git.pull(repo.path)
667
668 def source_dist_components(targets):
669         if not targets:
670                 targets = config.components.keys()
671
672         for name in targets:
673                 c = config.components[name]
674                 generate_component_changes(c, 'dist')
675                 package_component_sources(c, 'dist')
676
677 def generate_component_changes(c, location):
678         print 'Generating change log for', c.repo.path
679
680         path = os.path.join(location, c.name) + '.changes'
681
682         pathdir = os.path.dirname(path)
683         if not os.path.exists(pathdir):
684                 os.makedirs(pathdir)
685
686         fd = os.open(path, os.O_WRONLY | os.O_CREAT, 0644)
687         git.log(c.repo.path, [c.get_active_tag()], fd=fd)
688         os.close(fd)
689
690 def package_component_sources(c, location):
691         print 'Archiving', c.repo.path
692
693         rev = git.describe(c.repo.path)
694         if rev:
695                 rev = '_' + rev
696         else:
697                 rev = ''
698
699         path = os.path.join(location, c.name) + rev + '.tar.bz2'
700         if os.path.exists(path):
701                 os.remove(path)
702
703         pathdir = os.path.dirname(path)
704         if not os.path.exists(pathdir):
705                 os.makedirs(pathdir)
706
707         git.archive(c.repo.path, path,
708                     prefix=os.path.basename(c.name) + '/',
709                     branch=c.get_active_tag())
710
711 def build_rootfs(clean=True, rootfs_only=False, jffs2_only=False, devrootfs_only=False):
712         parse_config('rootfs')
713
714         f = os.popen('sb-conf cu')
715         target = f.read()
716         f.close()
717
718         rootfs = RootFS(target.strip())
719         rootfs.include_paths(config.include_paths)
720         rootfs.include_files(config.include_files)
721         rootfs.filter_paths(config.exclude_paths)
722         rootfs.filter_files(config.exclude_files)
723         rootfs.filter_expressions(config.exclude_expressions)
724         rootfs.add_paths(config.created_paths)
725         rootfs.set_devices(config.devices)
726         rootfs.set_change_owner(config.change_owner)
727         rootfs.set_erase_size(config.boards[config.board].flash_erase_size)
728         rootfs.set_pad_size(config.boards[config.board].flash_pad_size)
729
730         if rootfs_only:
731                 rootfs.generate(clean, build_target="rootfs")
732         elif jffs2_only:
733                 rootfs.generate(clean, build_target="jffs2")
734         elif devrootfs_only:
735                 rootfs.generate(clean, build_target="devrootfs")
736         else:
737                 rootfs.generate(clean, build_target="all")