remove flock dependency by invoking make separately for installation
[matrix.git] / matrix / matrix.py
1 # Copyright (C) 2006-2008 Movial Oy
2 # Authors: Timo Savola <tsavola@movial.fi>
3 #          Toni Timonen
4 #          Kalle Vahlman <kalle.vahlman@movial.fi>
5 #          Tuomas Kulve <tuomas.kulve@movial.fi>
6
7 import glob
8 import os
9 import re
10 import sys
11 import tarfile
12 from sets import Set as set
13
14 import config
15 import git
16 from rootfs import RootFS
17
18 Error = RuntimeError
19
20 def main():
21         parse_config('config.local')
22
23         command, params, targets = parse_args(sys.argv)
24
25         parse_config('config')
26         parse_config('boards')
27         parse_config('components')
28
29         if 'global-cache' in config.flags:
30                 config.cache_dir = config.global_cache_dir
31
32         if not os.path.exists(config.cache_dir):
33                 os.makedirs(config.cache_dir)
34
35         all_targets = update_components()
36
37         for name in targets:
38                 if name not in config.components:
39                         raise Error('Component "%s" does not exist' % name)
40
41         if not targets:
42                 targets = all_targets
43
44         if command == 'install':
45                 build_components(targets, **params)
46         elif command == 'clone':
47                 clone_components(targets)
48         elif command == 'clean':
49                 clean_components(targets)
50         elif command == 'rebase':
51                 rebase_components(targets)
52         elif command == 'pull':
53                 pull_components(targets)
54         elif command == 'source-dist':
55                 source_dist_components(targets)
56         elif command == 'rootfs':
57                 build_rootfs(**params)
58         else:
59                 raise Error('Invalid command: ' + command)
60
61 def help(file, args):
62         print >>file, '''
63 Copyright (C) 2006-2008 Movial Oy
64
65 Usage: %(progname)s [<options>] <command> [<params>] [<components>]
66
67 If no components are specified, all of them will be targeted.  All component
68 metadata will be downloaded regardless of the specified command and components.
69
70 Options:
71         -v              Verbose output.
72         -d              Debug output.
73         -r URL          Specify the root for component git repos to clone from.
74                         If this option is not given, the roots specified in the
75                         components file will be used.  This option may be
76                         specified multiple times.
77         -f              Build components with dirty files.
78         -h, --help      Print this help text.
79
80 Commands and parameters:
81         clone           Download the components' git repositories.
82         install         Download, build and install the components.
83                 -j N            Execute N build jobs in parallel.
84                 -mj N           Run component builds with make -j N.
85         clean           Remove all non-tracked files from the component git
86                         repository directories.
87         rebase          Update repositories from server by rebasing.
88         pull            Update repositories from server by merging.
89         source-dist     Download and package the component sources.
90         rootfs          Build a rootfs from the current target installation.
91                 -n              Do not clean env.faked and stripped directory.
92                 -r              Only generate a stripped rootfs.
93                 -j              Only generate a jffs2 image
94                 -d              Only generate a rootstrap (development rootfs).
95 ''' % {'progname': args[0]}
96
97 def parse_args(args):
98         def parse_jobs(arg):
99                 jobs = int(arg)
100                 if jobs <= 0:
101                         raise Error('Please specify a valid number of jobs')
102                 return jobs
103
104         command = None
105         params = {}
106         targets = []
107
108         i = 1
109         while i < len(args):
110                 if args[i].startswith('-'):
111                         if not command:
112                                 if args[i] == '-v':
113                                         config.verbose = True
114                                 elif args[i] == '-d':
115                                         config.debug = True
116                                 elif args[i] == '-r':
117                                         i += 1
118                                         config.roots.append(args[i])
119                                 elif args[i] == '-f':
120                                         config.force = True
121                                 elif args[i] in ('-h', '--help'):
122                                         help(sys.stdout, args)
123                                         sys.exit(0)
124                                 else:
125                                         raise Error('Bad option: ' + args[i])
126                         elif command == 'install':
127                                 if args[i] == '-j':
128                                         i += 1
129                                         params['build_jobs'] = parse_jobs(args[i])
130                                 elif args[i] == '-mj':
131                                         i += 1
132                                         params['make_jobs'] = parse_jobs(args[i])
133                                 else:
134                                         raise Error('Bad parameter: ' + args[i])
135                         elif command == 'rootfs':
136                                 if args[i] == '-n':
137                                         i += 1
138                                         params['clean'] = False
139                                 elif args[i] == '-r':
140                                         i += 1
141                                         params['rootfs_only'] = True
142                                 elif args[i] == '-j':
143                                         i += 1
144                                         params['jffs2_only'] = True
145                                 elif args[i] == '-d':
146                                         i += 1
147                                         params['devrootfs_only'] = True
148                         else:
149                                 raise Error('Command takes no parameters')
150                 elif not command:
151                         command = args[i]
152                 else:
153                         targets.append(args[i])
154                 i += 1
155
156         if not command:
157                 help(sys.stderr, args)
158                 sys.exit(1)
159
160         return command, params, targets
161
162 def parse_config(path):
163         if os.path.isabs(path):
164                 path = os.path.join(config.top_dir, path)
165
166         if os.path.exists(path):
167                 print 'Reading', path
168                 execfile(path, config.__dict__, config.__dict__)
169
170 def update_components():
171         targets = []
172         packages = {}
173
174         for c in config.components.itervalues():
175                 update_component_url(c)
176                 clone_metadata(c)
177                 update_component_packages(c, targets, packages)
178                 c.active_depends = []
179
180         update_component_depends(packages)
181
182         return targets
183
184 def update_component_url(c):
185         update_repository_url(c.repo)
186         update_repository_url(c.meta)
187
188 def update_repository_url(repo):
189         if git.contains_database(repo.path):
190                 url = git.getvar(repo.path, 'remote.origin.url')
191                 if url:
192                         repo.active_url = url
193                         if config.debug:
194                                 print 'Using', repo.active_url
195                         return
196
197         for root in config.roots:
198                 for suffix in ('.git', ''):
199                         url = '%s/%s%s' % (root, repo.name, suffix)
200
201                         if config.debug:
202                                 print 'Trying', url
203
204                         if not git.peek_remote(url, quiet=True):
205                                 continue
206
207                         repo.active_url = url
208                         if config.debug:
209                                 print 'Found', repo.active_url
210                         return
211
212         raise Error('Failed to locate repository: ' + repo.name)
213
214 def update_component_depends(packages):
215         for pkg in packages.itervalues():
216                 for spec in pkg.depends.split():
217                         depname = Depend(spec).name
218                         deppkg = packages.get(depname)
219
220                         if not deppkg:
221                                 print >>sys.stderr, 'Package', pkg.name, \
222                                         'depends on non-existent package', depname
223                                 continue
224
225                         if deppkg.component == pkg.component:
226                                 continue
227
228                         pkg.component.active_depends.append(deppkg.component)
229
230         fail = False
231         for pkg in packages.itervalues():
232                 for spec in pkg.depends.split():
233                         if not Depend(spec).check(packages):
234                                 fail = True
235                                 print >>sys.stderr, 'Dependency', spec, \
236                                         'failed for', pkg.name
237
238                 for spec in pkg.conflicts.split():
239                         if Depend(spec).check(packages):
240                                 fail = True
241                                 print >>sys.stderr, pkg.name, \
242                                         'conflicts with', spec
243
244         if fail:
245                 raise Error('Invalid component tree')
246
247 class Depend(object):
248         regex = re.compile(r'([@]?)([^\s:]+)[:]?([<>=]*)([^\s:]*)[:]?(.*)')
249
250         def __init__(self, spec):
251                 match = self.regex.match(spec)
252                 if not match:
253                         raise Error('Bad dependency specification: ' + spec)
254
255                 self.build, self.name, self.tag_op, self.tag, flags \
256                         = match.groups()
257                 self.flags = flags.split()
258
259         def check(self, packages):
260                 # TODO: check version and flags
261                 return self.name in packages
262
263 def execute(args):
264         if config.debug:
265                 print 'Executing:', ' '.join(args)
266
267         if os.spawnvp(os.P_WAIT, args[0], args) != 0:
268                 raise Error('Failed: ' + ' '.join(args))
269
270 def remove_tree(path):
271         execute(['rm', '-rf', path])
272
273 def clone_components(targets):
274         if not targets:
275                 targets = config.components.keys()
276
277         for name in targets:
278                 c = config.components[name]
279                 clone_component(c)
280
281 def clone_component(c, overwrite=False):
282         have_repo = git.contains_database(c.repo.path)
283         have_meta = git.contains_database(c.meta.path)
284
285         if not overwrite and have_repo and have_meta:
286                 return
287
288         if overwrite and os.path.exists(c.repo.path):
289                 print 'Removing', c.repo.path
290
291                 remove_tree(c.repo.path)
292                 have_repo = False
293                 have_meta = False
294
295         if not have_repo:
296                 clone_repository(c.repo)
297                 git.exclude(c.repo.path, 'meta')
298
299         if not have_meta:
300                 clone_repository(c.meta)
301
302 def clone_metadata(c):
303         if not git.contains_database(c.meta.path):
304                 clone_repository(c.meta)
305
306 def clone_repository(repo):
307         print 'Cloning', repo.path
308
309         if os.path.exists(repo.path):
310                 tmp = os.path.join(repo.path, 'tmp')
311                 git.clone(tmp, repo.active_url, checkout=False)
312                 try:
313                         tmpdb = os.path.join(tmp, '.git')
314                         repodb = os.path.join(repo.path, '.git')
315                         if config.debug:
316                                 print 'Renaming "%s" as "%s"' % (tmpdb, repodb)
317                         os.rename(tmpdb, repodb)
318                 finally:
319                         os.rmdir(tmp)
320                 git.checkout(repo.path)
321         else:
322                 git.clone(repo.path, repo.active_url, checkout=True)
323
324 def update_component_packages(c, targets, packages):
325         c.active_packages = {}
326
327         for path in glob.glob(os.path.join(c.meta.path, '*.package')):
328                 name = os.path.basename(path)[:-8]
329
330                 pkg = parse_package(name, c, path)
331                 if not pkg:
332                         continue
333
334                 c.active_packages[name] = pkg
335                 packages[name] = pkg
336
337                 if config.debug:
338                         print 'Component', c.name, 'provides', name
339
340         if c.active_packages:
341                 targets.append(c.name)
342         elif config.debug:
343                 print 'Component', c.name, 'does not provide any packages'
344
345 class Package(object):
346         def __init__(self, name, component):
347                 self.name = name
348                 self.component = component
349
350                 self.depends = []
351                 self.conflicts = []
352                 self.architectures = None
353
354 def parse_package(name, component, path):
355         pkg = Package(name, component)
356         execfile(path, pkg.__dict__, pkg.__dict__)
357
358         if pkg.architectures:
359                 arch = config.boards[config.board].arch
360                 if arch not in pkg.architectures:
361                         return None
362
363         return pkg
364
365 def build_components(targets, build_jobs=1, make_jobs=1):
366         selected = set([config.components[i] for i in targets])
367
368         depends = None
369         while depends is None or depends:
370                 depends = set()
371
372                 for c in selected:
373                         for dep in c.active_depends:
374                                 if dep not in selected:
375                                         depends.add(dep)
376
377                 selected |= depends
378
379         components = list(selected)
380         components.sort()
381
382         if config.debug:
383                 print 'Building components:'
384                 for c in components:
385                         print '\t' + c.name
386
387         jobs = {}
388         built = []
389         installing = None
390
391         while True:
392                 if not installing and built and len(jobs) < build_jobs:
393                         c = built.pop(0)
394                         print 'Installing', c.name
395                         start_install_job(c, jobs)
396                         installing = c
397
398                 found = False
399                 for c in components:
400                         if len(jobs) >= build_jobs:
401                                 break
402
403                         if component_buildable(c):
404                                 found = True
405
406                                 if not component_cached(c):
407                                         print 'Building', c.name
408                                         start_build_job(c, jobs, make_jobs)
409                                 elif config.debug:
410                                         print 'Component', c.name, \
411                                               'found from cache'
412                                         c.active_state = 'installed'
413
414                 if not found and not jobs:
415                         for c in components:
416                                 if c.active_state != 'installed':
417                                         raise Error('Internal error')
418                         break
419
420                 c = wait_for_job(jobs, built)
421                 if c is not None and c == installing:
422                         installing = None
423
424 def component_buildable(c):
425         if c.active_state:
426                 if config.debug:
427                         print 'Component', c.name, 'is in state:', c.active_state
428                 return False
429
430         for dep in c.active_depends:
431                 if config.components[dep.name].active_state != 'installed':
432                         if config.debug:
433                                 print 'Component', c.name, \
434                                       'depends on uninstalled', dep.name
435                         return False
436
437         return True
438
439 def component_cached(c):
440         if c.from_platform:
441                 return True
442
443         for repo in (c.repo, c.meta):
444                 if git.ls_files(repo.path, ['-m', '-d']):
445                         if config.force:
446                                 return False
447                         else:
448                                 raise Error('Dirty files in ' + repo.path)
449
450         path = os.path.join(config.cache_dir, c.name)
451         if not os.path.exists(path):
452                 return False
453
454         file = open(path, 'r')
455         line = file.readline()
456         file.close()
457
458         match = re.match(r'([^\s]+)[\s]?([^\s]*)', line)
459         if not match:
460                 return False
461
462         hash, flagstring = match.groups()
463
464         if hash != get_component_hash(c):
465                 print 'Component has been changed:', c.name
466                 return False
467
468         if flagstring:
469                 flags = flagstring.split(',')
470         else:
471                 flags = []
472
473         if set(flags) != set(c.flags):
474                 print 'Component flags have been changed:', c.name
475                 return False
476
477         return True
478
479 def update_cache(c):
480         path = os.path.join(config.cache_dir, c.name)
481
482         dir = os.path.dirname(path)
483         if not os.path.exists(dir):
484                 os.makedirs(dir)
485
486         file = open(path, 'w')
487         done = False
488         try:
489                 print >>file, get_component_hash(c), ','.join(c.flags)
490                 done = True
491         finally:
492                 file.close()
493                 if not done:
494                         os.remove(path)
495
496 def get_component_hash(c):
497         return '%s+%s' % (get_repository_hash(c.repo), get_repository_hash(c.meta))
498
499 def get_repository_hash(repo):
500         if repo.active_hash is None:
501                 repo.active_hash = git.rev_parse(repo.path, 'HEAD')
502         return repo.active_hash
503
504 def spawn_job(c, make_jobs, target):
505         board = config.boards[config.board]
506
507         makefile = os.path.join(config.script_dir, 'matrix.mak')
508
509         workdir = os.path.join(config.top_dir, 'src', c.name)
510         args = ['make', '--no-print-directory', '-f', makefile, '-C', workdir,
511                 '-j', str(make_jobs), target,
512                 'MATRIX_TOPDIR='    + os.path.abspath(config.top_dir),
513                 'MATRIX_SCRIPTDIR=' + os.path.abspath(config.script_dir),
514                 'MATRIX_COMPONENT=' + c.name,
515                 'MATRIX_ARCH='      + board.arch,
516                 'MATRIX_GCC_MARCH=' + board.gcc_march,
517                 'MATRIX_GCC_MCPU='  + board.gcc_mcpu,
518                 'MATRIX_GCC_MFPU='  + board.gcc_mfpu,
519                 'MATRIX_GCC_OPTIONS=' + board.gcc_options,
520                 'MATRIX_GNU_HOST='  + board.gnu_host,
521                 'MATRIX_LIBC='      + config.libc]
522
523         for flag in config.flags:
524                 args.append(flag + '=1')
525
526         if config.verbose:
527                 args.append('MATRIX_VERBOSE=1')
528
529         if config.debug:
530                 print 'Executing:', ' '.join(args)
531
532         return os.spawnvp(os.P_NOWAIT, args[0], args)
533
534 def start_build_job(c, jobs, make_jobs):
535         pid = spawn_job(c, make_jobs, 'build_matrix_component')
536         c.active_state = 'building'
537         jobs[pid] = c
538
539 def start_install_job(c, jobs):
540         pid = spawn_job(c, 1, 'install_matrix_component')
541         c.active_state = 'installing'
542         jobs[pid] = c
543
544 def wait_for_job(jobs, built):
545         if not jobs:
546                 return None
547
548         pid, status = os.wait()
549
550         c = jobs.get(pid)
551         if not c:
552                 return None
553
554         del jobs[pid]
555
556         if status == 0:
557                 if c.active_state == 'building':
558                         print 'Built', c.repo.path
559                         c.active_state = 'built'
560                         built.append(c)
561                 elif c.active_state == 'installing':
562                         print 'Installed', c.repo.path
563                         c.active_state = 'installed'
564                         update_cache(c)
565                 else:
566                         raise Error('Internal error')
567         else:
568                 c.active_state = 'error'
569                 print >>sys.stderr, c.name, 'failed with status:', status
570
571                 while jobs:
572                         wait_for_job(jobs, built)
573
574                 raise Error('Build failed')
575
576         return c
577
578 def clean_components(targets):
579         if not targets:
580                 targets = config.components.keys()
581
582         for name in targets:
583                 c = config.components[name]
584                 print 'Cleaning', c.repo.path
585
586                 files = git.ls_files(c.repo.path, ['-o'])
587                 paths = [os.path.join(c.repo.path, i) for i in files]
588                 paths.sort()
589                 paths.reverse()
590
591                 cache = os.path.join(config.cache_dir, name)
592                 if os.path.exists(cache):
593                         paths.append(cache)
594
595                 for path in paths:
596                         if git.contains_database(path):
597                                 continue
598
599                         if config.debug:
600                                 print 'Removing', path
601
602                         if os.path.islink(path) or not os.path.isdir(path):
603                                 os.remove(path)
604                         else:
605                                 remove_tree(path)
606
607                 for repo in (c.repo, c.meta):
608                         files = git.ls_files(repo.path, ['-m', '-d'])
609                         if files:
610                                 print len(files), 'dirty files left in', repo.path
611
612 def for_each_repository(func, targets=None):
613         if not targets:
614                 targets = config.components.keys()
615
616         for name in targets:
617                 c = config.components[name]
618                 if git.contains_database(c.repo.path):
619                         func(c.repo)
620                 func(c.meta)
621
622 def rebase_components(targets):
623         for_each_repository(rebase_repository, targets)
624
625 def pull_components(targets):
626         for_each_repository(pull_repository, targets)
627
628 def rebase_repository(repo):
629         print 'Rebasing', repo.path
630         git.remote_update(repo.path)
631         git.rebase(repo.path)
632
633 def pull_repository(repo):
634         print 'Pulling', repo.path
635         git.pull(repo.path)
636
637 def source_dist_components(targets):
638         if not targets:
639                 targets = config.components.keys()
640
641         for name in targets:
642                 c = config.components[name]
643                 generate_component_changes(c, 'dist')
644                 package_component_sources(c, 'dist')
645
646 def generate_component_changes(c, location):
647         print 'Generating change log for', c.repo.path
648
649         path = os.path.join(location, c.name) + '.changes'
650
651         pathdir = os.path.dirname(path)
652         if not os.path.exists(pathdir):
653                 os.makedirs(pathdir)
654
655         fd = os.open(path, os.O_WRONLY | os.O_CREAT, 0644)
656         git.log(c.repo.path, [c.get_active_tag()], fd=fd)
657         os.close(fd)
658
659 def package_component_sources(c, location):
660         print 'Archiving', c.repo.path
661
662         rev = git.describe(c.repo.path)
663         if rev:
664                 rev = '_' + rev
665         else:
666                 rev = ''
667
668         path = os.path.join(location, c.name) + rev + '.tar.bz2'
669         if os.path.exists(path):
670                 os.remove(path)
671
672         pathdir = os.path.dirname(path)
673         if not os.path.exists(pathdir):
674                 os.makedirs(pathdir)
675
676         git.archive(c.repo.path, path,
677                     prefix=os.path.basename(c.name) + '/',
678                     branch=c.get_active_tag())
679
680 def build_rootfs(clean=True, rootfs_only=False, jffs2_only=False, devrootfs_only=False):
681         parse_config('rootfs')
682
683         f = os.popen('sb-conf cu')
684         target = f.read()
685         f.close()
686
687         rootfs = RootFS(target.strip())
688         rootfs.include_paths(config.include_paths)
689         rootfs.include_files(config.include_files)
690         rootfs.filter_paths(config.exclude_paths)
691         rootfs.filter_files(config.exclude_files)
692         rootfs.filter_expressions(config.exclude_expressions)
693         rootfs.add_paths(config.created_paths)
694         rootfs.set_devices(config.devices)
695         rootfs.set_change_owner(config.change_owner)
696         rootfs.set_erase_size(config.boards[config.board].flash_erase_size)
697         rootfs.set_pad_size(config.boards[config.board].flash_pad_size)
698
699         if rootfs_only:
700                 rootfs.generate(clean, build_target="rootfs")
701         elif jffs2_only:
702                 rootfs.generate(clean, build_target="jffs2")
703         elif devrootfs_only:
704                 rootfs.generate(clean, build_target="devrootfs")
705         else:
706                 rootfs.generate(clean, build_target="all")