rebase command
[matrix.git] / matrix / matrix.py
1 # Copyright (C) 2006-2008 Movial Oy
2 # Authors: Timo Savola <tsavola@movial.fi>
3 #          Toni Timonen
4 #          Kalle Vahlman <kalle.vahlman@movial.fi>
5 #          Tuomas Kulve <tuomas.kulve@movial.fi>
6
7 import glob
8 import os
9 import re
10 import sys
11 import tarfile
12 from sets import Set as set
13
14 import config
15 import git
16 from rootfs import RootFS
17
18 Error = RuntimeError
19
20 def main():
21         parse_config('config.local')
22
23         command, params, targets = parse_args(sys.argv)
24
25         parse_config('config')
26         parse_config('boards')
27         parse_config('components')
28
29         if 'global-cache' in config.flags:
30                 config.cache_dir = config.global_cache_dir
31
32         if not os.path.exists(config.cache_dir):
33                 os.makedirs(config.cache_dir)
34
35         all_targets = update_components()
36
37         for name in targets:
38                 if name not in config.components:
39                         raise Error('Component "%s" does not exist' % name)
40
41         if not targets:
42                 targets = all_targets
43
44         if command == 'install':
45                 build_components(targets, **params)
46         elif command == 'clone':
47                 clone_components(targets)
48         elif command == 'clean':
49                 clean_components(targets)
50         elif command == 'rebase':
51                 rebase_components(targets)
52         elif command == 'pull':
53                 pull_components(targets)
54         elif command == 'source-dist':
55                 source_dist_components(targets)
56         elif command == 'rootfs':
57                 build_rootfs(**params)
58         else:
59                 raise Error('Invalid command: ' + command)
60
61 def help(file, args):
62         print >>file, '''
63 Copyright (C) 2006-2008 Movial Oy
64
65 Usage: %(progname)s [<options>] <command> [<params>] [<components>]
66
67 If no components are specified, all of them will be targeted.  All component
68 metadata will be downloaded regardless of the specified command and components.
69
70 Options:
71         -v              Verbose output.
72         -d              Debug output.
73         -r URL          Specify the root for component git repos to clone from.
74                         If this option is not given, the roots specified in the
75                         components file will be used.  This option may be
76                         specified multiple times.
77         -f              Build components with dirty files.
78         -h, --help      Print this help text.
79
80 Commands and parameters:
81         clone           Download the components' git repositories.
82         install         Download, build and install the components.
83                 -j N            Execute N build jobs in parallel.
84                 -mj N           Run component builds with make -j N.
85         clean           Remove all non-tracked files from the component git
86                         repository directories.
87         rebase          Update repositories from server by rebasing.
88         pull            Update repositories from server by merging.
89         source-dist     Download and package the component sources.
90         rootfs          Build a rootfs from the current target installation.
91                 -n              Do not clean env.faked and stripped directory.
92                 -r              Only generate a stripped rootfs.
93                 -j              Only generate a jffs2 image
94                 -d              Only generate a rootstrap (development rootfs).
95 ''' % {'progname': args[0]}
96
97 def parse_args(args):
98         def parse_jobs(arg):
99                 jobs = int(arg)
100                 if jobs <= 0:
101                         raise Error('Please specify a valid number of jobs')
102                 return jobs
103
104         command = None
105         params = {}
106         targets = []
107
108         i = 1
109         while i < len(args):
110                 if args[i].startswith('-'):
111                         if not command:
112                                 if args[i] == '-v':
113                                         config.verbose = True
114                                 elif args[i] == '-d':
115                                         config.debug = True
116                                 elif args[i] == '-r':
117                                         i += 1
118                                         config.roots.append(args[i])
119                                 elif args[i] == '-f':
120                                         config.force = True
121                                 elif args[i] in ('-h', '--help'):
122                                         help(sys.stdout, args)
123                                         sys.exit(0)
124                                 else:
125                                         raise Error('Bad option: ' + args[i])
126                         elif command == 'install':
127                                 if args[i] == '-j':
128                                         i += 1
129                                         params['build_jobs'] = parse_jobs(args[i])
130                                 elif args[i] == '-mj':
131                                         i += 1
132                                         params['make_jobs'] = parse_jobs(args[i])
133                                 else:
134                                         raise Error('Bad parameter: ' + args[i])
135                         elif command == 'rootfs':
136                                 if args[i] == '-n':
137                                         i += 1
138                                         params['clean'] = False
139                                 elif args[i] == '-r':
140                                         i += 1
141                                         params['rootfs_only'] = True
142                                 elif args[i] == '-j':
143                                         i += 1
144                                         params['jffs2_only'] = True
145                                 elif args[i] == '-d':
146                                         i += 1
147                                         params['devrootfs_only'] = True
148                         else:
149                                 raise Error('Command takes no parameters')
150                 elif not command:
151                         command = args[i]
152                 else:
153                         targets.append(args[i])
154                 i += 1
155
156         if not command:
157                 help(sys.stderr, args)
158                 sys.exit(1)
159
160         return command, params, targets
161
162 def parse_config(path):
163         if os.path.isabs(path):
164                 path = os.path.join(config.top_dir, path)
165
166         if os.path.exists(path):
167                 print 'Reading', path
168                 execfile(path, config.__dict__, config.__dict__)
169
170 def update_components():
171         targets = []
172         packages = {}
173
174         for c in config.components.itervalues():
175                 update_component_url(c)
176                 clone_metadata(c)
177                 update_component_packages(c, targets, packages)
178                 c.active_depends = []
179
180         update_component_depends(packages)
181
182         return targets
183
184 def update_component_url(c):
185         update_repository_url(c.repo)
186         update_repository_url(c.meta)
187
188 def update_repository_url(repo):
189         if git.contains_database(repo.path):
190                 url = git.getvar(repo.path, 'remote.origin.url')
191                 if url:
192                         repo.active_url = url
193                         if config.debug:
194                                 print 'Using', repo.active_url
195                         return
196
197         for root in config.roots:
198                 for suffix in ('.git', ''):
199                         url = '%s/%s%s' % (root, repo.name, suffix)
200
201                         if config.debug:
202                                 print 'Trying', url
203
204                         if not git.peek_remote(url, quiet=True):
205                                 continue
206
207                         repo.active_url = url
208                         if config.debug:
209                                 print 'Found', repo.active_url
210                         return
211
212         raise Error('Failed to locate repository: ' + repo.name)
213
214 def update_component_depends(packages):
215         for pkg in packages.itervalues():
216                 for spec in pkg.depends.split():
217                         depname = Depend(spec).name
218                         deppkg = packages.get(depname)
219
220                         if not deppkg:
221                                 print >>sys.stderr, 'Package', pkg.name, \
222                                         'depends on non-existent package', depname
223                                 continue
224
225                         if deppkg.component == pkg.component:
226                                 continue
227
228                         pkg.component.active_depends.append(deppkg.component)
229
230         fail = False
231         for pkg in packages.itervalues():
232                 for spec in pkg.depends.split():
233                         if not Depend(spec).check(packages):
234                                 fail = True
235                                 print >>sys.stderr, 'Dependency', spec, \
236                                         'failed for', pkg.name
237
238                 for spec in pkg.conflicts.split():
239                         if Depend(spec).check(packages):
240                                 fail = True
241                                 print >>sys.stderr, pkg.name, \
242                                         'conflicts with', spec
243
244         if fail:
245                 raise Error('Invalid component tree')
246
247 class Depend(object):
248         regex = re.compile(r'([@]?)([^\s:]+)[:]?([<>=]*)([^\s:]*)[:]?(.*)')
249
250         def __init__(self, spec):
251                 match = self.regex.match(spec)
252                 if not match:
253                         raise Error('Bad dependency specification: ' + spec)
254
255                 self.build, self.name, self.tag_op, self.tag, flags \
256                         = match.groups()
257                 self.flags = flags.split()
258
259         def check(self, packages):
260                 # TODO: check version and flags
261                 return self.name in packages
262
263 def execute(args):
264         if config.debug:
265                 print 'Executing:', ' '.join(args)
266
267         if os.spawnvp(os.P_WAIT, args[0], args) != 0:
268                 raise Error('Failed: ' + ' '.join(args))
269
270 def remove_tree(path):
271         execute(['rm', '-rf', path])
272
273 def clone_components(targets):
274         if not targets:
275                 targets = config.components.keys()
276
277         for name in targets:
278                 c = config.components[name]
279                 clone_component(c)
280
281 def clone_component(c, overwrite=False):
282         have_repo = git.contains_database(c.repo.path)
283         have_meta = git.contains_database(c.meta.path)
284
285         if not overwrite and have_repo and have_meta:
286                 return
287
288         if overwrite and os.path.exists(c.repo.path):
289                 print 'Removing', c.repo.path
290
291                 remove_tree(c.repo.path)
292                 have_repo = False
293                 have_meta = False
294
295         if not have_repo:
296                 clone_repository(c.repo)
297                 git.exclude(c.repo.path, 'meta')
298
299         if not have_meta:
300                 clone_repository(c.meta)
301
302 def clone_metadata(c):
303         if not git.contains_database(c.meta.path):
304                 clone_repository(c.meta)
305
306 def clone_repository(repo):
307         print 'Cloning', repo.path
308
309         if os.path.exists(repo.path):
310                 tmp = os.path.join(repo.path, 'tmp')
311                 git.clone(tmp, repo.active_url, checkout=False)
312                 try:
313                         tmpdb = os.path.join(tmp, '.git')
314                         repodb = os.path.join(repo.path, '.git')
315                         if config.debug:
316                                 print 'Renaming "%s" as "%s"' % (tmpdb, repodb)
317                         os.rename(tmpdb, repodb)
318                 finally:
319                         os.rmdir(tmp)
320                 git.checkout(repo.path)
321         else:
322                 git.clone(repo.path, repo.active_url, checkout=True)
323
324 def update_component_packages(c, targets, packages):
325         c.active_packages = {}
326
327         for path in glob.glob(os.path.join(c.meta.path, '*.package')):
328                 name = os.path.basename(path)[:-8]
329
330                 pkg = parse_package(name, c, path)
331                 if not pkg:
332                         continue
333
334                 c.active_packages[name] = pkg
335                 packages[name] = pkg
336
337                 if config.debug:
338                         print 'Component', c.name, 'provides', name
339
340         if c.active_packages:
341                 targets.append(c.name)
342         elif config.debug:
343                 print 'Component', c.name, 'does not provide any packages'
344
345 class Package(object):
346         def __init__(self, name, component):
347                 self.name = name
348                 self.component = component
349
350                 self.depends = []
351                 self.conflicts = []
352                 self.architectures = None
353
354 def parse_package(name, component, path):
355         pkg = Package(name, component)
356         execfile(path, pkg.__dict__, pkg.__dict__)
357
358         if pkg.architectures:
359                 arch = config.boards[config.board].arch
360                 if arch not in pkg.architectures:
361                         return None
362
363         return pkg
364
365 def build_components(targets, build_jobs=1, make_jobs=1):
366         selected = set([config.components[i] for i in targets])
367
368         depends = None
369         while depends is None or depends:
370                 depends = set()
371
372                 for c in selected:
373                         for dep in c.active_depends:
374                                 if dep not in selected:
375                                         depends.add(dep)
376
377                 selected |= depends
378
379         components = list(selected)
380         components.sort()
381
382         if config.debug:
383                 print 'Building components:'
384                 for c in components:
385                         print '\t' + c.name
386
387         jobs = {}
388
389         while True:
390                 found = False
391                 for c in components:
392                         if len(jobs) >= build_jobs:
393                                 break
394
395                         if component_buildable(c):
396                                 found = True
397
398                                 if not component_cached(c):
399                                         print 'Building', c.name
400                                         start_job(c, jobs, make_jobs)
401                                 elif config.debug:
402                                         print 'Component', c.name, \
403                                               'found from cache'
404                                         c.active_state = 'built'
405
406                 if not found and not jobs:
407                         for c in components:
408                                 if c.active_state != 'built':
409                                         raise Error('Internal error')
410                         break
411
412                 wait_for_job(jobs)
413
414 def component_buildable(c):
415         if c.active_state:
416                 if config.debug:
417                         print 'Component', c.name, 'is in state:', c.active_state
418                 return False
419
420         for dep in c.active_depends:
421                 if config.components[dep.name].active_state != 'built':
422                         if config.debug:
423                                 print 'Component', c.name, \
424                                       'depends on unbuilt', dep.name
425                         return False
426
427         return True
428
429 def component_cached(c):
430         if c.from_platform:
431                 return True
432
433         for repo in (c.repo, c.meta):
434                 if git.ls_files(repo.path, ['-m', '-d']):
435                         if config.force:
436                                 return False
437                         else:
438                                 raise Error('Dirty files in ' + repo.path)
439
440         path = os.path.join(config.cache_dir, c.name)
441         if not os.path.exists(path):
442                 return False
443
444         file = open(path, 'r')
445         line = file.readline()
446         file.close()
447
448         match = re.match(r'([^\s]+)[\s]?([^\s]*)', line)
449         if not match:
450                 return False
451
452         hash, flagstring = match.groups()
453
454         if hash != get_component_hash(c):
455                 print 'Component has been changed:', c.name
456                 return False
457
458         if flagstring:
459                 flags = flagstring.split(',')
460         else:
461                 flags = []
462
463         if set(flags) != set(c.flags):
464                 print 'Component flags have been changed:', c.name
465                 return False
466
467         return True
468
469 def update_cache(c):
470         path = os.path.join(config.cache_dir, c.name)
471
472         dir = os.path.dirname(path)
473         if not os.path.exists(dir):
474                 os.makedirs(dir)
475
476         file = open(path, 'w')
477         done = False
478         try:
479                 print >>file, get_component_hash(c), ','.join(c.flags)
480                 done = True
481         finally:
482                 file.close()
483                 if not done:
484                         os.remove(path)
485
486 def get_component_hash(c):
487         return '%s+%s' % (get_repository_hash(c.repo), get_repository_hash(c.meta))
488
489 def get_repository_hash(repo):
490         if repo.active_hash is None:
491                 repo.active_hash = git.rev_parse(repo.path, 'HEAD')
492         return repo.active_hash
493
494 def start_job(c, jobs, make_jobs):
495         board = config.boards[config.board]
496
497         makefile = os.path.join(config.script_dir, 'matrix.mak')
498
499         workdir = os.path.join(config.top_dir, 'src', c.name)
500         args = ['make', '--no-print-directory', '-f', makefile, '-C', workdir,
501                 '-j', str(make_jobs), 'build_matrix_component',
502                 'MATRIX_TOPDIR='    + os.path.abspath(config.top_dir),
503                 'MATRIX_SCRIPTDIR=' + os.path.abspath(config.script_dir),
504                 'MATRIX_COMPONENT=' + c.name,
505                 'MATRIX_ARCH='      + board.arch,
506                 'MATRIX_GCC_MARCH=' + board.gcc_march,
507                 'MATRIX_GCC_MCPU='  + board.gcc_mcpu,
508                 'MATRIX_GCC_MFPU='  + board.gcc_mfpu,
509                 'MATRIX_GCC_OPTIONS=' + board.gcc_options,
510                 'MATRIX_GNU_HOST='  + board.gnu_host,
511                 'MATRIX_LIBC='      + config.libc]
512
513         for flag in config.flags:
514                 args.append(flag + '=1')
515
516         if config.verbose:
517                 args.append('MATRIX_VERBOSE=1')
518
519         if config.debug:
520                 print 'Executing:', ' '.join(args)
521
522         pid = os.spawnvp(os.P_NOWAIT, args[0], args)
523
524         jobs[pid] = c
525         c.active_state = 'running'
526
527 def wait_for_job(jobs):
528         if not jobs:
529                 return
530
531         pid, status = os.wait()
532
533         c = jobs.get(pid)
534         if not c:
535                 return
536
537         del jobs[pid]
538
539         if status == 0:
540                 c.active_state = 'built'
541                 update_cache(c)
542                 print c.name, 'completed'
543         else:
544                 c.active_state = 'error'
545                 print >>sys.stderr, c.name, 'failed with status:', status
546
547                 wait_for_job(jobs)
548                 raise Error('Build failed')
549
550 def clean_components(targets):
551         if not targets:
552                 targets = config.components.keys()
553
554         for name in targets:
555                 c = config.components[name]
556                 print 'Cleaning', c.repo.path
557
558                 files = git.ls_files(c.repo.path, ['-o'])
559                 paths = [os.path.join(c.repo.path, i) for i in files]
560                 paths.sort()
561                 paths.reverse()
562
563                 cache = os.path.join(config.cache_dir, name)
564                 if os.path.exists(cache):
565                         paths.append(cache)
566
567                 for path in paths:
568                         if git.contains_database(path):
569                                 continue
570
571                         if config.debug:
572                                 print 'Removing', path
573
574                         if os.path.islink(path) or not os.path.isdir(path):
575                                 os.remove(path)
576                         else:
577                                 remove_tree(path)
578
579                 for repo in (c.repo, c.meta):
580                         files = git.ls_files(repo.path, ['-m', '-d'])
581                         if files:
582                                 print len(files), 'dirty files left in', repo.path
583
584 def for_each_repository(func, targets=None):
585         if not targets:
586                 targets = config.components.keys()
587
588         for name in targets:
589                 c = config.components[name]
590                 if git.contains_database(c.repo.path):
591                         func(c.repo)
592                 func(c.meta)
593
594 def rebase_components(targets):
595         for_each_repository(rebase_repository, targets)
596
597 def pull_components(targets):
598         for_each_repository(pull_repository, targets)
599
600 def rebase_repository(repo):
601         print 'Rebasing', repo.path
602         git.remote_update(repo.path)
603         git.rebase(repo.path)
604
605 def pull_repository(repo):
606         print 'Pulling', repo.path
607         git.pull(repo.path)
608
609 def source_dist_components(targets):
610         if not targets:
611                 targets = config.components.keys()
612
613         for name in targets:
614                 c = config.components[name]
615                 generate_component_changes(c, 'dist')
616                 package_component_sources(c, 'dist')
617
618 def generate_component_changes(c, location):
619         print 'Generating change log for', c.repo.path
620
621         path = os.path.join(location, c.name) + '.changes'
622
623         pathdir = os.path.dirname(path)
624         if not os.path.exists(pathdir):
625                 os.makedirs(pathdir)
626
627         fd = os.open(path, os.O_WRONLY | os.O_CREAT, 0644)
628         git.log(c.repo.path, [c.get_active_tag()], fd=fd)
629         os.close(fd)
630
631 def package_component_sources(c, location):
632         print 'Archiving', c.repo.path
633
634         rev = git.describe(c.repo.path)
635         if rev:
636                 rev = '_' + rev
637         else:
638                 rev = ''
639
640         path = os.path.join(location, c.name) + rev + '.tar.bz2'
641         if os.path.exists(path):
642                 os.remove(path)
643
644         pathdir = os.path.dirname(path)
645         if not os.path.exists(pathdir):
646                 os.makedirs(pathdir)
647
648         git.archive(c.repo.path, path,
649                     prefix=os.path.basename(c.name) + '/',
650                     branch=c.get_active_tag())
651
652 def build_rootfs(clean=True, rootfs_only=False, jffs2_only=False, devrootfs_only=False):
653         parse_config('rootfs')
654
655         f = os.popen('sb-conf cu')
656         target = f.read()
657         f.close()
658
659         rootfs = RootFS(target.strip())
660         rootfs.include_paths(config.include_paths)
661         rootfs.include_files(config.include_files)
662         rootfs.filter_paths(config.exclude_paths)
663         rootfs.filter_files(config.exclude_files)
664         rootfs.filter_expressions(config.exclude_expressions)
665         rootfs.add_paths(config.created_paths)
666         rootfs.set_devices(config.devices)
667         rootfs.set_change_owner(config.change_owner)
668         rootfs.set_erase_size(config.boards[config.board].flash_erase_size)
669         rootfs.set_pad_size(config.boards[config.board].flash_pad_size)
670
671         if rootfs_only:
672                 rootfs.generate(clean, build_target="rootfs")
673         elif jffs2_only:
674                 rootfs.generate(clean, build_target="jffs2")
675         elif devrootfs_only:
676                 rootfs.generate(clean, build_target="devrootfs")
677         else:
678                 rootfs.generate(clean, build_target="all")