more consistent build and pull messages
[matrix.git] / matrix / matrix.py
1 # Copyright (C) 2006-2008 Movial Oy
2 # Authors: Timo Savola <tsavola@movial.fi>
3 #          Toni Timonen
4 #          Kalle Vahlman <kalle.vahlman@movial.fi>
5 #          Tuomas Kulve <tuomas.kulve@movial.fi>
6
7 import glob
8 import os
9 import re
10 import sys
11 import tarfile
12 from sets import Set as set
13
14 import config
15 import git
16 from rootfs import RootFS
17
18 Error = RuntimeError
19
20 def main():
21         parse_config('config.local')
22
23         command, params, targets = parse_args(sys.argv)
24
25         parse_config('config')
26         parse_config('boards')
27         parse_config('components')
28
29         if 'global-cache' in config.flags:
30                 config.cache_dir = config.global_cache_dir
31
32         if not os.path.exists(config.cache_dir):
33                 os.makedirs(config.cache_dir)
34
35         all_targets = update_components()
36
37         for name in targets:
38                 if name not in config.components:
39                         raise Error('Component "%s" does not exist' % name)
40
41         if not targets:
42                 targets = all_targets
43
44         if command == 'install':
45                 build_components(targets, **params)
46         elif command == 'clone':
47                 clone_components(targets)
48         elif command == 'clean':
49                 clean_components(targets)
50         elif command == 'pull':
51                 pull_components(targets)
52         elif command == 'source-dist':
53                 source_dist_components(targets)
54         elif command == 'rootfs':
55                 build_rootfs(**params)
56         else:
57                 raise Error('Invalid command: ' + command)
58
59 def help(file, args):
60         print >>file, '''
61 Copyright (C) 2006-2008 Movial Oy
62
63 Usage: %(progname)s [<options>] <command> [<params>] [<components>]
64
65 If no components are specified, all of them will be targeted.  All component
66 metadata will be downloaded regardless of the specified command and components.
67
68 Options:
69         -v              Verbose output.
70         -d              Debug output.
71         -r URL          Specify the root for component git repos to clone from.
72                         If this option is not given, the roots specified in the
73                         components file will be used.  This option may be
74                         specified multiple times.
75         -f              Build components with dirty files.
76         -h, --help      Print this help text.
77
78 Commands and parameters:
79         clone           Download the components' git repositories.
80         install         Download, build and install the components.
81                 -j N            Execute N build jobs in parallel.
82                 -mj N           Run component builds with make -j N.
83         clean           Remove all non-tracked files from the component git
84                         repository directories.
85         pull            Run git pull for the components.
86         source-dist     Download and package the component sources.
87         rootfs          Build a rootfs from the current target installation.
88                 -n              Do not clean env.faked and stripped directory.
89                 -r              Only generate a stripped rootfs.
90                 -j              Only generate a jffs2 image
91                 -d              Only generate a rootstrap (development rootfs).
92 ''' % {'progname': args[0]}
93
94 def parse_args(args):
95         def parse_jobs(arg):
96                 jobs = int(arg)
97                 if jobs <= 0:
98                         raise Error('Please specify a valid number of jobs')
99                 return jobs
100
101         command = None
102         params = {}
103         targets = []
104
105         i = 1
106         while i < len(args):
107                 if args[i].startswith('-'):
108                         if not command:
109                                 if args[i] == '-v':
110                                         config.verbose = True
111                                 elif args[i] == '-d':
112                                         config.debug = True
113                                 elif args[i] == '-r':
114                                         i += 1
115                                         config.roots.append(args[i])
116                                 elif args[i] == '-f':
117                                         config.force = True
118                                 elif args[i] in ('-h', '--help'):
119                                         help(sys.stdout, args)
120                                         sys.exit(0)
121                                 else:
122                                         raise Error('Bad option: ' + args[i])
123                         elif command == 'install':
124                                 if args[i] == '-j':
125                                         i += 1
126                                         params['build_jobs'] = parse_jobs(args[i])
127                                 elif args[i] == '-mj':
128                                         i += 1
129                                         params['make_jobs'] = parse_jobs(args[i])
130                                 else:
131                                         raise Error('Bad parameter: ' + args[i])
132                         elif command == 'rootfs':
133                                 if args[i] == '-n':
134                                         i += 1
135                                         params['clean'] = False
136                                 elif args[i] == '-r':
137                                         i += 1
138                                         params['rootfs_only'] = True
139                                 elif args[i] == '-j':
140                                         i += 1
141                                         params['jffs2_only'] = True
142                                 elif args[i] == '-d':
143                                         i += 1
144                                         params['devrootfs_only'] = True
145                         else:
146                                 raise Error('Command takes no parameters')
147                 elif not command:
148                         command = args[i]
149                 else:
150                         targets.append(args[i])
151                 i += 1
152
153         if not command:
154                 help(sys.stderr, args)
155                 sys.exit(1)
156
157         return command, params, targets
158
159 def parse_config(path):
160         if os.path.isabs(path):
161                 path = os.path.join(config.top_dir, path)
162
163         if os.path.exists(path):
164                 print 'Reading', path
165                 execfile(path, config.__dict__, config.__dict__)
166
167 def update_components():
168         targets = []
169         packages = {}
170
171         for c in config.components.itervalues():
172                 update_component_url(c)
173                 clone_metadata(c)
174                 update_component_packages(c, targets, packages)
175                 c.active_depends = []
176
177         update_component_depends(packages)
178
179         return targets
180
181 def update_component_url(c):
182         update_repository_url(c.repo)
183         update_repository_url(c.meta)
184
185 def update_repository_url(repo):
186         if git.contains_database(repo.path):
187                 url = git.getvar(repo.path, 'remote.origin.url')
188                 if url:
189                         repo.active_url = url
190                         if config.debug:
191                                 print 'Using', repo.active_url
192                         return
193
194         for root in config.roots:
195                 for suffix in ('.git', ''):
196                         url = '%s/%s%s' % (root, repo.name, suffix)
197
198                         if config.debug:
199                                 print 'Trying', url
200
201                         if not git.peek_remote(url, quiet=True):
202                                 continue
203
204                         repo.active_url = url
205                         if config.debug:
206                                 print 'Found', repo.active_url
207                         return
208
209         raise Error('Failed to locate repository: ' + repo.name)
210
211 def update_component_depends(packages):
212         for pkg in packages.itervalues():
213                 for spec in pkg.depends.split():
214                         depname = Depend(spec).name
215                         deppkg = packages.get(depname)
216
217                         if not deppkg:
218                                 print >>sys.stderr, 'Package', pkg.name, \
219                                         'depends on non-existent package', depname
220                                 continue
221
222                         if deppkg.component == pkg.component:
223                                 continue
224
225                         pkg.component.active_depends.append(deppkg.component)
226
227         fail = False
228         for pkg in packages.itervalues():
229                 for spec in pkg.depends.split():
230                         if not Depend(spec).check(packages):
231                                 fail = True
232                                 print >>sys.stderr, 'Dependency', spec, \
233                                         'failed for', pkg.name
234
235                 for spec in pkg.conflicts.split():
236                         if Depend(spec).check(packages):
237                                 fail = True
238                                 print >>sys.stderr, pkg.name, \
239                                         'conflicts with', spec
240
241         if fail:
242                 raise Error('Invalid component tree')
243
244 class Depend(object):
245         regex = re.compile(r'([@]?)([^\s:]+)[:]?([<>=]*)([^\s:]*)[:]?(.*)')
246
247         def __init__(self, spec):
248                 match = self.regex.match(spec)
249                 if not match:
250                         raise Error('Bad dependency specification: ' + spec)
251
252                 self.build, self.name, self.tag_op, self.tag, flags \
253                         = match.groups()
254                 self.flags = flags.split()
255
256         def check(self, packages):
257                 # TODO: check version and flags
258                 return self.name in packages
259
260 def execute(args):
261         if config.debug:
262                 print 'Executing:', ' '.join(args)
263
264         if os.spawnvp(os.P_WAIT, args[0], args) != 0:
265                 raise Error('Failed: ' + ' '.join(args))
266
267 def remove_tree(path):
268         execute(['rm', '-rf', path])
269
270 def clone_components(targets):
271         if not targets:
272                 targets = config.components.keys()
273
274         for name in targets:
275                 c = config.components[name]
276                 clone_component(c)
277
278 def clone_component(c, overwrite=False):
279         have_repo = git.contains_database(c.repo.path)
280         have_meta = git.contains_database(c.meta.path)
281
282         if not overwrite and have_repo and have_meta:
283                 return
284
285         if overwrite and os.path.exists(c.repo.path):
286                 print 'Removing', c.repo.path
287
288                 remove_tree(c.repo.path)
289                 have_repo = False
290                 have_meta = False
291
292         if not have_repo:
293                 clone_repository(c.repo)
294                 git.exclude(c.repo.path, 'meta')
295
296         if not have_meta:
297                 clone_repository(c.meta)
298
299 def clone_metadata(c):
300         if not git.contains_database(c.meta.path):
301                 clone_repository(c.meta)
302
303 def clone_repository(repo):
304         print 'Cloning', repo.path
305
306         if os.path.exists(repo.path):
307                 tmp = os.path.join(repo.path, 'tmp')
308                 git.clone(tmp, repo.active_url, checkout=False)
309                 try:
310                         tmpdb = os.path.join(tmp, '.git')
311                         repodb = os.path.join(repo.path, '.git')
312                         if config.debug:
313                                 print 'Renaming "%s" as "%s"' % (tmpdb, repodb)
314                         os.rename(tmpdb, repodb)
315                 finally:
316                         os.rmdir(tmp)
317                 git.checkout(repo.path)
318         else:
319                 git.clone(repo.path, repo.active_url, checkout=True)
320
321 def update_component_packages(c, targets, packages):
322         c.active_packages = {}
323
324         for path in glob.glob(os.path.join(c.meta.path, '*.package')):
325                 name = os.path.basename(path)[:-8]
326
327                 pkg = parse_package(name, c, path)
328                 if not pkg:
329                         continue
330
331                 c.active_packages[name] = pkg
332                 packages[name] = pkg
333
334                 if config.debug:
335                         print 'Component', c.name, 'provides', name
336
337         if c.active_packages:
338                 targets.append(c.name)
339         elif config.debug:
340                 print 'Component', c.name, 'does not provide any packages'
341
342 class Package(object):
343         def __init__(self, name, component):
344                 self.name = name
345                 self.component = component
346
347                 self.depends = []
348                 self.conflicts = []
349                 self.architectures = None
350
351 def parse_package(name, component, path):
352         pkg = Package(name, component)
353         execfile(path, pkg.__dict__, pkg.__dict__)
354
355         if pkg.architectures:
356                 arch = config.boards[config.board].arch
357                 if arch not in pkg.architectures:
358                         return None
359
360         return pkg
361
362 def build_components(targets, build_jobs=1, make_jobs=1):
363         selected = set([config.components[i] for i in targets])
364
365         depends = None
366         while depends is None or depends:
367                 depends = set()
368
369                 for c in selected:
370                         for dep in c.active_depends:
371                                 if dep not in selected:
372                                         depends.add(dep)
373
374                 selected |= depends
375
376         components = list(selected)
377         components.sort()
378
379         if config.debug:
380                 print 'Building components:'
381                 for c in components:
382                         print '\t' + c.name
383
384         jobs = {}
385
386         while True:
387                 found = False
388                 for c in components:
389                         if len(jobs) >= build_jobs:
390                                 break
391
392                         if component_buildable(c):
393                                 found = True
394
395                                 if not component_cached(c):
396                                         print 'Building', c.name
397                                         start_job(c, jobs, make_jobs)
398                                 elif config.debug:
399                                         print 'Component', c.name, \
400                                               'found from cache'
401                                         c.active_state = 'built'
402
403                 if not found and not jobs:
404                         for c in components:
405                                 if c.active_state != 'built':
406                                         raise Error('Internal error')
407                         break
408
409                 wait_for_job(jobs)
410
411 def component_buildable(c):
412         if c.active_state:
413                 if config.debug:
414                         print 'Component', c.name, 'is in state:', c.active_state
415                 return False
416
417         for dep in c.active_depends:
418                 if config.components[dep.name].active_state != 'built':
419                         if config.debug:
420                                 print 'Component', c.name, \
421                                       'depends on unbuilt', dep.name
422                         return False
423
424         return True
425
426 def component_cached(c):
427         if c.from_platform:
428                 return True
429
430         for repo in (c.repo, c.meta):
431                 if git.ls_files(repo.path, ['-m', '-d']):
432                         if config.force:
433                                 return False
434                         else:
435                                 raise Error('Dirty files in ' + repo.path)
436
437         path = os.path.join(config.cache_dir, c.name)
438         if not os.path.exists(path):
439                 return False
440
441         file = open(path, 'r')
442         line = file.readline()
443         file.close()
444
445         match = re.match(r'([^\s]+)[\s]?([^\s]*)', line)
446         if not match:
447                 return False
448
449         hash, flagstring = match.groups()
450
451         if hash != get_component_hash(c):
452                 print 'Component has been changed:', c.name
453                 return False
454
455         if flagstring:
456                 flags = flagstring.split(',')
457         else:
458                 flags = []
459
460         if set(flags) != set(c.flags):
461                 print 'Component flags have been changed:', c.name
462                 return False
463
464         return True
465
466 def update_cache(c):
467         path = os.path.join(config.cache_dir, c.name)
468
469         dir = os.path.dirname(path)
470         if not os.path.exists(dir):
471                 os.makedirs(dir)
472
473         file = open(path, 'w')
474         done = False
475         try:
476                 print >>file, get_component_hash(c), ','.join(c.flags)
477                 done = True
478         finally:
479                 file.close()
480                 if not done:
481                         os.remove(path)
482
483 def get_component_hash(c):
484         return '%s+%s' % (get_repository_hash(c.repo), get_repository_hash(c.meta))
485
486 def get_repository_hash(repo):
487         if repo.active_hash is None:
488                 repo.active_hash = git.rev_parse(repo.path, 'HEAD')
489         return repo.active_hash
490
491 def start_job(c, jobs, make_jobs):
492         board = config.boards[config.board]
493
494         makefile = os.path.join(config.script_dir, 'matrix.mak')
495
496         workdir = os.path.join(config.top_dir, 'src', c.name)
497         args = ['make', '--no-print-directory', '-f', makefile, '-C', workdir,
498                 '-j', str(make_jobs), 'build_matrix_component',
499                 'MATRIX_TOPDIR='    + os.path.abspath(config.top_dir),
500                 'MATRIX_SCRIPTDIR=' + os.path.abspath(config.script_dir),
501                 'MATRIX_COMPONENT=' + c.name,
502                 'MATRIX_ARCH='      + board.arch,
503                 'MATRIX_GCC_MARCH=' + board.gcc_march,
504                 'MATRIX_GCC_MCPU='  + board.gcc_mcpu,
505                 'MATRIX_GCC_MFPU='  + board.gcc_mfpu,
506                 'MATRIX_GCC_OPTIONS=' + board.gcc_options,
507                 'MATRIX_GNU_HOST='  + board.gnu_host,
508                 'MATRIX_LIBC='      + config.libc]
509
510         for flag in config.flags:
511                 args.append(flag + '=1')
512
513         if config.verbose:
514                 args.append('MATRIX_VERBOSE=1')
515
516         if config.debug:
517                 print 'Executing:', ' '.join(args)
518
519         pid = os.spawnvp(os.P_NOWAIT, args[0], args)
520
521         jobs[pid] = c
522         c.active_state = 'running'
523
524 def wait_for_job(jobs):
525         if not jobs:
526                 return
527
528         pid, status = os.wait()
529
530         c = jobs.get(pid)
531         if not c:
532                 return
533
534         del jobs[pid]
535
536         if status == 0:
537                 c.active_state = 'built'
538                 update_cache(c)
539                 print c.name, 'completed'
540         else:
541                 c.active_state = 'error'
542                 print >>sys.stderr, c.name, 'failed with status:', status
543
544                 wait_for_job(jobs)
545                 raise Error('Build failed')
546
547 def clean_components(targets):
548         if not targets:
549                 targets = config.components.keys()
550
551         for name in targets:
552                 c = config.components[name]
553                 print 'Cleaning', c.repo.path
554
555                 files = git.ls_files(c.repo.path, ['-o'])
556                 paths = [os.path.join(c.repo.path, i) for i in files]
557                 paths.sort()
558                 paths.reverse()
559
560                 cache = os.path.join(config.cache_dir, name)
561                 if os.path.exists(cache):
562                         paths.append(cache)
563
564                 for path in paths:
565                         if git.contains_database(path):
566                                 continue
567
568                         if config.debug:
569                                 print 'Removing', path
570
571                         if os.path.islink(path) or not os.path.isdir(path):
572                                 os.remove(path)
573                         else:
574                                 remove_tree(path)
575
576                 for repo in (c.repo, c.meta):
577                         files = git.ls_files(repo.path, ['-m', '-d'])
578                         if files:
579                                 print len(files), 'dirty files left in', repo.path
580
581 def pull_components(targets):
582         if not targets:
583                 targets = config.components.keys()
584
585         for name in targets:
586                 c = config.components[name]
587                 pull_repository(c.repo)
588                 pull_repository(c.meta)
589
590 def pull_repository(repo):
591         print 'Pulling', repo.path
592         git.pull(repo.path)
593
594 def source_dist_components(targets):
595         if not targets:
596                 targets = config.components.keys()
597
598         for name in targets:
599                 c = config.components[name]
600                 generate_component_changes(c, 'dist')
601                 package_component_sources(c, 'dist')
602
603 def generate_component_changes(c, location):
604         print 'Generating change log for', c.repo.path
605
606         path = os.path.join(location, c.name) + '.changes'
607
608         pathdir = os.path.dirname(path)
609         if not os.path.exists(pathdir):
610                 os.makedirs(pathdir)
611
612         fd = os.open(path, os.O_WRONLY | os.O_CREAT, 0644)
613         git.log(c.repo.path, [c.get_active_tag()], fd=fd)
614         os.close(fd)
615
616 def package_component_sources(c, location):
617         print 'Archiving', c.repo.path
618
619         rev = git.describe(c.repo.path)
620         if rev:
621                 rev = '_' + rev
622         else:
623                 rev = ''
624
625         path = os.path.join(location, c.name) + rev + '.tar.bz2'
626         if os.path.exists(path):
627                 os.remove(path)
628
629         pathdir = os.path.dirname(path)
630         if not os.path.exists(pathdir):
631                 os.makedirs(pathdir)
632
633         git.archive(c.repo.path, path,
634                     prefix=os.path.basename(c.name) + '/',
635                     branch=c.get_active_tag())
636
637 def build_rootfs(clean=True, rootfs_only=False, jffs2_only=False, devrootfs_only=False):
638         parse_config('rootfs')
639
640         f = os.popen('sb-conf cu')
641         target = f.read()
642         f.close()
643
644         rootfs = RootFS(target.strip())
645         rootfs.include_paths(config.include_paths)
646         rootfs.include_files(config.include_files)
647         rootfs.filter_paths(config.exclude_paths)
648         rootfs.filter_files(config.exclude_files)
649         rootfs.filter_expressions(config.exclude_expressions)
650         rootfs.add_paths(config.created_paths)
651         rootfs.set_devices(config.devices)
652         rootfs.set_change_owner(config.change_owner)
653         rootfs.set_erase_size(config.boards[config.board].flash_erase_size)
654         rootfs.set_pad_size(config.boards[config.board].flash_pad_size)
655
656         if rootfs_only:
657                 rootfs.generate(clean, build_target="rootfs")
658         elif jffs2_only:
659                 rootfs.generate(clean, build_target="jffs2")
660         elif devrootfs_only:
661                 rootfs.generate(clean, build_target="devrootfs")
662         else:
663                 rootfs.generate(clean, build_target="all")