fix and clean up help message
[matrix.git] / matrix / matrix.py
1 # Copyright (C) 2006-2008 Movial Oy
2 # Authors: Timo Savola <tsavola@movial.fi>
3 #          Toni Timonen
4 #          Kalle Vahlman <kalle.vahlman@movial.fi>
5 #          Tuomas Kulve <tuomas.kulve@movial.fi>
6
7 import glob
8 import os
9 import re
10 import sys
11 import tarfile
12 from sets import Set as set
13
14 import config
15 import git
16 from rootfs import RootFS
17
18 Error = RuntimeError
19
20 def main():
21         parse_config('config.local')
22
23         command, params, targets = parse_args(sys.argv)
24
25         parse_config('config')
26         parse_config('boards')
27         parse_config('components')
28
29         if 'global-cache' in config.flags:
30                 config.cache_dir = config.global_cache_dir
31
32         if not os.path.exists(config.cache_dir):
33                 os.makedirs(config.cache_dir)
34
35         all_targets = update_components()
36
37         for name in targets:
38                 if name not in config.components:
39                         raise Error('Component "%s" does not exist' % name)
40
41         if not targets:
42                 targets = all_targets
43
44         if command == 'install':
45                 build_components(targets, **params)
46         elif command == 'download':
47                 # Actual downloading took place already in update_components()
48                 pass
49         elif command == 'clean':
50                 clean_components(targets)
51         elif command == 'pull':
52                 pull_components(targets)
53         elif command == 'source-dist':
54                 source_dist_components(targets)
55         elif command == 'rootfs':
56                 build_rootfs(**params)
57         else:
58                 raise Error('Invalid command: ' + command)
59
60 def help(file, args):
61         print >>file, '''
62 Copyright (C) 2006-2008 Movial Oy
63
64 Usage: %(progname)s [<options>] <command> [<params>] [<components>]
65
66 If no components are specified, all of them will be targeted.
67
68 Options:
69         -v              Verbose output.
70         -d              Debug output.
71         -r URL          Specify the root for component git repos to clone from.
72                         If this option is not given, the roots specified in the
73                         components file will be used.  This option may be
74                         specified multiple times.
75         -f              Build components with dirty files.
76         -h, --help      Print this help text.
77
78 Commands and parameters:
79         download        Only download the components and validate the tree.
80         install         Download, build and install the components.
81                 -j N            Execute N build jobs in parallel.
82                 -mj N           Run component builds with make -j N.
83         clean           Remove all non-tracked files from the component git
84                         repository directories.
85         pull            Run git pull for the components.
86         source-dist     Download and package the component sources.
87         rootfs          Build a rootfs from the current target installation.
88                 -n              Do not clean env.faked and stripped directory.
89                 -r              Only generate a stripped rootfs.
90                 -j              Only generate a jffs2 image
91                 -d              Only generate a rootstrap (development rootfs).
92 ''' % {'progname': args[0]}
93
94 def parse_args(args):
95         def parse_jobs(arg):
96                 jobs = int(arg)
97                 if jobs <= 0:
98                         raise Error('Please specify a valid number of jobs')
99                 return jobs
100
101         command = None
102         params = {}
103         targets = []
104
105         i = 1
106         while i < len(args):
107                 if args[i].startswith('-'):
108                         if not command:
109                                 if args[i] == '-v':
110                                         config.verbose = True
111                                 elif args[i] == '-d':
112                                         config.debug = True
113                                 elif args[i] == '-r':
114                                         i += 1
115                                         config.roots.append(args[i])
116                                 elif args[i] == '-f':
117                                         config.force = True
118                                 elif args[i] in ('-h', '--help'):
119                                         help(sys.stdout, args)
120                                         sys.exit(0)
121                                 else:
122                                         raise Error('Bad option: ' + args[i])
123                         elif command == 'install':
124                                 if args[i] == '-j':
125                                         i += 1
126                                         params['build_jobs'] = parse_jobs(args[i])
127                                 elif args[i] == '-mj':
128                                         i += 1
129                                         params['make_jobs'] = parse_jobs(args[i])
130                                 else:
131                                         raise Error('Bad parameter: ' + args[i])
132                         elif command == 'rootfs':
133                                 if args[i] == '-n':
134                                         i += 1
135                                         params['clean'] = False
136                                 elif args[i] == '-r':
137                                         i += 1
138                                         params['rootfs_only'] = True
139                                 elif args[i] == '-j':
140                                         i += 1
141                                         params['jffs2_only'] = True
142                                 elif args[i] == '-d':
143                                         i += 1
144                                         params['devrootfs_only'] = True
145                         else:
146                                 raise Error('Command takes no parameters')
147                 elif not command:
148                         command = args[i]
149                 else:
150                         targets.append(args[i])
151                 i += 1
152
153         if not command:
154                 help(sys.stderr, args)
155                 sys.exit(1)
156
157         return command, params, targets
158
159 def parse_config(path):
160         if os.path.isabs(path):
161                 path = os.path.join(config.top_dir, path)
162
163         if os.path.exists(path):
164                 print 'Reading', path
165                 execfile(path, config.__dict__, config.__dict__)
166
167 def update_components():
168         targets = []
169         packages = {}
170
171         for c in config.components.itervalues():
172                 update_component_url(c)
173                 download_component(c)
174                 update_component_packages(c, targets, packages)
175                 c.active_depends = []
176
177         update_component_depends(packages)
178
179         return targets
180
181 def update_component_url(c):
182         update_repository_url(c.repo)
183         update_repository_url(c.meta)
184
185 def update_repository_url(repo):
186         if git.contains_database(repo.path):
187                 url = git.getvar(repo.path, 'remote.origin.url')
188                 if url:
189                         repo.active_url = url
190                         if config.debug:
191                                 print 'Using', repo.active_url
192                         return
193
194         for root in config.roots:
195                 for suffix in ('.git', ''):
196                         url = '%s/%s%s' % (root, repo.name, suffix)
197
198                         if config.debug:
199                                 print 'Trying', url
200
201                         if not git.peek_remote(url, quiet=True):
202                                 continue
203
204                         repo.active_url = url
205                         if config.debug:
206                                 print 'Found', repo.active_url
207                         return
208
209         raise Error('Failed to locate repository: ' + repo.name)
210
211 def update_component_depends(packages):
212         for pkg in packages.itervalues():
213                 for spec in pkg.depends.split():
214                         depname = Depend(spec).name
215                         deppkg = packages.get(depname)
216
217                         if not deppkg:
218                                 print >>sys.stderr, 'Package', pkg.name, \
219                                         'depends on non-existent package', depname
220                                 continue
221
222                         if deppkg.component == pkg.component:
223                                 continue
224
225                         pkg.component.active_depends.append(deppkg.component)
226
227         fail = False
228         for pkg in packages.itervalues():
229                 for spec in pkg.depends.split():
230                         if not Depend(spec).check(packages):
231                                 fail = True
232                                 print >>sys.stderr, 'Dependency', spec, \
233                                         'failed for', pkg.name
234
235                 for spec in pkg.conflicts.split():
236                         if Depend(spec).check(packages):
237                                 fail = True
238                                 print >>sys.stderr, pkg.name, \
239                                         'conflicts with', spec
240
241         if fail:
242                 raise Error('Invalid component tree')
243
244 class Depend(object):
245         regex = re.compile(r'([@]?)([^\s:]+)[:]?([<>=]*)([^\s:]*)[:]?(.*)')
246
247         def __init__(self, spec):
248                 match = self.regex.match(spec)
249                 if not match:
250                         raise Error('Bad dependency specification: ' + spec)
251
252                 self.build, self.name, self.tag_op, self.tag, flags \
253                         = match.groups()
254                 self.flags = flags.split()
255
256         def check(self, packages):
257                 # TODO: check version and flags
258                 return self.name in packages
259
260 def execute(args):
261         if config.debug:
262                 print 'Executing:', ' '.join(args)
263
264         if os.spawnvp(os.P_WAIT, args[0], args) != 0:
265                 raise Error('Failed: ' + ' '.join(args))
266
267 def download_component(c, overwrite=False):
268         if os.path.exists(c.repo.path) and not overwrite:
269                 return
270
271         print 'Downloading component:', c.name
272
273         execute(['rm', '-rf', c.repo.path])
274         download_repository(c, c.repo)
275         download_repository(c, c.meta)
276
277 def download_repository(c, repo):
278         if config.debug:
279                 print 'Downloading repository:', repo.name
280
281         git.clone(repo.path, repo.active_url, checkout=False)
282
283         branch = c.get_active_tag()
284         git.create_branch(repo.path, branch, 'origin/%s' % branch,
285                           checkout=True, link=True, force=True)
286
287 def update_component_packages(c, targets, packages):
288         c.active_packages = {}
289
290         for path in glob.glob(os.path.join(c.meta.path, '*.package')):
291                 name = os.path.basename(path)[:-8]
292
293                 pkg = parse_package(name, c, path)
294                 if not pkg:
295                         continue
296
297                 c.active_packages[name] = pkg
298                 packages[name] = pkg
299
300                 if config.debug:
301                         print 'Component', c.name, 'provides', name
302
303         if c.active_packages:
304                 targets.append(c.name)
305         elif config.debug:
306                 print 'Component', c.name, 'does not provide any packages'
307
308 class Package(object):
309         def __init__(self, name, component):
310                 self.name = name
311                 self.component = component
312
313                 self.depends = []
314                 self.conflicts = []
315                 self.architectures = None
316
317 def parse_package(name, component, path):
318         pkg = Package(name, component)
319         execfile(path, pkg.__dict__, pkg.__dict__)
320
321         if pkg.architectures:
322                 arch = config.boards[config.board].arch
323                 if arch not in pkg.architectures:
324                         return None
325
326         return pkg
327
328 def build_components(targets, build_jobs=1, make_jobs=1):
329         selected = set([config.components[i] for i in targets])
330
331         depends = None
332         while depends is None or depends:
333                 depends = set()
334
335                 for c in selected:
336                         for dep in c.active_depends:
337                                 if dep not in selected:
338                                         depends.add(dep)
339
340                 selected |= depends
341
342         components = list(selected)
343         components.sort()
344
345         if config.debug:
346                 print 'Building components:'
347                 for c in components:
348                         print '\t' + c.name
349
350         jobs = {}
351
352         while True:
353                 found = False
354                 for c in components:
355                         if len(jobs) >= build_jobs:
356                                 break
357
358                         if component_buildable(c):
359                                 found = True
360
361                                 if not component_cached(c):
362                                         print 'Starting to build', c.name
363                                         start_job(c, jobs, make_jobs)
364                                 else:
365                                         print c.name, 'found from cache'
366                                         c.active_state = 'built'
367
368                 if not found and not jobs:
369                         for c in components:
370                                 if c.active_state != 'built':
371                                         raise Error('Internal error')
372                         break
373
374                 wait_for_job(jobs)
375
376 def component_buildable(c):
377         if c.active_state:
378                 if config.debug:
379                         print c.name, 'is in state:', c.active_state
380                 return False
381
382         for dep in c.active_depends:
383                 if config.components[dep.name].active_state != 'built':
384                         if config.debug:
385                                 print c.name, 'depends on unbuilt', dep.name
386                         return False
387
388         return True
389
390 def component_cached(c):
391         if c.from_platform:
392                 return True
393
394         for repo in (c.repo, c.meta):
395                 if git.ls_files(repo.path, ['-m', '-d']):
396                         if config.force:
397                                 return False
398                         else:
399                                 raise Error('Dirty files in ' + repo.path)
400
401         path = os.path.join(config.cache_dir, c.name)
402         if not os.path.exists(path):
403                 return False
404
405         file = open(path, 'r')
406         line = file.readline()
407         file.close()
408
409         match = re.match(r'([^\s]+)[\s]?([^\s]*)', line)
410         if not match:
411                 return False
412
413         hash, flagstring = match.groups()
414
415         if hash != get_component_hash(c):
416                 print 'Component has been changed:', c.name
417                 return False
418
419         if flagstring:
420                 flags = flagstring.split(',')
421         else:
422                 flags = []
423
424         if set(flags) != set(c.flags):
425                 print 'Component flags have been changed:', c.name
426                 return False
427
428         return True
429
430 def update_cache(c):
431         path = os.path.join(config.cache_dir, c.name)
432
433         dir = os.path.dirname(path)
434         if not os.path.exists(dir):
435                 os.makedirs(dir)
436
437         file = open(path, 'w')
438         done = False
439         try:
440                 print >>file, get_component_hash(c.repo), ','.join(c.flags)
441                 done = True
442         finally:
443                 file.close()
444                 if not done:
445                         os.remove(path)
446
447 def get_component_hash(c):
448         return '%s+%s' % (get_repository_hash(c.repo), get_repository_hash(c.meta))
449
450 def get_repository_hash(repo):
451         if repo.active_hash is None:
452                 repo.active_hash = git.rev_parse(repo.path, 'HEAD')
453         return repo.active_hash
454
455 def start_job(c, jobs, make_jobs):
456         board = config.boards[config.board]
457
458         makefile = os.path.join(config.script_dir, 'matrix.mak')
459
460         workdir = os.path.join(config.top_dir, 'src', c.name)
461         args = ['make', '--no-print-directory', '-f', makefile, '-C', workdir,
462                 '-j', str(make_jobs), 'build_matrix_component',
463                 'MATRIX_TOPDIR='    + os.path.abspath(config.top_dir),
464                 'MATRIX_SCRIPTDIR=' + os.path.abspath(config.script_dir),
465                 'MATRIX_COMPONENT=' + c.name,
466                 'MATRIX_ARCH='      + board.arch,
467                 'MATRIX_GCC_MARCH=' + board.gcc_march,
468                 'MATRIX_GCC_MCPU='  + board.gcc_mcpu,
469                 'MATRIX_GCC_MFPU='  + board.gcc_mfpu,
470                 'MATRIX_GCC_OPTIONS=' + board.gcc_options,
471                 'MATRIX_GNU_HOST='  + board.gnu_host,
472                 'MATRIX_LIBC='      + config.libc]
473
474         for flag in config.flags:
475                 args.append(flag + '=1')
476
477         if config.verbose:
478                 args.append('MATRIX_VERBOSE=1')
479
480         if config.debug:
481                 print 'Executing:', ' '.join(args)
482
483         pid = os.spawnvp(os.P_NOWAIT, args[0], args)
484
485         jobs[pid] = c
486         c.active_state = 'running'
487
488 def wait_for_job(jobs):
489         if not jobs:
490                 return
491
492         pid, status = os.wait()
493
494         c = jobs.get(pid)
495         if not c:
496                 return
497
498         del jobs[pid]
499
500         if status == 0:
501                 c.active_state = 'built'
502                 update_cache(c)
503                 print c.name, 'completed'
504         else:
505                 c.active_state = 'error'
506                 print >>sys.stderr, c.name, 'failed with status:', status
507
508                 wait_for_job(jobs)
509                 raise Error('Build failed')
510
511 def clean_components(targets):
512         if not targets:
513                 targets = config.components.keys()
514
515         for name in targets:
516                 print 'Cleaning component:', name
517
518                 c = config.components[name]
519
520                 paths = []
521                 for repo in (c.repo, c.meta):
522                         files = git.ls_files(repo.path, ['-o'])
523                         paths += [os.path.join(repo.path, i) for i in files]
524
525                 paths.sort()
526                 paths.reverse()
527
528                 cache = os.path.join(config.cache_dir, name)
529                 if os.path.exists(cache):
530                         paths.append(cache)
531
532                 for path in paths:
533                         if config.debug:
534                                 print 'Deleting', path
535
536                         if os.path.islink(path) or not os.path.isdir(path):
537                                 os.remove(path)
538                         elif not os.path.exists(os.path.join(path, '.git')):
539                                 os.rmdir(path)
540
541                 for repo in (c.repo, c.meta):
542                         files = git.ls_files(repo.path, ['-m', '-d'])
543                         if files:
544                                 print len(files), 'dirty files left in', repo.path
545
546 def pull_components(targets):
547         if not targets:
548                 targets = config.components.keys()
549
550         for name in targets:
551                 print 'Pulling component:', name
552
553                 c = config.components[name]
554                 git.pull(c.repo.path)
555                 git.pull(c.meta.path)
556
557 def source_dist_components(targets):
558         if not targets:
559                 targets = config.components.keys()
560
561         for name in targets:
562                 c = config.components[name]
563                 generate_component_changes(c, 'dist')
564                 package_component_sources(c, 'dist')
565
566 def generate_component_changes(c, location):
567         print 'Generating component change log:', c.name
568
569         path = os.path.join(location, c.name) + '.changes'
570
571         pathdir = os.path.dirname(path)
572         if not os.path.exists(pathdir):
573                 os.makedirs(pathdir)
574
575         fd = os.open(path, os.O_WRONLY | os.O_CREAT, 0644)
576         git.log(c.name, [c.get_active_tag()], fd=fd)
577         os.close(fd)
578
579 def package_component_sources(c, location):
580         print 'Packaging component sources:', c.name
581
582         rev=git.describe(c.name)
583         if rev:
584                 rev = '_'+rev
585         else:
586                 rev = ''
587         path = os.path.join(location, c.name) + rev + '.tar.bz2'
588         if os.path.exists(path):
589                 os.remove(path)
590
591         pathdir = os.path.dirname(path)
592         if not os.path.exists(pathdir):
593                 os.makedirs(pathdir)
594
595         git.archive(c.name,path,
596                                 prefix=os.path.basename(c.name)+'/',
597                                 branch=c.get_active_tag())
598
599 def build_rootfs(clean=True, rootfs_only=False, jffs2_only=False, devrootfs_only=False):
600         parse_config('rootfs')
601
602         f = os.popen('sb-conf cu')
603         target = f.read()
604         f.close()
605
606         rootfs = RootFS(target.strip())
607         rootfs.include_paths(config.include_paths)
608         rootfs.include_files(config.include_files)
609         rootfs.filter_paths(config.exclude_paths)
610         rootfs.filter_files(config.exclude_files)
611         rootfs.filter_expressions(config.exclude_expressions)
612         rootfs.add_paths(config.created_paths)
613         rootfs.set_devices(config.devices)
614         rootfs.set_change_owner(config.change_owner)
615         rootfs.set_erase_size(config.boards[config.board].flash_erase_size)
616         rootfs.set_pad_size(config.boards[config.board].flash_pad_size)
617
618         if rootfs_only:
619                 rootfs.generate(clean, build_target="rootfs")
620         elif jffs2_only:
621                 rootfs.generate(clean, build_target="jffs2")
622         elif devrootfs_only:
623                 rootfs.generate(clean, build_target="devrootfs")
624         else:
625                 rootfs.generate(clean, build_target="all")