reorganize repository layout
[matrix.git] / matrix / matrix.py
1 # Copyright (C) 2006-2008 Movial Oy
2 # Authors: Timo Savola <tsavola@movial.fi>
3 #          Toni Timonen
4 #          Kalle Vahlman <kalle.vahlman@movial.fi>
5 #          Tuomas Kulve <tuomas.kulve@movial.fi>
6
7 import glob
8 import os
9 import re
10 import sys
11 import tarfile
12 from sets import Set as set
13
14 import config
15 import git
16 from rootfs import RootFS
17
18 Error = RuntimeError
19
20 def main():
21         parse_config('config.local')
22
23         command, params, targets = parse_args(sys.argv)
24
25         parse_config('config')
26         parse_config('boards')
27         parse_config('components')
28
29         if 'global-cache' in config.flags:
30                 config.cache_dir = config.global_cache_dir
31
32         if not os.path.exists(config.cache_dir):
33                 os.makedirs(config.cache_dir)
34
35         all_targets = update_components()
36
37         for name in targets:
38                 if name not in config.components:
39                         raise Error('Component "%s" does not exist' % name)
40
41         if not targets:
42                 targets = all_targets
43
44         if command == 'install':
45                 build_components(targets, **params)
46         elif command == 'download':
47                 # Actual downloading took place already in update_components()
48                 pass
49         elif command == 'clean':
50                 clean_components(targets)
51         elif command == 'pull':
52                 pull_components(targets)
53         elif command == 'source-dist':
54                 source_dist_components(targets)
55         elif command == 'rootfs':
56                 build_rootfs(**params)
57         else:
58                 raise Error('Invalid command: ' + command)
59
60 def help(file, args):
61         print >>file, '''
62 Copyright (C) 2006-2008 Movial Oy
63
64 Usage: %(progname)s [options] [command] [params] [component1 component2 ...]
65
66 Components must be given like this: core/glibc core/busybox X/xserver
67 If no components are given, all of them will be subject to the operation.
68
69 Options:
70         -v              Verbose output
71         -d              Debug output
72         -r URL          Specify the root for component git repos to clone from.
73                         If this option is not given, the default specified in
74                         the components file will be used.  This option may be
75                         specified multiple times.
76         -h, --help      Print this help text.
77
78 Commands and their parameters:
79
80         download        Only download the components and validate the tree.
81         install         Download, build and install the components.
82                 -j N            Execute N build jobs in parallel.
83                 -mj N           Run component builds with make -j N.
84         clean           Remove all non-tracked files from the component git
85                         repository directories.
86         pull            Run git pull for the components.
87         source-dist     Download and package the component sources.
88         rootfs          Build a rootfs from the current target installation.
89                 -n              Do not clean env.faked and stripped directory.
90                 -r              Only generate a stripped rootfs.
91                 -j              Only generate a jffs2 image
92                 -d              Only generate a rootstrap (development rootfs).
93         list [parameter] [pattern]
94                 -p              List packages.
95                 -pv             List packages with full details.
96                 -c              List components (default).
97                 -cdep           List components and their dependencies.
98 ''' % {'progname': args[0]}
99
100 def parse_args(args):
101         def parse_jobs(arg):
102                 jobs = int(arg)
103                 if jobs <= 0:
104                         raise Error('Please specify a valid number of jobs')
105                 return jobs
106
107         command = None
108         params = {}
109         targets = []
110
111         i = 1
112         while i < len(args):
113                 if args[i].startswith('-'):
114                         if not command:
115                                 if args[i] == '-v':
116                                         config.verbose = True
117                                 elif args[i] == '-d':
118                                         config.debug = True
119                                 elif args[i] == '-r':
120                                         i += 1
121                                         config.roots.append(args[i])
122                                 elif args[i] in ('-h', '--help'):
123                                         help(sys.stdout, args)
124                                         sys.exit(0)
125                                 else:
126                                         raise Error('Bad option: ' + args[i])
127                         elif command == 'install':
128                                 if args[i] == '-j':
129                                         i += 1
130                                         params['build_jobs'] = parse_jobs(args[i])
131                                 elif args[i] == '-mj':
132                                         i += 1
133                                         params['make_jobs'] = parse_jobs(args[i])
134                                 else:
135                                         raise Error('Bad parameter: ' + args[i])
136                         elif command == 'rootfs':
137                                 if args[i] == '-n':
138                                         i += 1
139                                         params['clean'] = False
140                                 elif args[i] == '-r':
141                                         i += 1
142                                         params['rootfs_only'] = True
143                                 elif args[i] == '-j':
144                                         i += 1
145                                         params['jffs2_only'] = True
146                                 elif args[i] == '-d':
147                                         i += 1
148                                         params['devrootfs_only'] = True
149                         else:
150                                 raise Error('Command takes no parameters')
151                 elif not command:
152                         command = args[i]
153                 else:
154                         targets.append(args[i])
155                 i += 1
156
157         if not command:
158                 help(sys.stderr, args)
159                 sys.exit(1)
160
161         return command, params, targets
162
163 def parse_config(path):
164         if os.path.isabs(path):
165                 path = os.path.join(config.top_dir, path)
166
167         if os.path.exists(path):
168                 print 'Reading', path
169                 execfile(path, config.__dict__, config.__dict__)
170
171 def update_components():
172         targets = []
173         packages = {}
174
175         for c in config.components.itervalues():
176                 update_component_url(c)
177                 download_component(c)
178                 update_component_packages(c, targets, packages)
179                 c.active_depends = []
180
181         update_component_depends(packages)
182
183         return targets
184
185 def update_component_url(c):
186         update_repository_url(c.repo)
187         update_repository_url(c.meta)
188
189 def update_repository_url(repo):
190         url = git.getvar(repo.path, 'remote.origin.url')
191         if url:
192                 repo.active_url = url
193                 if config.debug:
194                         print 'Using', repo.active_url
195                 return
196
197         for root in config.roots:
198                 for suffix in ('.git', ''):
199                         url = '%s/%s%s' % (root, repo.name, suffix)
200
201                         if config.debug:
202                                 print 'Trying', url
203
204                         if not git.peek_remote(url, quiet=True):
205                                 continue
206
207                         repo.active_url = url
208                         if config.debug:
209                                 print 'Found', repo.active_url
210                         return
211
212         raise Error('Failed to locate repository: ' + repo.name)
213
214 def update_component_depends(packages):
215         for pkg in packages.itervalues():
216                 for spec in pkg.depends.split():
217                         depname = Depend(spec).name
218                         deppkg = packages.get(depname)
219
220                         if not deppkg:
221                                 print >>sys.stderr, 'Package', pkg.name, \
222                                         'depends on non-existent package', depname
223                                 continue
224
225                         if deppkg.component == pkg.component:
226                                 continue
227
228                         pkg.component.active_depends.append(deppkg.component)
229
230         fail = False
231         for pkg in packages.itervalues():
232                 for spec in pkg.depends.split():
233                         if not Depend(spec).check(packages):
234                                 fail = True
235                                 print >>sys.stderr, 'Dependency', spec, \
236                                         'failed for', pkg.name
237
238                 for spec in pkg.conflicts.split():
239                         if Depend(spec).check(packages):
240                                 fail = True
241                                 print >>sys.stderr, pkg.name, \
242                                         'conflicts with', spec
243
244         if fail:
245                 raise Error('Invalid component tree')
246
247 class Depend(object):
248         regex = re.compile(r'([@]?)([^\s:]+)[:]?([<>=]*)([^\s:]*)[:]?(.*)')
249
250         def __init__(self, spec):
251                 match = self.regex.match(spec)
252                 if not match:
253                         raise Error('Bad dependency specification: ' + spec)
254
255                 self.build, self.name, self.tag_op, self.tag, flags \
256                         = match.groups()
257                 self.flags = flags.split()
258
259         def check(self, packages):
260                 # TODO: check version and flags
261                 return self.name in packages
262
263 def execute(args):
264         if config.debug:
265                 print 'Executing:', ' '.join(args)
266
267         if os.spawnvp(os.P_WAIT, args[0], args) != 0:
268                 raise Error('Failed: ' + ' '.join(args))
269
270 def download_component(c, overwrite=False):
271         if os.path.exists(c.repo.path) and not overwrite:
272                 return
273
274         print 'Downloading component:', c.name
275
276         execute(['rm', '-rf', c.repo.path])
277         download_repository(c, c.repo)
278         download_repository(c, c.meta)
279
280 def download_repository(c, repo):
281         if config.debug:
282                 print 'Downloading repository:', repo.name
283
284         git.clone(repo.path, repo.active_url, checkout=False)
285
286         branch = c.get_active_tag()
287         git.create_branch(repo.path, branch, 'origin/%s' % branch,
288                           checkout=True, link=True, force=True)
289
290 def update_component_packages(c, targets, packages):
291         c.active_packages = {}
292
293         for path in glob.glob(os.path.join(c.meta.path, '*.package')):
294                 name = os.path.basename(path)[:-8]
295
296                 pkg = parse_package(name, c, path)
297                 if not pkg:
298                         continue
299
300                 c.active_packages[name] = pkg
301                 packages[name] = pkg
302
303                 if config.debug:
304                         print 'Component', c.name, 'provides', name
305
306         if c.active_packages:
307                 targets.append(c.name)
308         elif config.debug:
309                 print 'Component', c.name, 'does not provide any packages'
310
311 class Package(object):
312         def __init__(self, name, component):
313                 self.name = name
314                 self.component = component
315
316                 self.depends = []
317                 self.conflicts = []
318                 self.architectures = None
319
320 def parse_package(name, component, path):
321         pkg = Package(name, component)
322         execfile(path, pkg.__dict__, pkg.__dict__)
323
324         if pkg.architectures:
325                 arch = config.boards[config.board].arch
326                 if arch not in pkg.architectures:
327                         return None
328
329         return pkg
330
331 def build_components(targets, build_jobs=1, make_jobs=1):
332         selected = set([config.components[i] for i in targets])
333
334         depends = None
335         while depends is None or depends:
336                 depends = set()
337
338                 for c in selected:
339                         for dep in c.active_depends:
340                                 if dep not in selected:
341                                         depends.add(dep)
342
343                 selected |= depends
344
345         components = list(selected)
346         components.sort()
347
348         if config.debug:
349                 print 'Building components:'
350                 for c in components:
351                         print '\t' + c.name
352
353         jobs = {}
354
355         while True:
356                 found = False
357                 for c in components:
358                         if len(jobs) >= build_jobs:
359                                 break
360
361                         if component_buildable(c):
362                                 found = True
363
364                                 if not component_cached(c):
365                                         print 'Starting to build', c.name
366                                         start_job(c, jobs, make_jobs)
367                                 else:
368                                         print c.name, 'found from cache'
369                                         c.active_state = 'built'
370
371                 if not found and not jobs:
372                         for c in components:
373                                 if c.active_state != 'built':
374                                         raise Error('Internal error')
375                         break
376
377                 wait_for_job(jobs)
378
379 def component_buildable(c):
380         if c.active_state:
381                 if config.debug:
382                         print c.name, 'is in state:', c.active_state
383                 return False
384
385         for dep in c.active_depends:
386                 if config.components[dep.name].active_state != 'built':
387                         if config.debug:
388                                 print c.name, 'depends on unbuilt', dep.name
389                         return False
390
391         return True
392
393 def component_cached(c):
394         if c.from_platform:
395                 return True
396
397         ls_opts = ['-m', '-d']
398         if git.ls_files(c.repo.path, ls_opts):
399                 raise Error('Component contains dirty files: ' + c.name)
400         if git.ls_files(c.meta.path, ls_opts):
401                 raise Error('Component contains dirty meta files: ' + c.name)
402
403         path = os.path.join(config.cache_dir, c.name)
404         if not os.path.exists(path):
405                 return False
406
407         file = open(path, 'r')
408         line = file.readline()
409         file.close()
410
411         match = re.match(r'([^\s]+)[\s]?([^\s]*)', line)
412         if not match:
413                 return False
414
415         hash, flagstring = match.groups()
416
417         if hash != get_component_hash(c):
418                 print 'Component has been changed:', c.name
419                 return False
420
421         if flagstring:
422                 flags = flagstring.split(',')
423         else:
424                 flags = []
425
426         if set(flags) != set(c.flags):
427                 print 'Component flags have been changed:', c.name
428                 return False
429
430         return True
431
432 def update_cache(c):
433         path = os.path.join(config.cache_dir, c.name)
434
435         dir = os.path.dirname(path)
436         if not os.path.exists(dir):
437                 os.makedirs(dir)
438
439         file = open(path, 'w')
440         done = False
441         try:
442                 print >>file, get_component_hash(c.repo), ','.join(c.flags)
443                 done = True
444         finally:
445                 file.close()
446                 if not done:
447                         os.remove(path)
448
449 def get_component_hash(c):
450         return '%s+%s' % (get_repository_hash(c.repo), get_repository_hash(c.meta))
451
452 def get_repository_hash(repo):
453         if repo.active_hash is None:
454                 repo.active_hash = git.rev_parse(repo.path, 'HEAD')
455         return repo.active_hash
456
457 def start_job(c, jobs, make_jobs):
458         board = config.boards[config.board]
459
460         makefile = os.path.join(config.script_dir, 'matrix.mak')
461
462         workdir = os.path.join(config.top_dir, 'src', c.name)
463         args = ['make', '--no-print-directory', '-f', makefile, '-C', workdir,
464                 '-j', str(make_jobs), 'build_matrix_component',
465                 'MATRIX_TOPDIR='    + config.top_dir,
466                 'MATRIX_SCRIPTDIR=' + config.script_dir,
467                 'MATRIX_COMPONENT=' + c.name,
468                 'MATRIX_ARCH='      + board.arch,
469                 'MATRIX_GCC_MARCH=' + board.gcc_march,
470                 'MATRIX_GCC_MCPU='  + board.gcc_mcpu,
471                 'MATRIX_GCC_MFPU='  + board.gcc_mfpu,
472                 'MATRIX_GCC_OPTIONS=' + board.gcc_options,
473                 'MATRIX_GNU_HOST='  + board.gnu_host,
474                 'MATRIX_LIBC='      + config.libc]
475
476         for flag in config.flags:
477                 args.append(flag + '=1')
478
479         if config.verbose:
480                 args.append('MATRIX_VERBOSE=1')
481
482         if config.debug:
483                 print 'Executing:', ' '.join(args)
484
485         pid = os.spawnvp(os.P_NOWAIT, args[0], args)
486
487         jobs[pid] = c
488         c.active_state = 'running'
489
490 def wait_for_job(jobs):
491         if not jobs:
492                 return
493
494         pid, status = os.wait()
495
496         c = jobs.get(pid)
497         if not c:
498                 return
499
500         del jobs[pid]
501
502         if status == 0:
503                 c.active_state = 'built'
504                 update_cache(c)
505                 print c.name, 'completed'
506         else:
507                 c.active_state = 'error'
508                 print >>sys.stderr, c.name, 'failed with status:', status
509
510                 wait_for_job(jobs)
511                 raise Error('Build failed')
512
513 def clean_components(targets):
514         if not targets:
515                 targets = config.components.keys()
516
517         for name in targets:
518                 print 'Cleaning component:', name
519
520                 c = config.components[name]
521
522                 paths = []
523                 for repo in (c.repo, c.meta):
524                         files = git.ls_files(repo.path, ['-o'])
525                         paths += [os.path.join(repo.path, i) for i in files]
526
527                 paths.sort()
528                 paths.reverse()
529
530                 cache = os.path.join(config.cache_dir, name)
531                 if os.path.exists(cache):
532                         paths.append(cache)
533
534                 for path in paths:
535                         if config.debug:
536                                 print 'Deleting', path
537
538                         if os.path.islink(path) or not os.path.isdir(path):
539                                 os.remove(path)
540                         elif not os.path.exists(os.path.join(path, '.git')):
541                                 os.rmdir(path)
542
543                 for repo in (c.repo, c.meta):
544                         files = git.ls_files(repo.path, ['-m', '-d'])
545                         if files:
546                                 print len(files), 'dirty files left in', repo.path
547
548 def pull_components(targets):
549         if not targets:
550                 targets = config.components.keys()
551
552         for name in targets:
553                 print 'Pulling component:', name
554
555                 c = config.components[name]
556                 git.pull(c.repo.path)
557                 git.pull(c.meta.path)
558
559 def source_dist_components(targets):
560         if not targets:
561                 targets = config.components.keys()
562
563         for name in targets:
564                 c = config.components[name]
565                 generate_component_changes(c, 'dist')
566                 package_component_sources(c, 'dist')
567
568 def generate_component_changes(c, location):
569         print 'Generating component change log:', c.name
570
571         path = os.path.join(location, c.name) + '.changes'
572
573         pathdir = os.path.dirname(path)
574         if not os.path.exists(pathdir):
575                 os.makedirs(pathdir)
576
577         fd = os.open(path, os.O_WRONLY | os.O_CREAT, 0644)
578         git.log(c.name, [c.get_active_tag()], fd=fd)
579         os.close(fd)
580
581 def package_component_sources(c, location):
582         print 'Packaging component sources:', c.name
583
584         rev=git.describe(c.name)
585         if rev:
586                 rev = '_'+rev
587         else:
588                 rev = ''
589         path = os.path.join(location, c.name) + rev + '.tar.bz2'
590         if os.path.exists(path):
591                 os.remove(path)
592
593         pathdir = os.path.dirname(path)
594         if not os.path.exists(pathdir):
595                 os.makedirs(pathdir)
596
597         git.archive(c.name,path,
598                                 prefix=os.path.basename(c.name)+'/',
599                                 branch=c.get_active_tag())
600
601 def build_rootfs(clean=True, rootfs_only=False, jffs2_only=False, devrootfs_only=False):
602         parse_config('rootfs')
603
604         f = os.popen('sb-conf cu')
605         target = f.read()
606         f.close()
607
608         rootfs = RootFS(target.strip())
609         rootfs.include_paths(config.include_paths)
610         rootfs.include_files(config.include_files)
611         rootfs.filter_paths(config.exclude_paths)
612         rootfs.filter_files(config.exclude_files)
613         rootfs.filter_expressions(config.exclude_expressions)
614         rootfs.add_paths(config.created_paths)
615         rootfs.set_devices(config.devices)
616         rootfs.set_change_owner(config.change_owner)
617         rootfs.set_erase_size(config.boards[config.board].flash_erase_size)
618         rootfs.set_pad_size(config.boards[config.board].flash_pad_size)
619
620         if rootfs_only:
621                 rootfs.generate(clean, build_target="rootfs")
622         elif jffs2_only:
623                 rootfs.generate(clean, build_target="jffs2")
624         elif devrootfs_only:
625                 rootfs.generate(clean, build_target="devrootfs")
626         else:
627                 rootfs.generate(clean, build_target="all")