use built-in set type
[matrix.git] / matrix / matrix.py
1 # Copyright (C) 2006-2008 Movial Oy
2 # Authors: Timo Savola <tsavola@movial.fi>
3 #          Toni Timonen
4 #          Kalle Vahlman <kalle.vahlman@movial.fi>
5 #          Tuomas Kulve <tuomas.kulve@movial.fi>
6
7 import glob
8 import os
9 import re
10 import sys
11 import tarfile
12
13 import config
14 import git
15 from rootfs import RootFS
16
17 Error = RuntimeError
18
19 def main():
20         parse_config('config.local')
21
22         command, params, targets = parse_args(sys.argv)
23
24         parse_config('config')
25         parse_config('boards')
26         parse_config('components/components')
27
28         if 'global-cache' in config.flags:
29                 config.cache_dir = config.global_cache_dir
30
31         if not os.path.exists(config.cache_dir):
32                 os.makedirs(config.cache_dir)
33
34         all_targets = update_components()
35
36         for name in targets:
37                 if name not in config.components:
38                         raise Error('Component "%s" does not exist' % name)
39
40         if not targets:
41                 targets = all_targets
42
43         if command == 'install':
44                 build_components(targets, **params)
45         elif command == 'download':
46                 # Actual downloading took place already in update_components()
47                 pass
48         elif command == 'clean':
49                 clean_components(targets)
50         elif command == 'pull':
51                 pull_components(targets)
52         elif command == 'source-dist':
53                 source_dist_components(targets)
54         elif command == 'rootfs':
55                 build_rootfs(**params)
56         else:
57                 raise Error('Invalid command: ' + command)
58
59 def help(file, args):
60         print >>file, '''
61 Copyright (C) 2006-2008 Movial Oy
62
63 Usage: %(progname)s [options] [command] [params] [component1 component2 ...]
64
65 Components must be given like this: core/glibc core/busybox X/xserver
66 If no components are given, all of them will be subject to the operation.
67
68 Options:
69         -v              Verbose output
70         -d              Debug output
71         -r URL          Specify the root for component git repos to clone from.
72                         If this option is not given, the default specified in
73                         the components file will be used.  This option may be
74                         specified multiple times.
75         -h, --help      Print this help text.
76
77 Commands and their parameters:
78
79         download        Only download the components and validate the tree.
80         install         Download, build and install the components.
81                 -j N            Execute N build jobs in parallel.
82                 -mj N           Run component builds with make -j N.
83         clean           Remove all non-tracked files from the component git
84                         repository directories.
85         pull            Run git pull for the components.
86         source-dist     Download and package the component sources.
87         rootfs          Build a rootfs from the current target installation.
88                 -n              Do not clean env.faked and stripped directory.
89                 -r              Only generate a stripped rootfs.
90                 -j              Only generate a jffs2 image
91                 -d              Only generate a rootstrap (development rootfs).
92         list [parameter] [pattern]
93                 -p              List packages.
94                 -pv             List packages with full details.
95                 -c              List components (default).
96                 -cdep           List components and their dependencies.
97 ''' % {'progname': args[0]}
98
99 def parse_args(args):
100         def parse_jobs(arg):
101                 jobs = int(arg)
102                 if jobs <= 0:
103                         raise Error('Please specify a valid number of jobs')
104                 return jobs
105
106         command = None
107         params = {}
108         targets = []
109
110         i = 1
111         while i < len(args):
112                 if args[i].startswith('-'):
113                         if not command:
114                                 if args[i] == '-v':
115                                         config.verbose = True
116                                 elif args[i] == '-d':
117                                         config.debug = True
118                                 elif args[i] == '-r':
119                                         i += 1
120                                         config.roots.append(args[i])
121                                 elif args[i] in ('-h', '--help'):
122                                         help(sys.stdout, args)
123                                         sys.exit(0)
124                                 else:
125                                         raise Error('Bad option: ' + args[i])
126                         elif command == 'install':
127                                 if args[i] == '-j':
128                                         i += 1
129                                         params['build_jobs'] = parse_jobs(args[i])
130                                 elif args[i] == '-mj':
131                                         i += 1
132                                         params['make_jobs'] = parse_jobs(args[i])
133                                 else:
134                                         raise Error('Bad parameter: ' + args[i])
135                         elif command == 'rootfs':
136                                 if args[i] == '-n':
137                                         i += 1
138                                         params['clean'] = False
139                                 elif args[i] == '-r':
140                                         i += 1
141                                         params['rootfs_only'] = True
142                                 elif args[i] == '-j':
143                                         i += 1
144                                         params['jffs2_only'] = True
145                                 elif args[i] == '-d':
146                                         i += 1
147                                         params['devrootfs_only'] = True
148                         else:
149                                 raise Error('Command takes no parameters')
150                 elif not command:
151                         command = args[i]
152                 else:
153                         targets.append(args[i])
154                 i += 1
155
156         if not command:
157                 help(sys.stderr, args)
158                 sys.exit(1)
159
160         return command, params, targets
161
162 def parse_config(path):
163         if os.path.isabs(path):
164                 path = os.path.join(config.top_dir, path)
165
166         if os.path.exists(path):
167                 print 'Reading', path
168                 execfile(path, config.__dict__, config.__dict__)
169
170 def update_components():
171         targets = []
172         packages = {}
173
174         for c in config.components.itervalues():
175                 update_component_url(c)
176                 download_component(c)
177                 update_component_packages(c, targets, packages)
178                 c.active_depends = []
179
180         update_component_depends(packages)
181
182         return targets
183
184 def update_component_url(c):
185         url=git.getvar(c.name,'remote.origin.url')
186         if url:
187                 c.active_url=url
188                 if config.debug:
189                         print 'Using', c.active_url
190                 return
191
192         for root in config.roots:
193                 for suffix in ('.git', ''):
194                         url = root + '/' + c.name + suffix
195
196                         if config.debug:
197                                 print 'Trying', url
198
199                         if not git.peek_remote(url, quiet=True):
200                                 continue
201
202                         c.active_url = url
203                         if config.debug:
204                                 print 'Found', c.active_url
205                         return
206
207         raise Error('Failed to locate repository of ' + c.name)
208
209 def update_component_depends(packages):
210         for pkg in packages.itervalues():
211                 for spec in pkg.depends.split():
212                         depname = Depend(spec).name
213                         deppkg = packages.get(depname)
214
215                         if not deppkg:
216                                 print >>sys.stderr, 'Package', pkg.name, \
217                                         'depends on non-existent package', depname
218                                 continue
219
220                         if deppkg.component == pkg.component:
221                                 continue
222
223                         pkg.component.active_depends.append(deppkg.component)
224
225         fail = False
226         for pkg in packages.itervalues():
227                 for spec in pkg.depends.split():
228                         if not Depend(spec).check(packages):
229                                 fail = True
230                                 print >>sys.stderr, 'Dependency', spec, \
231                                         'failed for', pkg.name
232
233                 for spec in pkg.conflicts.split():
234                         if Depend(spec).check(packages):
235                                 fail = True
236                                 print >>sys.stderr, pkg.name, \
237                                         'conflicts with', spec
238
239         if fail:
240                 raise Error('Invalid component tree')
241
242 class Depend(object):
243         regex = re.compile(r'([@]?)([^\s:]+)[:]?([<>=]*)([^\s:]*)[:]?(.*)')
244
245         def __init__(self, spec):
246                 match = self.regex.match(spec)
247                 if not match:
248                         raise Error('Bad dependency specification: ' + spec)
249
250                 self.build, self.name, self.tag_op, self.tag, flags \
251                         = match.groups()
252                 self.flags = flags.split()
253
254         def check(self, packages):
255                 # TODO: check version and flags
256                 return self.name in packages
257
258 def execute(args):
259         if config.debug:
260                 print 'Executing:', ' '.join(args)
261
262         if os.spawnvp(os.P_WAIT, args[0], args) != 0:
263                 raise Error('Failed: ' + ' '.join(args))
264
265 def download_component(c, overwrite=False):
266         if os.path.exists(c.name) and not overwrite:
267                 return
268
269         print 'Downloading component:', c.name
270
271         execute(['rm', '-rf', c.name])
272         git.clone(c.name, c.active_url, checkout=False)
273
274         branch = c.get_active_tag()
275         git.create_branch(c.name, branch, 'origin/%s'%branch, checkout=True, link=True,force=True)
276
277 def update_component_packages(c, targets, packages):
278         c.active_packages = {}
279
280         for path in glob.glob(os.path.join(c.name, '.matrix', '*.package')):
281                 name = os.path.basename(path)[:-8]
282
283                 pkg = parse_package(name, c, path)
284                 if not pkg:
285                         continue
286
287                 c.active_packages[name] = pkg
288                 packages[name] = pkg
289
290                 if config.debug:
291                         print 'Component', c.name, 'provides', name
292
293         if c.active_packages:
294                 targets.append(c.name)
295         elif config.debug:
296                 print 'Component', c.name, 'does not provide any packages'
297
298 class Package(object):
299         def __init__(self, name, component):
300                 self.name = name
301                 self.component = component
302
303                 self.depends = []
304                 self.conflicts = []
305                 self.architectures = None
306
307 def parse_package(name, component, path):
308         pkg = Package(name, component)
309         execfile(path, pkg.__dict__, pkg.__dict__)
310
311         if pkg.architectures:
312                 arch = config.boards[config.board].arch
313                 if arch not in pkg.architectures:
314                         return None
315
316         return pkg
317
318 def get_active_arch():
319         config.boards[config.board].arch
320
321 def build_components(targets, build_jobs=1, make_jobs=1):
322         selected = set([config.components[i] for i in targets])
323
324         depends = None
325         while depends is None or depends:
326                 depends = set()
327
328                 for c in selected:
329                         for dep in c.active_depends:
330                                 if dep not in selected:
331                                         depends.add(dep)
332
333                 selected |= depends
334
335         components = list(selected)
336         components.sort()
337
338         if config.debug:
339                 print 'Building components:'
340                 for c in components:
341                         print '\t' + c.name
342
343         jobs = {}
344
345         while True:
346                 found = False
347                 for c in components:
348                         if len(jobs) >= build_jobs:
349                                 break
350
351                         if component_buildable(c):
352                                 found = True
353
354                                 if not component_cached(c):
355                                         print 'Starting to build', c.name
356                                         start_job(c, jobs, make_jobs)
357                                 else:
358                                         print c.name, 'found from cache'
359                                         c.active_state = 'built'
360
361                 if not found and not jobs:
362                         for c in components:
363                                 if c.active_state != 'built':
364                                         raise Error('Internal error')
365                         break
366
367                 wait_for_job(jobs)
368
369 def component_buildable(c):
370         if c.active_state:
371                 if config.debug:
372                         print c.name, 'is in state:', c.active_state
373                 return False
374
375         for dep in c.active_depends:
376                 if config.components[dep.name].active_state != 'built':
377                         if config.debug:
378                                 print c.name, 'depends on unbuilt', dep.name
379                         return False
380
381         return True
382
383 def component_cached(c):
384         if c.from_platform:
385                 return True
386
387         if git.ls_files(c.name, ['-m', '-d']):
388                 raise Error('Component contains dirty files: ' + c.name)
389
390         path = os.path.join(config.cache_dir, c.name)
391         if not os.path.exists(path):
392                 return False
393
394         file = open(path, 'r')
395         line = file.readline()
396         file.close()
397
398         match = re.match(r'([^\s]+)[\s]?([^\s]*)', line)
399         if not match:
400                 return False
401
402         hash, flagstring = match.groups()
403
404         if hash != c.get_active_hash():
405                 print 'Component has been changed:', c.name
406                 return False
407
408         if flagstring:
409                 flags = flagstring.split(',')
410         else:
411                 flags = []
412
413         if set(flags) != set(c.flags):
414                 print 'Component flags have been changed:', c.name
415                 return False
416
417         return True
418
419 def update_cache(c):
420         path = os.path.join(config.cache_dir, c.name)
421
422         dir = os.path.dirname(path)
423         if not os.path.exists(dir):
424                 os.makedirs(dir)
425
426         file = open(path, 'w')
427         print >>file, c.get_active_hash(), ','.join(c.flags)
428         file.close()
429
430 def start_job(c, jobs, make_jobs):
431         board = config.boards[config.board]
432
433         makefile = os.path.join(config.top_dir, 'scripts', 'matrix.mak')
434
435         args = ['make', '--no-print-directory', '-f', makefile, '-C', c.name,
436                 '-j', str(make_jobs), 'build_matrix_component',
437                 'MATRIX_TOPDIR='    + config.top_dir,
438                 'MATRIX_COMPONENT=' + c.name,
439                 'MATRIX_ARCH='      + board.arch,
440                 'MATRIX_GCC_MARCH=' + board.gcc_march,
441                 'MATRIX_GCC_MCPU='  + board.gcc_mcpu,
442                 'MATRIX_GCC_MFPU='  + board.gcc_mfpu,
443                 'MATRIX_GCC_OPTIONS=' + board.gcc_options,
444                 'MATRIX_GNU_HOST='  + board.gnu_host,
445                 'MATRIX_LIBC='      + config.libc]
446
447         for flag in config.flags:
448                 args.append(flag + '=1')
449
450         if config.verbose:
451                 args.append('MATRIX_VERBOSE=1')
452
453         if config.debug:
454                 print 'Executing:', ' '.join(args)
455
456         pid = os.spawnvp(os.P_NOWAIT, args[0], args)
457
458         jobs[pid] = c
459         c.active_state = 'running'
460
461 def wait_for_job(jobs):
462         if not jobs:
463                 return
464
465         pid, status = os.wait()
466
467         c = jobs.get(pid)
468         if not c:
469                 return
470
471         del jobs[pid]
472
473         if status == 0:
474                 c.active_state = 'built'
475                 update_cache(c)
476                 print c.name, 'completed'
477         else:
478                 c.active_state = 'error'
479                 print >>sys.stderr, c.name, 'failed with status:', status
480
481                 wait_for_job(jobs)
482                 raise Error('Build failed')
483
484 def clean_components(targets):
485         if not targets:
486                 targets = config.components.keys()
487
488         for name in targets:
489                 print 'Cleaning component:', name
490
491                 files = git.ls_files(name, ['-o'])
492                 paths = [os.path.join(name, i) for i in files]
493                 paths.sort()
494                 paths.reverse()
495
496                 cache = os.path.join(config.cache_dir, name)
497                 if os.path.exists(cache):
498                         paths.append(cache)
499
500                 for path in paths:
501                         if config.debug:
502                                 print 'Deleting', path
503
504                         if os.path.islink(path) or not os.path.isdir(path):
505                                 os.remove(path)
506                         else:
507                                 os.rmdir(path)
508
509                 files = git.ls_files(name, ['-m', '-d'])
510                 if files:
511                         print len(files), 'dirty files left in', name
512
513 def pull_components(targets):
514         if not targets:
515                 targets = config.components.keys()
516
517         for name in targets:
518                 print 'Pulling component:', name
519                 c = config.components[name]
520                 git.pull(name)
521
522 def source_dist_components(targets):
523         if not targets:
524                 targets = config.components.keys()
525
526         for name in targets:
527                 c = config.components[name]
528                 generate_component_changes(c, 'dist')
529                 package_component_sources(c, 'dist')
530
531 def generate_component_changes(c, location):
532         print 'Generating component change log:', c.name
533
534         path = os.path.join(location, c.name) + '.changes'
535
536         pathdir = os.path.dirname(path)
537         if not os.path.exists(pathdir):
538                 os.makedirs(pathdir)
539
540         fd = os.open(path, os.O_WRONLY | os.O_CREAT, 0644)
541         git.log(c.name, [c.get_active_tag()], fd=fd)
542         os.close(fd)
543
544 def package_component_sources(c, location):
545         print 'Packaging component sources:', c.name
546
547         rev=git.describe(c.name)
548         if rev:
549                 rev = '_'+rev
550         else:
551                 rev = ''
552         path = os.path.join(location, c.name) + rev + '.tar.bz2'
553         if os.path.exists(path):
554                 os.remove(path)
555
556         pathdir = os.path.dirname(path)
557         if not os.path.exists(pathdir):
558                 os.makedirs(pathdir)
559
560         git.archive(c.name,path,
561                                 prefix=os.path.basename(c.name)+'/',
562                                 branch=c.get_active_tag())
563
564 def build_rootfs(clean=True, rootfs_only=False, jffs2_only=False, devrootfs_only=False):
565         parse_config('rootfs')
566
567         f = os.popen('sb-conf cu')
568         target = f.read()
569         f.close()
570
571         rootfs = RootFS(target.strip())
572         rootfs.include_paths(config.include_paths)
573         rootfs.include_files(config.include_files)
574         rootfs.filter_paths(config.exclude_paths)
575         rootfs.filter_files(config.exclude_files)
576         rootfs.filter_expressions(config.exclude_expressions)
577         rootfs.add_paths(config.created_paths)
578         rootfs.set_devices(config.devices)
579         rootfs.set_change_owner(config.change_owner)
580         rootfs.set_erase_size(config.boards[config.board].flash_erase_size)
581         rootfs.set_pad_size(config.boards[config.board].flash_pad_size)
582
583         if rootfs_only:
584                 rootfs.generate(clean, build_target="rootfs")
585         elif jffs2_only:
586                 rootfs.generate(clean, build_target="jffs2")
587         elif devrootfs_only:
588                 rootfs.generate(clean, build_target="devrootfs")
589         else:
590                 rootfs.generate(clean, build_target="all")