use new-style classes
[matrix.git] / matrix / matrix.py
1 # Copyright (C) 2006-2008 Movial Oy
2 # Authors: Timo Savola <tsavola@movial.fi>
3 #          Toni Timonen
4 #          Kalle Vahlman <kalle.vahlman@movial.fi>
5 #          Tuomas Kulve <tuomas.kulve@movial.fi>
6
7 import sys
8 import os
9 import glob
10 import re
11 import tarfile
12 from sets import Set as set
13
14 import config
15 import git
16 from rootfs import RootFS
17
18 Error = RuntimeError
19
20 def main():
21         parse_config('config.local')
22
23         command, params, targets = parse_args(sys.argv)
24
25         parse_config('config')
26         parse_config('boards')
27         parse_config('components/components')
28
29         if 'global-cache' in config.flags:
30                 config.cache_dir = config.global_cache_dir
31
32         if not os.path.exists(config.cache_dir):
33                 os.makedirs(config.cache_dir)
34
35         all_targets = update_components()
36
37         for name in targets:
38                 if name not in config.components:
39                         raise Error('Component "%s" does not exist' % name)
40
41         if not targets:
42                 targets = all_targets
43
44         if command == 'install':
45                 build_components(targets, **params)
46         elif command == 'download':
47                 # Actual downloading took place already in update_components()
48                 pass
49         elif command == 'clean':
50                 clean_components(targets)
51         elif command == 'pull':
52                 pull_components(targets)
53         elif command == 'source-dist':
54                 source_dist_components(targets)
55         elif command == 'rootfs':
56                 build_rootfs(**params)
57         else:
58                 raise Error('Invalid command: ' + command)
59
60 def help(file, args):
61         print >>file, '''
62 Copyright (C) 2006-2008 Movial Oy
63
64 Usage: %(progname)s [options] [command] [params] [component1 component2 ...]
65
66 Components must be given like this: core/glibc core/busybox X/xserver
67 If no components are given, all of them will be subject to the operation.
68
69 Options:
70         -v              Verbose output
71         -d              Debug output
72         -r URL          Specify the root for component git repos to clone from.
73                         If this option is not given, the default specified in
74                         the components file will be used.  This option may be
75                         specified multiple times.
76         -h, --help      Print this help text.
77
78 Commands and their parameters:
79
80         download        Only download the components and validate the tree.
81         install         Download, build and install the components.
82                 -j N            Execute N build jobs in parallel.
83                 -mj N           Run component builds with make -j N.
84         clean           Remove all non-tracked files from the component git
85                         repository directories.
86         pull            Run git pull for the components.
87         source-dist     Download and package the component sources.
88         rootfs          Build a rootfs from the current target installation.
89                 -n              Do not clean env.faked and stripped directory.
90                 -r              Only generate a stripped rootfs.
91                 -j              Only generate a jffs2 image
92                 -d              Only generate a rootstrap (development rootfs).
93         list [parameter] [pattern]
94                 -p              List packages.
95                 -pv             List packages with full details.
96                 -c              List components (default).
97                 -cdep           List components and their dependencies.
98 ''' % {'progname': args[0]}
99
100 def parse_args(args):
101         def parse_jobs(arg):
102                 jobs = int(arg)
103                 if jobs <= 0:
104                         raise Error('Please specify a valid number of jobs')
105                 return jobs
106
107         command = None
108         params = {}
109         targets = []
110
111         i = 1
112         while i < len(args):
113                 if args[i].startswith('-'):
114                         if not command:
115                                 if args[i] == '-v':
116                                         config.verbose = True
117                                 elif args[i] == '-d':
118                                         config.debug = True
119                                 elif args[i] == '-r':
120                                         i += 1
121                                         config.roots.append(args[i])
122                                 elif args[i] in ('-h', '--help'):
123                                         help(sys.stdout, args)
124                                         sys.exit(0)
125                                 else:
126                                         raise Error('Bad option: ' + args[i])
127                         elif command == 'install':
128                                 if args[i] == '-j':
129                                         i += 1
130                                         params['build_jobs'] = parse_jobs(args[i])
131                                 elif args[i] == '-mj':
132                                         i += 1
133                                         params['make_jobs'] = parse_jobs(args[i])
134                                 else:
135                                         raise Error('Bad parameter: ' + args[i])
136                         elif command == 'rootfs':
137                                 if args[i] == '-n':
138                                         i += 1
139                                         params['clean'] = False
140                                 elif args[i] == '-r':
141                                         i += 1
142                                         params['rootfs_only'] = True
143                                 elif args[i] == '-j':
144                                         i += 1
145                                         params['jffs2_only'] = True
146                                 elif args[i] == '-d':
147                                         i += 1
148                                         params['devrootfs_only'] = True
149                         else:
150                                 raise Error('Command takes no parameters')
151                 elif not command:
152                         command = args[i]
153                 else:
154                         targets.append(args[i])
155                 i += 1
156
157         if not command:
158                 help(sys.stderr, args)
159                 sys.exit(1)
160
161         return command, params, targets
162
163 def parse_config(path):
164         if os.path.isabs(path):
165                 path = os.path.join(config.top_dir, path)
166
167         if os.path.exists(path):
168                 print 'Reading', path
169                 execfile(path, config.__dict__, config.__dict__)
170
171 def update_components():
172         targets = []
173         packages = {}
174
175         for c in config.components.itervalues():
176                 update_component_url(c)
177                 download_component(c)
178                 update_component_packages(c, targets, packages)
179                 c.active_depends = []
180
181         update_component_depends(packages)
182
183         return targets
184
185 def update_component_url(c):
186         url=git.getvar(c.name,'remote.origin.url')
187         if url:
188                 c.active_url=url
189                 if config.debug:
190                         print 'Using', c.active_url
191                 return
192
193         for root in config.roots:
194                 for suffix in ('.git', ''):
195                         url = root + '/' + c.name + suffix
196
197                         if config.debug:
198                                 print 'Trying', url
199
200                         if not git.peek_remote(url, quiet=True):
201                                 continue
202
203                         c.active_url = url
204                         if config.debug:
205                                 print 'Found', c.active_url
206                         return
207
208         raise Error('Failed to locate repository of ' + c.name)
209
210 def update_component_depends(packages):
211         for pkg in packages.itervalues():
212                 for spec in pkg.depends.split():
213                         depname = Depend(spec).name
214                         deppkg = packages.get(depname)
215
216                         if not deppkg:
217                                 print >>sys.stderr, 'Package', pkg.name, \
218                                         'depends on non-existent package', depname
219                                 continue
220
221                         if deppkg.component == pkg.component:
222                                 continue
223
224                         pkg.component.active_depends.append(deppkg.component)
225
226         fail = False
227         for pkg in packages.itervalues():
228                 for spec in pkg.depends.split():
229                         if not Depend(spec).check(packages):
230                                 fail = True
231                                 print >>sys.stderr, 'Dependency', spec, \
232                                         'failed for', pkg.name
233
234                 for spec in pkg.conflicts.split():
235                         if Depend(spec).check(packages):
236                                 fail = True
237                                 print >>sys.stderr, pkg.name, \
238                                         'conflicts with', spec
239
240         if fail:
241                 raise Error('Invalid component tree')
242
243 class Depend(object):
244         regex = re.compile(r'([@]?)([^\s:]+)[:]?([<>=]*)([^\s:]*)[:]?(.*)')
245
246         def __init__(self, spec):
247                 match = self.regex.match(spec)
248                 if not match:
249                         raise Error('Bad dependency specification: ' + spec)
250
251                 self.build, self.name, self.tag_op, self.tag, flags \
252                         = match.groups()
253                 self.flags = flags.split()
254
255         def check(self, packages):
256                 # TODO: check version and flags
257                 return self.name in packages
258
259 def execute(args):
260         if config.debug:
261                 print 'Executing:', ' '.join(args)
262
263         if os.spawnvp(os.P_WAIT, args[0], args) != 0:
264                 raise Error('Failed: ' + ' '.join(args))
265
266 def download_component(c, overwrite=False):
267         if os.path.exists(c.name) and not overwrite:
268                 return
269
270         print 'Downloading component:', c.name
271
272         execute(['rm', '-rf', c.name])
273         git.clone(c.name, c.active_url, checkout=False)
274
275         branch = c.get_active_tag()
276         git.create_branch(c.name, branch, 'origin/%s'%branch, checkout=True, link=True,force=True)
277
278 def update_component_packages(c, targets, packages):
279         c.active_packages = {}
280
281         for path in glob.glob(os.path.join(c.name, '.matrix', '*.package')):
282                 name = os.path.basename(path)[:-8]
283
284                 pkg = parse_package(name, c, path)
285                 if not pkg:
286                         continue
287
288                 c.active_packages[name] = pkg
289                 packages[name] = pkg
290
291                 if config.debug:
292                         print 'Component', c.name, 'provides', name
293
294         if c.active_packages:
295                 targets.append(c.name)
296         elif config.debug:
297                 print 'Component', c.name, 'does not provide any packages'
298
299 class Package(object):
300         def __init__(self, name, component):
301                 self.name = name
302                 self.component = component
303
304                 self.depends = []
305                 self.conflicts = []
306                 self.architectures = None
307
308 def parse_package(name, component, path):
309         pkg = Package(name, component)
310         execfile(path, pkg.__dict__, pkg.__dict__)
311
312         if pkg.architectures:
313                 arch = config.boards[config.board].arch
314                 if arch not in pkg.architectures:
315                         return None
316
317         return pkg
318
319 def get_active_arch():
320         config.boards[config.board].arch
321
322 def build_components(targets, build_jobs=1, make_jobs=1):
323         selected = set([config.components[i] for i in targets])
324
325         depends = None
326         while depends is None or depends:
327                 depends = set()
328
329                 for c in selected:
330                         for dep in c.active_depends:
331                                 if dep not in selected:
332                                         depends.add(dep)
333
334                 selected |= depends
335
336         components = list(selected)
337         components.sort()
338
339         if config.debug:
340                 print 'Building components:'
341                 for c in components:
342                         print '\t' + c.name
343
344         jobs = {}
345
346         while True:
347                 found = False
348                 for c in components:
349                         if len(jobs) >= build_jobs:
350                                 break
351
352                         if component_buildable(c):
353                                 found = True
354
355                                 if not component_cached(c):
356                                         print 'Starting to build', c.name
357                                         start_job(c, jobs, make_jobs)
358                                 else:
359                                         print c.name, 'found from cache'
360                                         c.active_state = 'built'
361
362                 if not found and not jobs:
363                         for c in components:
364                                 if c.active_state != 'built':
365                                         raise Error('Internal error')
366                         break
367
368                 wait_for_job(jobs)
369
370 def component_buildable(c):
371         if c.active_state:
372                 if config.debug:
373                         print c.name, 'is in state:', c.active_state
374                 return False
375
376         for dep in c.active_depends:
377                 if config.components[dep.name].active_state != 'built':
378                         if config.debug:
379                                 print c.name, 'depends on unbuilt', dep.name
380                         return False
381
382         return True
383
384 def component_cached(c):
385         if c.from_platform:
386                 return True
387
388         if git.ls_files(c.name, ['-m', '-d']):
389                 raise Error('Component contains dirty files: ' + c.name)
390
391         path = os.path.join(config.cache_dir, c.name)
392         if not os.path.exists(path):
393                 return False
394
395         file = open(path, 'r')
396         line = file.readline()
397         file.close()
398
399         match = re.match(r'([^\s]+)[\s]?([^\s]*)', line)
400         if not match:
401                 return False
402
403         hash, flagstring = match.groups()
404
405         if hash != c.get_active_hash():
406                 print 'Component has been changed:', c.name
407                 return False
408
409         if flagstring:
410                 flags = flagstring.split(',')
411         else:
412                 flags = []
413
414         if set(flags) != set(c.flags):
415                 print 'Component flags have been changed:', c.name
416                 return False
417
418         return True
419
420 def update_cache(c):
421         path = os.path.join(config.cache_dir, c.name)
422
423         dir = os.path.dirname(path)
424         if not os.path.exists(dir):
425                 os.makedirs(dir)
426
427         file = open(path, 'w')
428         print >>file, c.get_active_hash(), ','.join(c.flags)
429         file.close()
430
431 def start_job(c, jobs, make_jobs):
432         board = config.boards[config.board]
433
434         makefile = os.path.join(config.top_dir, 'scripts', 'matrix.mak')
435
436         args = ['make', '--no-print-directory', '-f', makefile, '-C', c.name,
437                 '-j', str(make_jobs), 'build_matrix_component',
438                 'MATRIX_TOPDIR='    + config.top_dir,
439                 'MATRIX_COMPONENT=' + c.name,
440                 'MATRIX_ARCH='      + board.arch,
441                 'MATRIX_GCC_MARCH=' + board.gcc_march,
442                 'MATRIX_GCC_MCPU='  + board.gcc_mcpu,
443                 'MATRIX_GCC_MFPU='  + board.gcc_mfpu,
444                 'MATRIX_GCC_OPTIONS=' + board.gcc_options,
445                 'MATRIX_GNU_HOST='  + board.gnu_host,
446                 'MATRIX_LIBC='      + config.libc]
447
448         for flag in config.flags:
449                 args.append(flag + '=1')
450
451         if config.verbose:
452                 args.append('MATRIX_VERBOSE=1')
453
454         if config.debug:
455                 print 'Executing:', ' '.join(args)
456
457         pid = os.spawnvp(os.P_NOWAIT, args[0], args)
458
459         jobs[pid] = c
460         c.active_state = 'running'
461
462 def wait_for_job(jobs):
463         if not jobs:
464                 return
465
466         pid, status = os.wait()
467
468         c = jobs.get(pid)
469         if not c:
470                 return
471
472         del jobs[pid]
473
474         if status == 0:
475                 c.active_state = 'built'
476                 update_cache(c)
477                 print c.name, 'completed'
478         else:
479                 c.active_state = 'error'
480                 print >>sys.stderr, c.name, 'failed with status:', status
481
482                 wait_for_job(jobs)
483                 raise Error('Build failed')
484
485 def clean_components(targets):
486         if not targets:
487                 targets = config.components.keys()
488
489         for name in targets:
490                 print 'Cleaning component:', name
491
492                 files = git.ls_files(name, ['-o'])
493                 paths = [os.path.join(name, i) for i in files]
494                 paths.sort()
495                 paths.reverse()
496
497                 cache = os.path.join(config.cache_dir, name)
498                 if os.path.exists(cache):
499                         paths.append(cache)
500
501                 for path in paths:
502                         if config.debug:
503                                 print 'Deleting', path
504
505                         if os.path.islink(path) or not os.path.isdir(path):
506                                 os.remove(path)
507                         else:
508                                 os.rmdir(path)
509
510                 files = git.ls_files(name, ['-m', '-d'])
511                 if files:
512                         print len(files), 'dirty files left in', name
513
514 def pull_components(targets):
515         if not targets:
516                 targets = config.components.keys()
517
518         for name in targets:
519                 print 'Pulling component:', name
520                 c = config.components[name]
521                 git.pull(name)
522
523 def source_dist_components(targets):
524         if not targets:
525                 targets = config.components.keys()
526
527         for name in targets:
528                 c = config.components[name]
529                 generate_component_changes(c, 'dist')
530                 package_component_sources(c, 'dist')
531
532 def generate_component_changes(c, location):
533         print 'Generating component change log:', c.name
534
535         path = os.path.join(location, c.name) + '.changes'
536
537         pathdir = os.path.dirname(path)
538         if not os.path.exists(pathdir):
539                 os.makedirs(pathdir)
540
541         fd = os.open(path, os.O_WRONLY | os.O_CREAT, 0644)
542         git.log(c.name, [c.get_active_tag()], fd=fd)
543         os.close(fd)
544
545 def package_component_sources(c, location):
546         print 'Packaging component sources:', c.name
547
548         rev=git.describe(c.name)
549         if rev:
550                 rev = '_'+rev
551         else:
552                 rev = ''
553         path = os.path.join(location, c.name) + rev + '.tar.bz2'
554         if os.path.exists(path):
555                 os.remove(path)
556
557         pathdir = os.path.dirname(path)
558         if not os.path.exists(pathdir):
559                 os.makedirs(pathdir)
560
561         git.archive(c.name,path,
562                                 prefix=os.path.basename(c.name)+'/',
563                                 branch=c.get_active_tag())
564
565 def build_rootfs(clean=True, rootfs_only=False, jffs2_only=False, devrootfs_only=False):
566         parse_config('rootfs')
567
568         f = os.popen('sb-conf cu')
569         target = f.read()
570         f.close()
571
572         rootfs = RootFS(target.strip())
573         rootfs.include_paths(config.include_paths)
574         rootfs.include_files(config.include_files)
575         rootfs.filter_paths(config.exclude_paths)
576         rootfs.filter_files(config.exclude_files)
577         rootfs.filter_expressions(config.exclude_expressions)
578         rootfs.add_paths(config.created_paths)
579         rootfs.set_devices(config.devices)
580         rootfs.set_change_owner(config.change_owner)
581         rootfs.set_erase_size(config.boards[config.board].flash_erase_size)
582         rootfs.set_pad_size(config.boards[config.board].flash_pad_size)
583
584         if rootfs_only:
585                 rootfs.generate(clean, build_target="rootfs")
586         elif jffs2_only:
587                 rootfs.generate(clean, build_target="jffs2")
588         elif devrootfs_only:
589                 rootfs.generate(clean, build_target="devrootfs")
590         else:
591                 rootfs.generate(clean, build_target="all")