use MATRIX_SCRIPTDIR in scripts
[matrix.git] / matrix / matrix.py
1 # Copyright (C) 2006-2008 Movial Oy
2 # Authors: Timo Savola <tsavola@movial.fi>
3 #          Toni Timonen
4 #          Kalle Vahlman <kalle.vahlman@movial.fi>
5 #          Tuomas Kulve <tuomas.kulve@movial.fi>
6
7 import glob
8 import os
9 import re
10 import sys
11 import tarfile
12 from sets import Set as set
13
14 import config
15 import git
16 from rootfs import RootFS
17
18 Error = RuntimeError
19
20 def main():
21         parse_config('config.local')
22
23         command, params, targets = parse_args(sys.argv)
24
25         parse_config('config')
26         parse_config('boards')
27         parse_config('components/components')
28
29         if 'global-cache' in config.flags:
30                 config.cache_dir = config.global_cache_dir
31
32         if not os.path.exists(config.cache_dir):
33                 os.makedirs(config.cache_dir)
34
35         all_targets = update_components()
36
37         for name in targets:
38                 if name not in config.components:
39                         raise Error('Component "%s" does not exist' % name)
40
41         if not targets:
42                 targets = all_targets
43
44         if command == 'install':
45                 build_components(targets, **params)
46         elif command == 'download':
47                 # Actual downloading took place already in update_components()
48                 pass
49         elif command == 'clean':
50                 clean_components(targets)
51         elif command == 'pull':
52                 pull_components(targets)
53         elif command == 'source-dist':
54                 source_dist_components(targets)
55         elif command == 'rootfs':
56                 build_rootfs(**params)
57         else:
58                 raise Error('Invalid command: ' + command)
59
60 def help(file, args):
61         print >>file, '''
62 Copyright (C) 2006-2008 Movial Oy
63
64 Usage: %(progname)s [options] [command] [params] [component1 component2 ...]
65
66 Components must be given like this: core/glibc core/busybox X/xserver
67 If no components are given, all of them will be subject to the operation.
68
69 Options:
70         -v              Verbose output
71         -d              Debug output
72         -r URL          Specify the root for component git repos to clone from.
73                         If this option is not given, the default specified in
74                         the components file will be used.  This option may be
75                         specified multiple times.
76         -h, --help      Print this help text.
77
78 Commands and their parameters:
79
80         download        Only download the components and validate the tree.
81         install         Download, build and install the components.
82                 -j N            Execute N build jobs in parallel.
83                 -mj N           Run component builds with make -j N.
84         clean           Remove all non-tracked files from the component git
85                         repository directories.
86         pull            Run git pull for the components.
87         source-dist     Download and package the component sources.
88         rootfs          Build a rootfs from the current target installation.
89                 -n              Do not clean env.faked and stripped directory.
90                 -r              Only generate a stripped rootfs.
91                 -j              Only generate a jffs2 image
92                 -d              Only generate a rootstrap (development rootfs).
93         list [parameter] [pattern]
94                 -p              List packages.
95                 -pv             List packages with full details.
96                 -c              List components (default).
97                 -cdep           List components and their dependencies.
98 ''' % {'progname': args[0]}
99
100 def parse_args(args):
101         def parse_jobs(arg):
102                 jobs = int(arg)
103                 if jobs <= 0:
104                         raise Error('Please specify a valid number of jobs')
105                 return jobs
106
107         command = None
108         params = {}
109         targets = []
110
111         i = 1
112         while i < len(args):
113                 if args[i].startswith('-'):
114                         if not command:
115                                 if args[i] == '-v':
116                                         config.verbose = True
117                                 elif args[i] == '-d':
118                                         config.debug = True
119                                 elif args[i] == '-r':
120                                         i += 1
121                                         config.roots.append(args[i])
122                                 elif args[i] in ('-h', '--help'):
123                                         help(sys.stdout, args)
124                                         sys.exit(0)
125                                 else:
126                                         raise Error('Bad option: ' + args[i])
127                         elif command == 'install':
128                                 if args[i] == '-j':
129                                         i += 1
130                                         params['build_jobs'] = parse_jobs(args[i])
131                                 elif args[i] == '-mj':
132                                         i += 1
133                                         params['make_jobs'] = parse_jobs(args[i])
134                                 else:
135                                         raise Error('Bad parameter: ' + args[i])
136                         elif command == 'rootfs':
137                                 if args[i] == '-n':
138                                         i += 1
139                                         params['clean'] = False
140                                 elif args[i] == '-r':
141                                         i += 1
142                                         params['rootfs_only'] = True
143                                 elif args[i] == '-j':
144                                         i += 1
145                                         params['jffs2_only'] = True
146                                 elif args[i] == '-d':
147                                         i += 1
148                                         params['devrootfs_only'] = True
149                         else:
150                                 raise Error('Command takes no parameters')
151                 elif not command:
152                         command = args[i]
153                 else:
154                         targets.append(args[i])
155                 i += 1
156
157         if not command:
158                 help(sys.stderr, args)
159                 sys.exit(1)
160
161         return command, params, targets
162
163 def parse_config(path):
164         if os.path.isabs(path):
165                 path = os.path.join(config.top_dir, path)
166
167         if os.path.exists(path):
168                 print 'Reading', path
169                 execfile(path, config.__dict__, config.__dict__)
170
171 def update_components():
172         targets = []
173         packages = {}
174
175         for c in config.components.itervalues():
176                 update_component_url(c)
177                 download_component(c)
178                 update_component_packages(c, targets, packages)
179                 c.active_depends = []
180
181         update_component_depends(packages)
182
183         return targets
184
185 def update_component_url(c):
186         url=git.getvar(c.name,'remote.origin.url')
187         if url:
188                 c.active_url=url
189                 if config.debug:
190                         print 'Using', c.active_url
191                 return
192
193         for root in config.roots:
194                 for suffix in ('.git', ''):
195                         url = root + '/' + c.name + suffix
196
197                         if config.debug:
198                                 print 'Trying', url
199
200                         if not git.peek_remote(url, quiet=True):
201                                 continue
202
203                         c.active_url = url
204                         if config.debug:
205                                 print 'Found', c.active_url
206                         return
207
208         raise Error('Failed to locate repository of ' + c.name)
209
210 def update_component_depends(packages):
211         for pkg in packages.itervalues():
212                 for spec in pkg.depends.split():
213                         depname = Depend(spec).name
214                         deppkg = packages.get(depname)
215
216                         if not deppkg:
217                                 print >>sys.stderr, 'Package', pkg.name, \
218                                         'depends on non-existent package', depname
219                                 continue
220
221                         if deppkg.component == pkg.component:
222                                 continue
223
224                         pkg.component.active_depends.append(deppkg.component)
225
226         fail = False
227         for pkg in packages.itervalues():
228                 for spec in pkg.depends.split():
229                         if not Depend(spec).check(packages):
230                                 fail = True
231                                 print >>sys.stderr, 'Dependency', spec, \
232                                         'failed for', pkg.name
233
234                 for spec in pkg.conflicts.split():
235                         if Depend(spec).check(packages):
236                                 fail = True
237                                 print >>sys.stderr, pkg.name, \
238                                         'conflicts with', spec
239
240         if fail:
241                 raise Error('Invalid component tree')
242
243 class Depend(object):
244         regex = re.compile(r'([@]?)([^\s:]+)[:]?([<>=]*)([^\s:]*)[:]?(.*)')
245
246         def __init__(self, spec):
247                 match = self.regex.match(spec)
248                 if not match:
249                         raise Error('Bad dependency specification: ' + spec)
250
251                 self.build, self.name, self.tag_op, self.tag, flags \
252                         = match.groups()
253                 self.flags = flags.split()
254
255         def check(self, packages):
256                 # TODO: check version and flags
257                 return self.name in packages
258
259 def execute(args):
260         if config.debug:
261                 print 'Executing:', ' '.join(args)
262
263         if os.spawnvp(os.P_WAIT, args[0], args) != 0:
264                 raise Error('Failed: ' + ' '.join(args))
265
266 def download_component(c, overwrite=False):
267         if os.path.exists(c.name) and not overwrite:
268                 return
269
270         print 'Downloading component:', c.name
271
272         execute(['rm', '-rf', c.name])
273         git.clone(c.name, c.active_url, checkout=False)
274
275         branch = c.get_active_tag()
276         git.create_branch(c.name, branch, 'origin/%s'%branch, checkout=True, link=True,force=True)
277
278 def update_component_packages(c, targets, packages):
279         c.active_packages = {}
280
281         for path in glob.glob(os.path.join(c.name, '.matrix', '*.package')):
282                 name = os.path.basename(path)[:-8]
283
284                 pkg = parse_package(name, c, path)
285                 if not pkg:
286                         continue
287
288                 c.active_packages[name] = pkg
289                 packages[name] = pkg
290
291                 if config.debug:
292                         print 'Component', c.name, 'provides', name
293
294         if c.active_packages:
295                 targets.append(c.name)
296         elif config.debug:
297                 print 'Component', c.name, 'does not provide any packages'
298
299 class Package(object):
300         def __init__(self, name, component):
301                 self.name = name
302                 self.component = component
303
304                 self.depends = []
305                 self.conflicts = []
306                 self.architectures = None
307
308 def parse_package(name, component, path):
309         pkg = Package(name, component)
310         execfile(path, pkg.__dict__, pkg.__dict__)
311
312         if pkg.architectures:
313                 arch = config.boards[config.board].arch
314                 if arch not in pkg.architectures:
315                         return None
316
317         return pkg
318
319 def build_components(targets, build_jobs=1, make_jobs=1):
320         selected = set([config.components[i] for i in targets])
321
322         depends = None
323         while depends is None or depends:
324                 depends = set()
325
326                 for c in selected:
327                         for dep in c.active_depends:
328                                 if dep not in selected:
329                                         depends.add(dep)
330
331                 selected |= depends
332
333         components = list(selected)
334         components.sort()
335
336         if config.debug:
337                 print 'Building components:'
338                 for c in components:
339                         print '\t' + c.name
340
341         jobs = {}
342
343         while True:
344                 found = False
345                 for c in components:
346                         if len(jobs) >= build_jobs:
347                                 break
348
349                         if component_buildable(c):
350                                 found = True
351
352                                 if not component_cached(c):
353                                         print 'Starting to build', c.name
354                                         start_job(c, jobs, make_jobs)
355                                 else:
356                                         print c.name, 'found from cache'
357                                         c.active_state = 'built'
358
359                 if not found and not jobs:
360                         for c in components:
361                                 if c.active_state != 'built':
362                                         raise Error('Internal error')
363                         break
364
365                 wait_for_job(jobs)
366
367 def component_buildable(c):
368         if c.active_state:
369                 if config.debug:
370                         print c.name, 'is in state:', c.active_state
371                 return False
372
373         for dep in c.active_depends:
374                 if config.components[dep.name].active_state != 'built':
375                         if config.debug:
376                                 print c.name, 'depends on unbuilt', dep.name
377                         return False
378
379         return True
380
381 def component_cached(c):
382         if c.from_platform:
383                 return True
384
385         if git.ls_files(c.name, ['-m', '-d']):
386                 raise Error('Component contains dirty files: ' + c.name)
387
388         path = os.path.join(config.cache_dir, c.name)
389         if not os.path.exists(path):
390                 return False
391
392         file = open(path, 'r')
393         line = file.readline()
394         file.close()
395
396         match = re.match(r'([^\s]+)[\s]?([^\s]*)', line)
397         if not match:
398                 return False
399
400         hash, flagstring = match.groups()
401
402         if hash != c.get_active_hash():
403                 print 'Component has been changed:', c.name
404                 return False
405
406         if flagstring:
407                 flags = flagstring.split(',')
408         else:
409                 flags = []
410
411         if set(flags) != set(c.flags):
412                 print 'Component flags have been changed:', c.name
413                 return False
414
415         return True
416
417 def update_cache(c):
418         path = os.path.join(config.cache_dir, c.name)
419
420         dir = os.path.dirname(path)
421         if not os.path.exists(dir):
422                 os.makedirs(dir)
423
424         file = open(path, 'w')
425         print >>file, c.get_active_hash(), ','.join(c.flags)
426         file.close()
427
428 def start_job(c, jobs, make_jobs):
429         board = config.boards[config.board]
430
431         makefile = os.path.join(config.script_dir, 'matrix.mak')
432
433         args = ['make', '--no-print-directory', '-f', makefile, '-C', c.name,
434                 '-j', str(make_jobs), 'build_matrix_component',
435                 'MATRIX_TOPDIR='    + config.top_dir,
436                 'MATRIX_SCRIPTDIR=' + config.script_dir,
437                 'MATRIX_COMPONENT=' + c.name,
438                 'MATRIX_ARCH='      + board.arch,
439                 'MATRIX_GCC_MARCH=' + board.gcc_march,
440                 'MATRIX_GCC_MCPU='  + board.gcc_mcpu,
441                 'MATRIX_GCC_MFPU='  + board.gcc_mfpu,
442                 'MATRIX_GCC_OPTIONS=' + board.gcc_options,
443                 'MATRIX_GNU_HOST='  + board.gnu_host,
444                 'MATRIX_LIBC='      + config.libc]
445
446         for flag in config.flags:
447                 args.append(flag + '=1')
448
449         if config.verbose:
450                 args.append('MATRIX_VERBOSE=1')
451
452         if config.debug:
453                 print 'Executing:', ' '.join(args)
454
455         pid = os.spawnvp(os.P_NOWAIT, args[0], args)
456
457         jobs[pid] = c
458         c.active_state = 'running'
459
460 def wait_for_job(jobs):
461         if not jobs:
462                 return
463
464         pid, status = os.wait()
465
466         c = jobs.get(pid)
467         if not c:
468                 return
469
470         del jobs[pid]
471
472         if status == 0:
473                 c.active_state = 'built'
474                 update_cache(c)
475                 print c.name, 'completed'
476         else:
477                 c.active_state = 'error'
478                 print >>sys.stderr, c.name, 'failed with status:', status
479
480                 wait_for_job(jobs)
481                 raise Error('Build failed')
482
483 def clean_components(targets):
484         if not targets:
485                 targets = config.components.keys()
486
487         for name in targets:
488                 print 'Cleaning component:', name
489
490                 files = git.ls_files(name, ['-o'])
491                 paths = [os.path.join(name, i) for i in files]
492                 paths.sort()
493                 paths.reverse()
494
495                 cache = os.path.join(config.cache_dir, name)
496                 if os.path.exists(cache):
497                         paths.append(cache)
498
499                 for path in paths:
500                         if config.debug:
501                                 print 'Deleting', path
502
503                         if os.path.islink(path) or not os.path.isdir(path):
504                                 os.remove(path)
505                         else:
506                                 os.rmdir(path)
507
508                 files = git.ls_files(name, ['-m', '-d'])
509                 if files:
510                         print len(files), 'dirty files left in', name
511
512 def pull_components(targets):
513         if not targets:
514                 targets = config.components.keys()
515
516         for name in targets:
517                 print 'Pulling component:', name
518                 c = config.components[name]
519                 git.pull(name)
520
521 def source_dist_components(targets):
522         if not targets:
523                 targets = config.components.keys()
524
525         for name in targets:
526                 c = config.components[name]
527                 generate_component_changes(c, 'dist')
528                 package_component_sources(c, 'dist')
529
530 def generate_component_changes(c, location):
531         print 'Generating component change log:', c.name
532
533         path = os.path.join(location, c.name) + '.changes'
534
535         pathdir = os.path.dirname(path)
536         if not os.path.exists(pathdir):
537                 os.makedirs(pathdir)
538
539         fd = os.open(path, os.O_WRONLY | os.O_CREAT, 0644)
540         git.log(c.name, [c.get_active_tag()], fd=fd)
541         os.close(fd)
542
543 def package_component_sources(c, location):
544         print 'Packaging component sources:', c.name
545
546         rev=git.describe(c.name)
547         if rev:
548                 rev = '_'+rev
549         else:
550                 rev = ''
551         path = os.path.join(location, c.name) + rev + '.tar.bz2'
552         if os.path.exists(path):
553                 os.remove(path)
554
555         pathdir = os.path.dirname(path)
556         if not os.path.exists(pathdir):
557                 os.makedirs(pathdir)
558
559         git.archive(c.name,path,
560                                 prefix=os.path.basename(c.name)+'/',
561                                 branch=c.get_active_tag())
562
563 def build_rootfs(clean=True, rootfs_only=False, jffs2_only=False, devrootfs_only=False):
564         parse_config('rootfs')
565
566         f = os.popen('sb-conf cu')
567         target = f.read()
568         f.close()
569
570         rootfs = RootFS(target.strip())
571         rootfs.include_paths(config.include_paths)
572         rootfs.include_files(config.include_files)
573         rootfs.filter_paths(config.exclude_paths)
574         rootfs.filter_files(config.exclude_files)
575         rootfs.filter_expressions(config.exclude_expressions)
576         rootfs.add_paths(config.created_paths)
577         rootfs.set_devices(config.devices)
578         rootfs.set_change_owner(config.change_owner)
579         rootfs.set_erase_size(config.boards[config.board].flash_erase_size)
580         rootfs.set_pad_size(config.boards[config.board].flash_pad_size)
581
582         if rootfs_only:
583                 rootfs.generate(clean, build_target="rootfs")
584         elif jffs2_only:
585                 rootfs.generate(clean, build_target="jffs2")
586         elif devrootfs_only:
587                 rootfs.generate(clean, build_target="devrootfs")
588         else:
589                 rootfs.generate(clean, build_target="all")