build module
[matrix.git] / matrix / matrix.py
1 # Copyright (C) 2006-2008 Movial Oy
2 # Authors: Timo Savola <tsavola@movial.fi>
3 #          Toni Timonen
4 #          Kalle Vahlman <kalle.vahlman@movial.fi>
5 #          Tuomas Kulve <tuomas.kulve@movial.fi>
6
7 import glob
8 import os
9 import re
10 import sys
11 import tarfile
12 from sets import Set as set
13
14 import cache
15 import config
16 import git
17 import log
18 from build import Builder
19 from rootfs import RootFS
20
21 Error = RuntimeError
22
23 def main():
24         parse_config('config.local')
25
26         command, params, targets = parse_args(sys.argv)
27
28         parse_config('config')
29         parse_config('boards')
30         parse_config('components')
31
32         if 'global-cache' in config.flags:
33                 config.cache_dir = config.global_cache_dir
34
35         if not os.path.exists(config.cache_dir):
36                 os.makedirs(config.cache_dir)
37
38         all_targets = update_components()
39
40         for name in targets:
41                 if name not in config.components:
42                         raise Error('Component "%s" does not exist' % name)
43
44         if not targets:
45                 targets = all_targets
46
47         if command == 'install':
48                 build_components(targets, **params)
49         elif command == 'clone':
50                 clone_components(targets)
51         elif command == 'clean':
52                 clean_components(targets)
53         elif command == 'rebase':
54                 rebase_components(targets)
55         elif command == 'pull':
56                 pull_components(targets)
57         elif command == 'source-dist':
58                 source_dist_components(targets)
59         elif command == 'rootfs':
60                 build_rootfs(**params)
61         else:
62                 raise Error('Invalid command: ' + command)
63
64 def help(file, args):
65         print >>file, '''
66 Copyright (C) 2006-2008 Movial Oy
67
68 Usage: %(progname)s [<options>] <command> [<params>] [<components>]
69
70 If no components are specified, all of them will be targeted.  All component
71 metadata will be downloaded regardless of the specified command and components.
72
73 Options:
74         -v              Verbose output.
75         -d              Debug output.
76         -r URL          Specify the root for component git repos to clone from.
77                         If this option is not given, the roots specified in the
78                         components file will be used.  This option may be
79                         specified multiple times.
80         -f              Build components with dirty files.
81         -h, --help      Print this help text.
82
83 Commands and parameters:
84         clone           Download the components' git repositories.
85         install         Download, build and install the components.
86                 -j N            Execute N build jobs in parallel.
87                 -mj N           Run component builds with make -j N.
88         clean           Remove all non-tracked files from the component git
89                         repository directories.
90         rebase          Update repositories from server by rebasing.
91         pull            Update repositories from server by merging.
92         source-dist     Download and package the component sources.
93         rootfs          Build a rootfs from the current target installation.
94                 -n              Do not clean env.faked and stripped directory.
95                 -r              Only generate a stripped rootfs.
96                 -j              Only generate a jffs2 image
97                 -d              Only generate a rootstrap (development rootfs).
98 ''' % {'progname': args[0]}
99
100 def parse_args(args):
101         def parse_jobs(arg):
102                 jobs = int(arg)
103                 if jobs <= 0:
104                         raise Error('Please specify a valid number of jobs')
105                 return jobs
106
107         command = None
108         params = {}
109         targets = []
110
111         i = 1
112         while i < len(args):
113                 if args[i].startswith('-'):
114                         if not command:
115                                 if args[i] == '-v':
116                                         config.verbose = True
117                                 elif args[i] == '-d':
118                                         config.debug = True
119                                 elif args[i] == '-r':
120                                         i += 1
121                                         config.roots.append(args[i])
122                                 elif args[i] == '-f':
123                                         config.force = True
124                                 elif args[i] in ('-h', '--help'):
125                                         help(sys.stdout, args)
126                                         sys.exit(0)
127                                 else:
128                                         raise Error('Bad option: ' + args[i])
129                         elif command == 'install':
130                                 if args[i] == '-j':
131                                         i += 1
132                                         params['build_jobs'] = parse_jobs(args[i])
133                                 elif args[i] == '-mj':
134                                         i += 1
135                                         params['make_jobs'] = parse_jobs(args[i])
136                                 else:
137                                         raise Error('Bad parameter: ' + args[i])
138                         elif command == 'rootfs':
139                                 if args[i] == '-n':
140                                         i += 1
141                                         params['clean'] = False
142                                 elif args[i] == '-r':
143                                         i += 1
144                                         params['rootfs_only'] = True
145                                 elif args[i] == '-j':
146                                         i += 1
147                                         params['jffs2_only'] = True
148                                 elif args[i] == '-d':
149                                         i += 1
150                                         params['devrootfs_only'] = True
151                         else:
152                                 raise Error('Command takes no parameters')
153                 elif not command:
154                         command = args[i]
155                 else:
156                         targets.append(args[i])
157                 i += 1
158
159         if not command:
160                 help(sys.stderr, args)
161                 sys.exit(1)
162
163         return command, params, targets
164
165 def parse_config(path):
166         if os.path.isabs(path):
167                 path = os.path.join(config.top_dir, path)
168
169         if os.path.exists(path):
170                 print 'Reading', path
171                 execfile(path, config.__dict__, config.__dict__)
172
173 def update_components():
174         targets = []
175         packages = {}
176
177         for c in config.components.itervalues():
178                 update_component_url(c)
179                 clone_metadata(c)
180                 update_component_packages(c, targets, packages)
181                 c.active_depends = []
182
183         update_component_depends(packages)
184
185         return targets
186
187 def update_component_url(c):
188         update_repository_url(c.repo)
189         update_repository_url(c.meta)
190
191 def update_repository_url(repo):
192         if git.contains_database(repo.path):
193                 url = git.getvar(repo.path, 'remote.origin.url')
194                 if url:
195                         repo.active_url = url
196                         if config.debug:
197                                 print 'Using', repo.active_url
198                         return
199
200         for root in config.roots:
201                 for suffix in ('.git', ''):
202                         url = '%s/%s%s' % (root, repo.name, suffix)
203
204                         if config.debug:
205                                 print 'Trying', url
206
207                         if not git.peek_remote(url, quiet=True):
208                                 continue
209
210                         repo.active_url = url
211                         if config.debug:
212                                 print 'Found', repo.active_url
213                         return
214
215         raise Error('Failed to locate repository: ' + repo.name)
216
217 def update_component_depends(packages):
218         for pkg in packages.itervalues():
219                 for spec in pkg.depends.split():
220                         depname = Depend(spec).name
221                         deppkg = packages.get(depname)
222
223                         if not deppkg:
224                                 log.error('Package %s depends on ' \
225                                           'non-existent package %s', \
226                                           pkg.name, depname)
227                                 continue
228
229                         if deppkg.component == pkg.component:
230                                 continue
231
232                         pkg.component.active_depends.append(deppkg.component)
233
234         fail = False
235         for pkg in packages.itervalues():
236                 for spec in pkg.depends.split():
237                         if not Depend(spec).check(packages):
238                                 fail = True
239                                 log.error('Dependency %s failed for %s' % \
240                                           (spec, pkg.name))
241
242                 for spec in pkg.conflicts.split():
243                         if Depend(spec).check(packages):
244                                 fail = True
245                                 log.error('Package %s conflicts with %s' % \
246                                           (pkg.name, spec))
247
248         if fail:
249                 raise Error('Invalid component tree')
250
251 class Depend(object):
252         regex = re.compile(r'([@]?)([^\s:]+)[:]?([<>=]*)([^\s:]*)[:]?(.*)')
253
254         def __init__(self, spec):
255                 match = self.regex.match(spec)
256                 if not match:
257                         raise Error('Bad dependency specification: ' + spec)
258
259                 self.build, self.name, self.tag_op, self.tag, flags \
260                         = match.groups()
261                 self.flags = flags.split()
262
263         def check(self, packages):
264                 # TODO: check version and flags
265                 return self.name in packages
266
267 def execute(args):
268         if config.debug:
269                 print 'Executing:', ' '.join(args)
270
271         if os.spawnvp(os.P_WAIT, args[0], args) != 0:
272                 raise Error('Failed: ' + ' '.join(args))
273
274 def remove_tree(path):
275         execute(['rm', '-rf', path])
276
277 def clone_components(targets):
278         if not targets:
279                 targets = config.components.keys()
280
281         for name in targets:
282                 c = config.components[name]
283                 clone_component(c)
284
285 def clone_component(c, overwrite=False):
286         have_repo = git.contains_database(c.repo.path)
287         have_meta = git.contains_database(c.meta.path)
288
289         if not overwrite and have_repo and have_meta:
290                 return
291
292         if overwrite and os.path.exists(c.repo.path):
293                 print 'Removing', c.repo.path
294
295                 remove_tree(c.repo.path)
296                 have_repo = False
297                 have_meta = False
298
299         if not have_repo:
300                 clone_repository(c.repo)
301                 git.exclude(c.repo.path, 'meta')
302
303         if not have_meta:
304                 clone_repository(c.meta)
305
306 def clone_metadata(c):
307         if not git.contains_database(c.meta.path):
308                 clone_repository(c.meta)
309
310 def clone_repository(repo):
311         print 'Cloning', repo.path
312
313         if os.path.exists(repo.path):
314                 tmp = os.path.join(repo.path, 'tmp')
315                 git.clone(tmp, repo.active_url, checkout=False)
316                 try:
317                         tmpdb = os.path.join(tmp, '.git')
318                         repodb = os.path.join(repo.path, '.git')
319                         if config.debug:
320                                 print 'Renaming "%s" as "%s"' % (tmpdb, repodb)
321                         os.rename(tmpdb, repodb)
322                 finally:
323                         os.rmdir(tmp)
324                 git.checkout(repo.path)
325         else:
326                 git.clone(repo.path, repo.active_url, checkout=True)
327
328 def update_component_packages(c, targets, packages):
329         c.active_packages = {}
330
331         for path in glob.glob(os.path.join(c.meta.path, '*.package')):
332                 name = os.path.basename(path)[:-8]
333
334                 pkg = parse_package(name, c, path)
335                 if not pkg:
336                         continue
337
338                 c.active_packages[name] = pkg
339                 packages[name] = pkg
340
341                 if config.debug:
342                         print 'Component', c.name, 'provides', name
343
344         if c.active_packages:
345                 targets.append(c.name)
346         elif config.debug:
347                 print 'Component', c.name, 'does not provide any packages'
348
349 class Package(object):
350         def __init__(self, name, component):
351                 self.name = name
352                 self.component = component
353
354                 self.depends = []
355                 self.conflicts = []
356                 self.architectures = None
357
358 def parse_package(name, component, path):
359         pkg = Package(name, component)
360         execfile(path, pkg.__dict__, pkg.__dict__)
361
362         if pkg.architectures:
363                 arch = config.boards[config.board].arch
364                 if arch not in pkg.architectures:
365                         return None
366
367         return pkg
368
369 def build_components(targets, build_jobs=1, make_jobs=1):
370         selected = set([config.components[i] for i in targets])
371
372         depends = None
373         while depends is None or depends:
374                 depends = set()
375
376                 for c in selected:
377                         for dep in c.active_depends:
378                                 if dep not in selected:
379                                         depends.add(dep)
380
381                 selected |= depends
382
383         components = list(selected)
384         components.sort()
385
386         if config.debug:
387                 print 'Building components:'
388                 for c in components:
389                         print '\t' + c.name
390
391         builder = Builder(components, build_jobs, make_jobs)
392         builder.run()
393
394 def clean_components(targets):
395         if not targets:
396                 targets = config.components.keys()
397
398         for name in targets:
399                 c = config.components[name]
400                 print 'Cleaning', c.repo.path
401
402                 cache.remove(c)
403
404                 files = git.ls_files(c.repo.path, ['-o'])
405                 paths = [os.path.join(c.repo.path, i) for i in files]
406                 paths.sort()
407                 paths.reverse()
408
409                 for path in paths:
410                         if git.contains_database(path):
411                                 continue
412
413                         if config.debug:
414                                 print 'Removing', path
415
416                         if os.path.islink(path) or not os.path.isdir(path):
417                                 os.remove(path)
418                         else:
419                                 remove_tree(path)
420
421                 for repo in (c.repo, c.meta):
422                         files = git.ls_files(repo.path, ['-m', '-d'])
423                         if files:
424                                 log.error('Dirty files left in %s' % repo.path)
425
426 def for_each_repository(func, targets=None):
427         if not targets:
428                 targets = config.components.keys()
429
430         for name in targets:
431                 c = config.components[name]
432                 if git.contains_database(c.repo.path):
433                         func(c.repo)
434                 func(c.meta)
435
436 def rebase_components(targets):
437         for_each_repository(rebase_repository, targets)
438
439 def pull_components(targets):
440         for_each_repository(pull_repository, targets)
441
442 def rebase_repository(repo):
443         print 'Rebasing', repo.path
444         git.remote_update(repo.path)
445         git.rebase(repo.path)
446
447 def pull_repository(repo):
448         print 'Pulling', repo.path
449         git.pull(repo.path)
450
451 def source_dist_components(targets):
452         if not targets:
453                 targets = config.components.keys()
454
455         for name in targets:
456                 c = config.components[name]
457                 generate_component_changes(c, 'dist')
458                 package_component_sources(c, 'dist')
459
460 def generate_component_changes(c, location):
461         print 'Generating change log for', c.repo.path
462
463         path = os.path.join(location, c.name) + '.changes'
464
465         pathdir = os.path.dirname(path)
466         if not os.path.exists(pathdir):
467                 os.makedirs(pathdir)
468
469         fd = os.open(path, os.O_WRONLY | os.O_CREAT, 0644)
470         git.log(c.repo.path, [c.get_active_tag()], fd=fd)
471         os.close(fd)
472
473 def package_component_sources(c, location):
474         print 'Archiving', c.repo.path
475
476         rev = git.describe(c.repo.path)
477         if rev:
478                 rev = '_' + rev
479         else:
480                 rev = ''
481
482         path = os.path.join(location, c.name) + rev + '.tar.bz2'
483         if os.path.exists(path):
484                 os.remove(path)
485
486         pathdir = os.path.dirname(path)
487         if not os.path.exists(pathdir):
488                 os.makedirs(pathdir)
489
490         git.archive(c.repo.path, path,
491                     prefix=os.path.basename(c.name) + '/',
492                     branch=c.get_active_tag())
493
494 def build_rootfs(clean=True, rootfs_only=False, jffs2_only=False, devrootfs_only=False):
495         parse_config('rootfs')
496
497         f = os.popen('sb-conf cu')
498         target = f.read()
499         f.close()
500
501         rootfs = RootFS(target.strip())
502         rootfs.include_paths(config.include_paths)
503         rootfs.include_files(config.include_files)
504         rootfs.filter_paths(config.exclude_paths)
505         rootfs.filter_files(config.exclude_files)
506         rootfs.filter_expressions(config.exclude_expressions)
507         rootfs.add_paths(config.created_paths)
508         rootfs.set_devices(config.devices)
509         rootfs.set_change_owner(config.change_owner)
510         rootfs.set_erase_size(config.boards[config.board].flash_erase_size)
511         rootfs.set_pad_size(config.boards[config.board].flash_pad_size)
512
513         if rootfs_only:
514                 rootfs.generate(clean, build_target="rootfs")
515         elif jffs2_only:
516                 rootfs.generate(clean, build_target="jffs2")
517         elif devrootfs_only:
518                 rootfs.generate(clean, build_target="devrootfs")
519         else:
520                 rootfs.generate(clean, build_target="all")