use git reset to check out specified commit
[matrix.git] / matrix / matrix.py
1 # Copyright (C) 2006-2008 Movial Oy
2 # Authors: Timo Savola <tsavola@movial.fi>
3 #          Toni Timonen
4 #          Kalle Vahlman <kalle.vahlman@movial.fi>
5 #          Tuomas Kulve <tuomas.kulve@movial.fi>
6
7 import glob
8 import os
9 import re
10 import sys
11 import tarfile
12 from sets import Set as set
13
14 import cache
15 import config
16 import git
17 import log
18 from build import Builder
19 from rootfs import RootFS
20
21 Error = RuntimeError
22
23 def main():
24         parse_config('config.local')
25
26         command, params, targets = parse_args(sys.argv)
27
28         parse_config('config')
29         parse_config('boards')
30         parse_config('components')
31
32         if 'global-cache' in config.flags:
33                 config.cache_dir = config.global_cache_dir
34
35         if not os.path.exists(config.cache_dir):
36                 os.makedirs(config.cache_dir)
37
38         all_targets = update_components()
39
40         for name in targets:
41                 if name not in config.components:
42                         raise Error('Component "%s" does not exist' % name)
43
44         if not targets:
45                 targets = all_targets
46
47         if command == 'install':
48                 build_components(targets, **params)
49         elif command == 'clone':
50                 clone_components(targets)
51         elif command == 'clean':
52                 clean_components(targets)
53         elif command == 'rebase':
54                 rebase_components(targets)
55         elif command == 'pull':
56                 pull_components(targets)
57         elif command == 'source-dist':
58                 source_dist_components(targets)
59         elif command == 'rootfs':
60                 build_rootfs(**params)
61         else:
62                 raise Error('Invalid command: ' + command)
63
64 def help(file, args):
65         print >>file, '''
66 Copyright (C) 2006-2008 Movial Oy
67
68 Usage: %(progname)s [<options>] <command> [<params>] [<components>]
69
70 If no components are specified, all of them will be targeted.  All component
71 metadata will be downloaded regardless of the specified command and components.
72
73 Options:
74         -v              Verbose output.
75         -d              Debug output.
76         -r URL          Specify the root for component git repos to clone from.
77                         If this option is not given, the roots specified in the
78                         components file will be used.  This option may be
79                         specified multiple times.
80         -f              Build components with dirty files.
81         -h, --help      Print this help text.
82
83 Commands and parameters:
84         clone           Download the components' git repositories.
85         install         Download, build and install the components.
86                 -j N            Execute N build jobs in parallel.
87                 -mj N           Run component builds with make -j N.
88         clean           Remove all non-tracked files from the component git
89                         repository directories.
90         rebase          Update repositories from server by rebasing.
91         pull            Update repositories from server by merging.
92         source-dist     Download and package the component sources.
93         rootfs          Build a rootfs from the current target installation.
94                 -n              Do not clean env.faked and stripped directory.
95                 -r              Only generate a stripped rootfs.
96                 -j              Only generate a jffs2 image
97                 -d              Only generate a rootstrap (development rootfs).
98 ''' % {'progname': args[0]}
99
100 def parse_args(args):
101         def parse_jobs(arg):
102                 jobs = int(arg)
103                 if jobs <= 0:
104                         raise Error('Please specify a valid number of jobs')
105                 return jobs
106
107         command = None
108         params = {}
109         targets = []
110
111         i = 1
112         while i < len(args):
113                 if args[i].startswith('-'):
114                         if not command:
115                                 if args[i] == '-v':
116                                         config.verbose = True
117                                 elif args[i] == '-d':
118                                         config.debug = True
119                                 elif args[i] == '-r':
120                                         i += 1
121                                         config.roots.append(args[i])
122                                 elif args[i] == '-f':
123                                         config.force = True
124                                 elif args[i] in ('-h', '--help'):
125                                         help(sys.stdout, args)
126                                         sys.exit(0)
127                                 else:
128                                         raise Error('Bad option: ' + args[i])
129                         elif command == 'install':
130                                 if args[i] == '-j':
131                                         i += 1
132                                         params['build_jobs'] = parse_jobs(args[i])
133                                 elif args[i] == '-mj':
134                                         i += 1
135                                         params['make_jobs'] = parse_jobs(args[i])
136                                 else:
137                                         raise Error('Bad parameter: ' + args[i])
138                         elif command == 'rootfs':
139                                 if args[i] == '-n':
140                                         i += 1
141                                         params['clean'] = False
142                                 elif args[i] == '-r':
143                                         i += 1
144                                         params['rootfs_only'] = True
145                                 elif args[i] == '-j':
146                                         i += 1
147                                         params['jffs2_only'] = True
148                                 elif args[i] == '-d':
149                                         i += 1
150                                         params['devrootfs_only'] = True
151                         else:
152                                 raise Error('Command takes no parameters')
153                 elif not command:
154                         command = args[i]
155                 else:
156                         targets.append(args[i])
157                 i += 1
158
159         if not command:
160                 help(sys.stderr, args)
161                 sys.exit(1)
162
163         return command, params, targets
164
165 def parse_config(path):
166         if os.path.isabs(path):
167                 path = os.path.join(config.top_dir, path)
168
169         if os.path.exists(path):
170                 print 'Reading', path
171                 execfile(path, config.__dict__, config.__dict__)
172
173 def update_components():
174         targets = []
175         packages = {}
176
177         for c in config.components.itervalues():
178                 update_component_url(c)
179                 clone_metadata(c)
180                 update_component_packages(c, targets, packages)
181                 c.active_depends = []
182
183         update_component_depends(packages)
184
185         return targets
186
187 def update_component_url(c):
188         update_repository_url(c.repo)
189         update_repository_url(c.meta)
190
191 def update_repository_url(repo):
192         if git.contains_database(repo.path):
193                 url = git.getvar(repo.path, 'remote.origin.url')
194                 if url:
195                         repo.active_url = url
196                         if config.debug:
197                                 print 'Using', repo.active_url
198                         return
199
200         for root in config.roots:
201                 for suffix in ('.git', ''):
202                         url = '%s/%s%s' % (root, repo.name, suffix)
203
204                         if config.debug:
205                                 print 'Trying', url
206
207                         if not git.peek_remote(url, quiet=True):
208                                 continue
209
210                         repo.active_url = url
211                         if config.debug:
212                                 print 'Found', repo.active_url
213                         return
214
215         raise Error('Failed to locate repository: ' + repo.name)
216
217 def update_component_depends(packages):
218         for pkg in packages.itervalues():
219                 for spec in pkg.depends.split():
220                         depname = Depend(spec).name
221                         deppkg = packages.get(depname)
222
223                         if not deppkg:
224                                 log.error('Package %s depends on ' \
225                                           'non-existent package %s' % \
226                                           (pkg.name, depname))
227                                 continue
228
229                         if deppkg.component == pkg.component:
230                                 continue
231
232                         pkg.component.active_depends.append(deppkg.component)
233
234         fail = False
235         for pkg in packages.itervalues():
236                 for spec in pkg.depends.split():
237                         if not Depend(spec).check(packages):
238                                 fail = True
239                                 log.error('Dependency %s failed for %s' % \
240                                           (spec, pkg.name))
241
242                 for spec in pkg.conflicts.split():
243                         if Depend(spec).check(packages):
244                                 fail = True
245                                 log.error('Package %s conflicts with %s' % \
246                                           (pkg.name, spec))
247
248         if fail:
249                 raise Error('Invalid component tree')
250
251 class Depend(object):
252         regex = re.compile(r'([@]?)([^\s:]+)[:]?([<>=]*)([^\s:]*)[:]?(.*)')
253
254         def __init__(self, spec):
255                 match = self.regex.match(spec)
256                 if not match:
257                         raise Error('Bad dependency specification: ' + spec)
258
259                 self.build, self.name, self.tag_op, self.tag, flags \
260                         = match.groups()
261                 self.flags = flags.split()
262
263         def check(self, packages):
264                 # TODO: check version and flags
265                 return self.name in packages
266
267 def execute(args):
268         if config.debug:
269                 print 'Executing:', ' '.join(args)
270
271         if os.spawnvp(os.P_WAIT, args[0], args) != 0:
272                 raise Error('Failed: ' + ' '.join(args))
273
274 def remove_tree(path):
275         execute(['rm', '-rf', path])
276
277 def clone_components(targets):
278         if not targets:
279                 targets = config.components.keys()
280
281         for name in targets:
282                 c = config.components[name]
283                 clone_component(c)
284
285 def clone_component(c, overwrite=False):
286         have_repo = git.contains_database(c.repo.path)
287         have_meta = git.contains_database(c.meta.path)
288
289         if not overwrite and have_repo and have_meta:
290                 return
291
292         if overwrite and os.path.exists(c.repo.path):
293                 print 'Removing', c.repo.path
294
295                 remove_tree(c.repo.path)
296                 have_repo = False
297                 have_meta = False
298
299         if not have_repo:
300                 clone_repository(c.repo)
301                 git.exclude(c.repo.path, 'meta')
302
303         if not have_meta:
304                 clone_repository(c.meta)
305
306 def clone_metadata(c):
307         if not git.contains_database(c.meta.path):
308                 clone_repository(c.meta)
309
310 def clone_repository(repo):
311         print 'Cloning', repo.path
312
313         if os.path.exists(repo.path):
314                 tmp = os.path.join(repo.path, 'tmp')
315                 git.clone(tmp, repo.active_url, checkout=False)
316                 try:
317                         tmpdb = os.path.join(tmp, '.git')
318                         repodb = os.path.join(repo.path, '.git')
319                         if config.debug:
320                                 print 'Renaming "%s" as "%s"' % (tmpdb, repodb)
321                         os.rename(tmpdb, repodb)
322                 finally:
323                         os.rmdir(tmp)
324         else:
325                 git.clone(repo.path, repo.active_url, checkout=False)
326
327         git.reset(repo.path, repo.get_commit(), hard=True)
328
329 def update_component_packages(c, targets, packages):
330         c.active_packages = {}
331
332         for path in glob.glob(os.path.join(c.meta.path, '*.package')):
333                 name = os.path.basename(path)[:-8]
334
335                 pkg = parse_package(name, c, path)
336                 if not pkg:
337                         continue
338
339                 c.active_packages[name] = pkg
340                 packages[name] = pkg
341
342                 if config.debug:
343                         print 'Component', c.name, 'provides', name
344
345         if c.active_packages:
346                 targets.append(c.name)
347         elif config.debug:
348                 print 'Component', c.name, 'does not provide any packages'
349
350 class Package(object):
351         def __init__(self, name, component):
352                 self.name = name
353                 self.component = component
354
355                 self.depends = []
356                 self.conflicts = []
357                 self.architectures = None
358
359 def parse_package(name, component, path):
360         pkg = Package(name, component)
361         execfile(path, pkg.__dict__, pkg.__dict__)
362
363         if pkg.architectures:
364                 arch = config.boards[config.board].arch
365                 if arch not in pkg.architectures:
366                         return None
367
368         return pkg
369
370 def build_components(targets, build_jobs=1, make_jobs=1):
371         selected = set([config.components[i] for i in targets])
372
373         depends = None
374         while depends is None or depends:
375                 depends = set()
376
377                 for c in selected:
378                         for dep in c.active_depends:
379                                 if dep not in selected:
380                                         depends.add(dep)
381
382                 selected |= depends
383
384         components = list(selected)
385         components.sort()
386
387         if config.debug:
388                 print 'Building components:'
389                 for c in components:
390                         print '\t' + c.name
391
392         builder = Builder(components, build_jobs, make_jobs)
393         builder.run()
394
395 def clean_components(targets):
396         if not targets:
397                 targets = config.components.keys()
398
399         for name in targets:
400                 c = config.components[name]
401                 print 'Cleaning', c.repo.path
402
403                 cache.remove(c)
404
405                 files = git.ls_files(c.repo.path, ['-o'])
406                 paths = [os.path.join(c.repo.path, i) for i in files]
407                 paths.sort()
408                 paths.reverse()
409
410                 for path in paths:
411                         if git.contains_database(path):
412                                 continue
413
414                         if config.debug:
415                                 print 'Removing', path
416
417                         if os.path.islink(path) or not os.path.isdir(path):
418                                 os.remove(path)
419                         else:
420                                 remove_tree(path)
421
422                 for repo in (c.repo, c.meta):
423                         files = git.ls_files(repo.path, ['-m', '-d'])
424                         if files:
425                                 log.error('Dirty files left in %s' % repo.path)
426
427 def for_each_repository(func, targets=None):
428         if not targets:
429                 targets = config.components.keys()
430
431         for name in targets:
432                 c = config.components[name]
433                 if git.contains_database(c.repo.path):
434                         func(c.repo)
435                 func(c.meta)
436
437 def rebase_components(targets):
438         for_each_repository(rebase_repository, targets)
439
440 def pull_components(targets):
441         for_each_repository(pull_repository, targets)
442
443 def rebase_repository(repo):
444         print 'Rebasing', repo.path
445         git.remote_update(repo.path)
446         git.rebase(repo.path)
447
448 def pull_repository(repo):
449         print 'Pulling', repo.path
450         git.pull(repo.path)
451
452 def source_dist_components(targets):
453         if not targets:
454                 targets = config.components.keys()
455
456         for name in targets:
457                 c = config.components[name]
458                 generate_component_changes(c, 'dist')
459                 package_component_sources(c, 'dist')
460
461 def generate_component_changes(c, location):
462         print 'Generating change log for', c.repo.path
463
464         path = os.path.join(location, c.name) + '.changes'
465
466         pathdir = os.path.dirname(path)
467         if not os.path.exists(pathdir):
468                 os.makedirs(pathdir)
469
470         fd = os.open(path, os.O_WRONLY | os.O_CREAT, 0644)
471         git.log(c.repo.path, [c.repo.get_commit()], fd=fd)
472         os.close(fd)
473
474 def package_component_sources(c, location):
475         print 'Archiving', c.repo.path
476
477         rev = git.describe(c.repo.path)
478         if rev:
479                 rev = '_' + rev
480         else:
481                 rev = ''
482
483         path = os.path.join(location, c.name) + rev + '.tar.bz2'
484         if os.path.exists(path):
485                 os.remove(path)
486
487         pathdir = os.path.dirname(path)
488         if not os.path.exists(pathdir):
489                 os.makedirs(pathdir)
490
491         git.archive(c.repo.path, path,
492                     prefix=os.path.basename(c.name) + '/',
493                     treeish=c.repo.get_commit())
494
495 def build_rootfs(clean=True, rootfs_only=False, jffs2_only=False, devrootfs_only=False):
496         parse_config('rootfs')
497
498         f = os.popen('sb-conf cu')
499         target = f.read()
500         f.close()
501
502         rootfs = RootFS(target.strip())
503         rootfs.include_paths(config.include_paths)
504         rootfs.include_files(config.include_files)
505         rootfs.filter_paths(config.exclude_paths)
506         rootfs.filter_files(config.exclude_files)
507         rootfs.filter_expressions(config.exclude_expressions)
508         rootfs.add_paths(config.created_paths)
509         rootfs.set_devices(config.devices)
510         rootfs.set_change_owner(config.change_owner)
511         rootfs.set_erase_size(config.boards[config.board].flash_erase_size)
512         rootfs.set_pad_size(config.boards[config.board].flash_pad_size)
513
514         if rootfs_only:
515                 rootfs.generate(clean, build_target="rootfs")
516         elif jffs2_only:
517                 rootfs.generate(clean, build_target="jffs2")
518         elif devrootfs_only:
519                 rootfs.generate(clean, build_target="devrootfs")
520         else:
521                 rootfs.generate(clean, build_target="all")