move git repository existence check out of getvar
[matrix.git] / matrix / matrix.py
1 # Copyright (C) 2006-2008 Movial Oy
2 # Authors: Timo Savola <tsavola@movial.fi>
3 #          Toni Timonen
4 #          Kalle Vahlman <kalle.vahlman@movial.fi>
5 #          Tuomas Kulve <tuomas.kulve@movial.fi>
6
7 import glob
8 import os
9 import re
10 import sys
11 import tarfile
12 from sets import Set as set
13
14 import config
15 import git
16 from rootfs import RootFS
17
18 Error = RuntimeError
19
20 def main():
21         parse_config('config.local')
22
23         command, params, targets = parse_args(sys.argv)
24
25         parse_config('config')
26         parse_config('boards')
27         parse_config('components')
28
29         if 'global-cache' in config.flags:
30                 config.cache_dir = config.global_cache_dir
31
32         if not os.path.exists(config.cache_dir):
33                 os.makedirs(config.cache_dir)
34
35         all_targets = update_components()
36
37         for name in targets:
38                 if name not in config.components:
39                         raise Error('Component "%s" does not exist' % name)
40
41         if not targets:
42                 targets = all_targets
43
44         if command == 'install':
45                 build_components(targets, **params)
46         elif command == 'download':
47                 # Actual downloading took place already in update_components()
48                 pass
49         elif command == 'clean':
50                 clean_components(targets)
51         elif command == 'pull':
52                 pull_components(targets)
53         elif command == 'source-dist':
54                 source_dist_components(targets)
55         elif command == 'rootfs':
56                 build_rootfs(**params)
57         else:
58                 raise Error('Invalid command: ' + command)
59
60 def help(file, args):
61         print >>file, '''
62 Copyright (C) 2006-2008 Movial Oy
63
64 Usage: %(progname)s [options] [command] [params] [component1 component2 ...]
65
66 Components must be given like this: core/glibc core/busybox X/xserver
67 If no components are given, all of them will be subject to the operation.
68
69 Options:
70         -v              Verbose output
71         -d              Debug output
72         -r URL          Specify the root for component git repos to clone from.
73                         If this option is not given, the default specified in
74                         the components file will be used.  This option may be
75                         specified multiple times.
76         -f              Build components with dirty files.
77         -h, --help      Print this help text.
78
79 Commands and their parameters:
80
81         download        Only download the components and validate the tree.
82         install         Download, build and install the components.
83                 -j N            Execute N build jobs in parallel.
84                 -mj N           Run component builds with make -j N.
85         clean           Remove all non-tracked files from the component git
86                         repository directories.
87         pull            Run git pull for the components.
88         source-dist     Download and package the component sources.
89         rootfs          Build a rootfs from the current target installation.
90                 -n              Do not clean env.faked and stripped directory.
91                 -r              Only generate a stripped rootfs.
92                 -j              Only generate a jffs2 image
93                 -d              Only generate a rootstrap (development rootfs).
94         list [parameter] [pattern]
95                 -p              List packages.
96                 -pv             List packages with full details.
97                 -c              List components (default).
98                 -cdep           List components and their dependencies.
99 ''' % {'progname': args[0]}
100
101 def parse_args(args):
102         def parse_jobs(arg):
103                 jobs = int(arg)
104                 if jobs <= 0:
105                         raise Error('Please specify a valid number of jobs')
106                 return jobs
107
108         command = None
109         params = {}
110         targets = []
111
112         i = 1
113         while i < len(args):
114                 if args[i].startswith('-'):
115                         if not command:
116                                 if args[i] == '-v':
117                                         config.verbose = True
118                                 elif args[i] == '-d':
119                                         config.debug = True
120                                 elif args[i] == '-r':
121                                         i += 1
122                                         config.roots.append(args[i])
123                                 elif args[i] == '-f':
124                                         config.force = True
125                                 elif args[i] in ('-h', '--help'):
126                                         help(sys.stdout, args)
127                                         sys.exit(0)
128                                 else:
129                                         raise Error('Bad option: ' + args[i])
130                         elif command == 'install':
131                                 if args[i] == '-j':
132                                         i += 1
133                                         params['build_jobs'] = parse_jobs(args[i])
134                                 elif args[i] == '-mj':
135                                         i += 1
136                                         params['make_jobs'] = parse_jobs(args[i])
137                                 else:
138                                         raise Error('Bad parameter: ' + args[i])
139                         elif command == 'rootfs':
140                                 if args[i] == '-n':
141                                         i += 1
142                                         params['clean'] = False
143                                 elif args[i] == '-r':
144                                         i += 1
145                                         params['rootfs_only'] = True
146                                 elif args[i] == '-j':
147                                         i += 1
148                                         params['jffs2_only'] = True
149                                 elif args[i] == '-d':
150                                         i += 1
151                                         params['devrootfs_only'] = True
152                         else:
153                                 raise Error('Command takes no parameters')
154                 elif not command:
155                         command = args[i]
156                 else:
157                         targets.append(args[i])
158                 i += 1
159
160         if not command:
161                 help(sys.stderr, args)
162                 sys.exit(1)
163
164         return command, params, targets
165
166 def parse_config(path):
167         if os.path.isabs(path):
168                 path = os.path.join(config.top_dir, path)
169
170         if os.path.exists(path):
171                 print 'Reading', path
172                 execfile(path, config.__dict__, config.__dict__)
173
174 def update_components():
175         targets = []
176         packages = {}
177
178         for c in config.components.itervalues():
179                 update_component_url(c)
180                 download_component(c)
181                 update_component_packages(c, targets, packages)
182                 c.active_depends = []
183
184         update_component_depends(packages)
185
186         return targets
187
188 def update_component_url(c):
189         update_repository_url(c.repo)
190         update_repository_url(c.meta)
191
192 def update_repository_url(repo):
193         if git.contains_database(repo.path):
194                 url = git.getvar(repo.path, 'remote.origin.url')
195                 if url:
196                         repo.active_url = url
197                         if config.debug:
198                                 print 'Using', repo.active_url
199                         return
200
201         for root in config.roots:
202                 for suffix in ('.git', ''):
203                         url = '%s/%s%s' % (root, repo.name, suffix)
204
205                         if config.debug:
206                                 print 'Trying', url
207
208                         if not git.peek_remote(url, quiet=True):
209                                 continue
210
211                         repo.active_url = url
212                         if config.debug:
213                                 print 'Found', repo.active_url
214                         return
215
216         raise Error('Failed to locate repository: ' + repo.name)
217
218 def update_component_depends(packages):
219         for pkg in packages.itervalues():
220                 for spec in pkg.depends.split():
221                         depname = Depend(spec).name
222                         deppkg = packages.get(depname)
223
224                         if not deppkg:
225                                 print >>sys.stderr, 'Package', pkg.name, \
226                                         'depends on non-existent package', depname
227                                 continue
228
229                         if deppkg.component == pkg.component:
230                                 continue
231
232                         pkg.component.active_depends.append(deppkg.component)
233
234         fail = False
235         for pkg in packages.itervalues():
236                 for spec in pkg.depends.split():
237                         if not Depend(spec).check(packages):
238                                 fail = True
239                                 print >>sys.stderr, 'Dependency', spec, \
240                                         'failed for', pkg.name
241
242                 for spec in pkg.conflicts.split():
243                         if Depend(spec).check(packages):
244                                 fail = True
245                                 print >>sys.stderr, pkg.name, \
246                                         'conflicts with', spec
247
248         if fail:
249                 raise Error('Invalid component tree')
250
251 class Depend(object):
252         regex = re.compile(r'([@]?)([^\s:]+)[:]?([<>=]*)([^\s:]*)[:]?(.*)')
253
254         def __init__(self, spec):
255                 match = self.regex.match(spec)
256                 if not match:
257                         raise Error('Bad dependency specification: ' + spec)
258
259                 self.build, self.name, self.tag_op, self.tag, flags \
260                         = match.groups()
261                 self.flags = flags.split()
262
263         def check(self, packages):
264                 # TODO: check version and flags
265                 return self.name in packages
266
267 def execute(args):
268         if config.debug:
269                 print 'Executing:', ' '.join(args)
270
271         if os.spawnvp(os.P_WAIT, args[0], args) != 0:
272                 raise Error('Failed: ' + ' '.join(args))
273
274 def download_component(c, overwrite=False):
275         if os.path.exists(c.repo.path) and not overwrite:
276                 return
277
278         print 'Downloading component:', c.name
279
280         execute(['rm', '-rf', c.repo.path])
281         download_repository(c, c.repo)
282         download_repository(c, c.meta)
283
284 def download_repository(c, repo):
285         if config.debug:
286                 print 'Downloading repository:', repo.name
287
288         git.clone(repo.path, repo.active_url, checkout=False)
289
290         branch = c.get_active_tag()
291         git.create_branch(repo.path, branch, 'origin/%s' % branch,
292                           checkout=True, link=True, force=True)
293
294 def update_component_packages(c, targets, packages):
295         c.active_packages = {}
296
297         for path in glob.glob(os.path.join(c.meta.path, '*.package')):
298                 name = os.path.basename(path)[:-8]
299
300                 pkg = parse_package(name, c, path)
301                 if not pkg:
302                         continue
303
304                 c.active_packages[name] = pkg
305                 packages[name] = pkg
306
307                 if config.debug:
308                         print 'Component', c.name, 'provides', name
309
310         if c.active_packages:
311                 targets.append(c.name)
312         elif config.debug:
313                 print 'Component', c.name, 'does not provide any packages'
314
315 class Package(object):
316         def __init__(self, name, component):
317                 self.name = name
318                 self.component = component
319
320                 self.depends = []
321                 self.conflicts = []
322                 self.architectures = None
323
324 def parse_package(name, component, path):
325         pkg = Package(name, component)
326         execfile(path, pkg.__dict__, pkg.__dict__)
327
328         if pkg.architectures:
329                 arch = config.boards[config.board].arch
330                 if arch not in pkg.architectures:
331                         return None
332
333         return pkg
334
335 def build_components(targets, build_jobs=1, make_jobs=1):
336         selected = set([config.components[i] for i in targets])
337
338         depends = None
339         while depends is None or depends:
340                 depends = set()
341
342                 for c in selected:
343                         for dep in c.active_depends:
344                                 if dep not in selected:
345                                         depends.add(dep)
346
347                 selected |= depends
348
349         components = list(selected)
350         components.sort()
351
352         if config.debug:
353                 print 'Building components:'
354                 for c in components:
355                         print '\t' + c.name
356
357         jobs = {}
358
359         while True:
360                 found = False
361                 for c in components:
362                         if len(jobs) >= build_jobs:
363                                 break
364
365                         if component_buildable(c):
366                                 found = True
367
368                                 if not component_cached(c):
369                                         print 'Starting to build', c.name
370                                         start_job(c, jobs, make_jobs)
371                                 else:
372                                         print c.name, 'found from cache'
373                                         c.active_state = 'built'
374
375                 if not found and not jobs:
376                         for c in components:
377                                 if c.active_state != 'built':
378                                         raise Error('Internal error')
379                         break
380
381                 wait_for_job(jobs)
382
383 def component_buildable(c):
384         if c.active_state:
385                 if config.debug:
386                         print c.name, 'is in state:', c.active_state
387                 return False
388
389         for dep in c.active_depends:
390                 if config.components[dep.name].active_state != 'built':
391                         if config.debug:
392                                 print c.name, 'depends on unbuilt', dep.name
393                         return False
394
395         return True
396
397 def component_cached(c):
398         if c.from_platform:
399                 return True
400
401         for repo in (c.repo, c.meta):
402                 if git.ls_files(repo.path, ['-m', '-d']):
403                         if config.force:
404                                 return False
405                         else:
406                                 raise Error('Dirty files in ' + repo.path)
407
408         path = os.path.join(config.cache_dir, c.name)
409         if not os.path.exists(path):
410                 return False
411
412         file = open(path, 'r')
413         line = file.readline()
414         file.close()
415
416         match = re.match(r'([^\s]+)[\s]?([^\s]*)', line)
417         if not match:
418                 return False
419
420         hash, flagstring = match.groups()
421
422         if hash != get_component_hash(c):
423                 print 'Component has been changed:', c.name
424                 return False
425
426         if flagstring:
427                 flags = flagstring.split(',')
428         else:
429                 flags = []
430
431         if set(flags) != set(c.flags):
432                 print 'Component flags have been changed:', c.name
433                 return False
434
435         return True
436
437 def update_cache(c):
438         path = os.path.join(config.cache_dir, c.name)
439
440         dir = os.path.dirname(path)
441         if not os.path.exists(dir):
442                 os.makedirs(dir)
443
444         file = open(path, 'w')
445         done = False
446         try:
447                 print >>file, get_component_hash(c.repo), ','.join(c.flags)
448                 done = True
449         finally:
450                 file.close()
451                 if not done:
452                         os.remove(path)
453
454 def get_component_hash(c):
455         return '%s+%s' % (get_repository_hash(c.repo), get_repository_hash(c.meta))
456
457 def get_repository_hash(repo):
458         if repo.active_hash is None:
459                 repo.active_hash = git.rev_parse(repo.path, 'HEAD')
460         return repo.active_hash
461
462 def start_job(c, jobs, make_jobs):
463         board = config.boards[config.board]
464
465         makefile = os.path.join(config.script_dir, 'matrix.mak')
466
467         workdir = os.path.join(config.top_dir, 'src', c.name)
468         args = ['make', '--no-print-directory', '-f', makefile, '-C', workdir,
469                 '-j', str(make_jobs), 'build_matrix_component',
470                 'MATRIX_TOPDIR='    + os.path.abspath(config.top_dir),
471                 'MATRIX_SCRIPTDIR=' + os.path.abspath(config.script_dir),
472                 'MATRIX_COMPONENT=' + c.name,
473                 'MATRIX_ARCH='      + board.arch,
474                 'MATRIX_GCC_MARCH=' + board.gcc_march,
475                 'MATRIX_GCC_MCPU='  + board.gcc_mcpu,
476                 'MATRIX_GCC_MFPU='  + board.gcc_mfpu,
477                 'MATRIX_GCC_OPTIONS=' + board.gcc_options,
478                 'MATRIX_GNU_HOST='  + board.gnu_host,
479                 'MATRIX_LIBC='      + config.libc]
480
481         for flag in config.flags:
482                 args.append(flag + '=1')
483
484         if config.verbose:
485                 args.append('MATRIX_VERBOSE=1')
486
487         if config.debug:
488                 print 'Executing:', ' '.join(args)
489
490         pid = os.spawnvp(os.P_NOWAIT, args[0], args)
491
492         jobs[pid] = c
493         c.active_state = 'running'
494
495 def wait_for_job(jobs):
496         if not jobs:
497                 return
498
499         pid, status = os.wait()
500
501         c = jobs.get(pid)
502         if not c:
503                 return
504
505         del jobs[pid]
506
507         if status == 0:
508                 c.active_state = 'built'
509                 update_cache(c)
510                 print c.name, 'completed'
511         else:
512                 c.active_state = 'error'
513                 print >>sys.stderr, c.name, 'failed with status:', status
514
515                 wait_for_job(jobs)
516                 raise Error('Build failed')
517
518 def clean_components(targets):
519         if not targets:
520                 targets = config.components.keys()
521
522         for name in targets:
523                 print 'Cleaning component:', name
524
525                 c = config.components[name]
526
527                 paths = []
528                 for repo in (c.repo, c.meta):
529                         files = git.ls_files(repo.path, ['-o'])
530                         paths += [os.path.join(repo.path, i) for i in files]
531
532                 paths.sort()
533                 paths.reverse()
534
535                 cache = os.path.join(config.cache_dir, name)
536                 if os.path.exists(cache):
537                         paths.append(cache)
538
539                 for path in paths:
540                         if config.debug:
541                                 print 'Deleting', path
542
543                         if os.path.islink(path) or not os.path.isdir(path):
544                                 os.remove(path)
545                         elif not os.path.exists(os.path.join(path, '.git')):
546                                 os.rmdir(path)
547
548                 for repo in (c.repo, c.meta):
549                         files = git.ls_files(repo.path, ['-m', '-d'])
550                         if files:
551                                 print len(files), 'dirty files left in', repo.path
552
553 def pull_components(targets):
554         if not targets:
555                 targets = config.components.keys()
556
557         for name in targets:
558                 print 'Pulling component:', name
559
560                 c = config.components[name]
561                 git.pull(c.repo.path)
562                 git.pull(c.meta.path)
563
564 def source_dist_components(targets):
565         if not targets:
566                 targets = config.components.keys()
567
568         for name in targets:
569                 c = config.components[name]
570                 generate_component_changes(c, 'dist')
571                 package_component_sources(c, 'dist')
572
573 def generate_component_changes(c, location):
574         print 'Generating component change log:', c.name
575
576         path = os.path.join(location, c.name) + '.changes'
577
578         pathdir = os.path.dirname(path)
579         if not os.path.exists(pathdir):
580                 os.makedirs(pathdir)
581
582         fd = os.open(path, os.O_WRONLY | os.O_CREAT, 0644)
583         git.log(c.name, [c.get_active_tag()], fd=fd)
584         os.close(fd)
585
586 def package_component_sources(c, location):
587         print 'Packaging component sources:', c.name
588
589         rev=git.describe(c.name)
590         if rev:
591                 rev = '_'+rev
592         else:
593                 rev = ''
594         path = os.path.join(location, c.name) + rev + '.tar.bz2'
595         if os.path.exists(path):
596                 os.remove(path)
597
598         pathdir = os.path.dirname(path)
599         if not os.path.exists(pathdir):
600                 os.makedirs(pathdir)
601
602         git.archive(c.name,path,
603                                 prefix=os.path.basename(c.name)+'/',
604                                 branch=c.get_active_tag())
605
606 def build_rootfs(clean=True, rootfs_only=False, jffs2_only=False, devrootfs_only=False):
607         parse_config('rootfs')
608
609         f = os.popen('sb-conf cu')
610         target = f.read()
611         f.close()
612
613         rootfs = RootFS(target.strip())
614         rootfs.include_paths(config.include_paths)
615         rootfs.include_files(config.include_files)
616         rootfs.filter_paths(config.exclude_paths)
617         rootfs.filter_files(config.exclude_files)
618         rootfs.filter_expressions(config.exclude_expressions)
619         rootfs.add_paths(config.created_paths)
620         rootfs.set_devices(config.devices)
621         rootfs.set_change_owner(config.change_owner)
622         rootfs.set_erase_size(config.boards[config.board].flash_erase_size)
623         rootfs.set_pad_size(config.boards[config.board].flash_pad_size)
624
625         if rootfs_only:
626                 rootfs.generate(clean, build_target="rootfs")
627         elif jffs2_only:
628                 rootfs.generate(clean, build_target="jffs2")
629         elif devrootfs_only:
630                 rootfs.generate(clean, build_target="devrootfs")
631         else:
632                 rootfs.generate(clean, build_target="all")