36ce2c83712da175c34808508dc8271b417e0191
[matrix.git] / matrix / matrix.py
1 # Copyright (C) 2006-2008 Movial Oy
2 # Authors: Timo Savola <tsavola@movial.fi>
3 #          Toni Timonen
4 #          Kalle Vahlman <kalle.vahlman@movial.fi>
5 #          Tuomas Kulve <tuomas.kulve@movial.fi>
6
7 import glob
8 import os
9 import re
10 import sys
11 import tarfile
12 from sets import Set as set
13
14 import config
15 import git
16 from rootfs import RootFS
17
18 Error = RuntimeError
19
20 def main():
21         parse_config('config.local')
22
23         command, params, targets = parse_args(sys.argv)
24
25         parse_config('config')
26         parse_config('boards')
27         parse_config('components')
28
29         if 'global-cache' in config.flags:
30                 config.cache_dir = config.global_cache_dir
31
32         if not os.path.exists(config.cache_dir):
33                 os.makedirs(config.cache_dir)
34
35         all_targets = update_components()
36
37         for name in targets:
38                 if name not in config.components:
39                         raise Error('Component "%s" does not exist' % name)
40
41         if not targets:
42                 targets = all_targets
43
44         if command == 'install':
45                 build_components(targets, **params)
46         elif command == 'clone':
47                 clone_components(targets)
48         elif command == 'clean':
49                 clean_components(targets)
50         elif command == 'pull':
51                 pull_components(targets)
52         elif command == 'source-dist':
53                 source_dist_components(targets)
54         elif command == 'rootfs':
55                 build_rootfs(**params)
56         else:
57                 raise Error('Invalid command: ' + command)
58
59 def help(file, args):
60         print >>file, '''
61 Copyright (C) 2006-2008 Movial Oy
62
63 Usage: %(progname)s [<options>] <command> [<params>] [<components>]
64
65 If no components are specified, all of them will be targeted.  All component
66 metadata will be downloaded regardless of the specified command and components.
67
68 Options:
69         -v              Verbose output.
70         -d              Debug output.
71         -r URL          Specify the root for component git repos to clone from.
72                         If this option is not given, the roots specified in the
73                         components file will be used.  This option may be
74                         specified multiple times.
75         -f              Build components with dirty files.
76         -h, --help      Print this help text.
77
78 Commands and parameters:
79         clone           Download the components' git repositories.
80         install         Download, build and install the components.
81                 -j N            Execute N build jobs in parallel.
82                 -mj N           Run component builds with make -j N.
83         clean           Remove all non-tracked files from the component git
84                         repository directories.
85         pull            Run git pull for the components.
86         source-dist     Download and package the component sources.
87         rootfs          Build a rootfs from the current target installation.
88                 -n              Do not clean env.faked and stripped directory.
89                 -r              Only generate a stripped rootfs.
90                 -j              Only generate a jffs2 image
91                 -d              Only generate a rootstrap (development rootfs).
92 ''' % {'progname': args[0]}
93
94 def parse_args(args):
95         def parse_jobs(arg):
96                 jobs = int(arg)
97                 if jobs <= 0:
98                         raise Error('Please specify a valid number of jobs')
99                 return jobs
100
101         command = None
102         params = {}
103         targets = []
104
105         i = 1
106         while i < len(args):
107                 if args[i].startswith('-'):
108                         if not command:
109                                 if args[i] == '-v':
110                                         config.verbose = True
111                                 elif args[i] == '-d':
112                                         config.debug = True
113                                 elif args[i] == '-r':
114                                         i += 1
115                                         config.roots.append(args[i])
116                                 elif args[i] == '-f':
117                                         config.force = True
118                                 elif args[i] in ('-h', '--help'):
119                                         help(sys.stdout, args)
120                                         sys.exit(0)
121                                 else:
122                                         raise Error('Bad option: ' + args[i])
123                         elif command == 'install':
124                                 if args[i] == '-j':
125                                         i += 1
126                                         params['build_jobs'] = parse_jobs(args[i])
127                                 elif args[i] == '-mj':
128                                         i += 1
129                                         params['make_jobs'] = parse_jobs(args[i])
130                                 else:
131                                         raise Error('Bad parameter: ' + args[i])
132                         elif command == 'rootfs':
133                                 if args[i] == '-n':
134                                         i += 1
135                                         params['clean'] = False
136                                 elif args[i] == '-r':
137                                         i += 1
138                                         params['rootfs_only'] = True
139                                 elif args[i] == '-j':
140                                         i += 1
141                                         params['jffs2_only'] = True
142                                 elif args[i] == '-d':
143                                         i += 1
144                                         params['devrootfs_only'] = True
145                         else:
146                                 raise Error('Command takes no parameters')
147                 elif not command:
148                         command = args[i]
149                 else:
150                         targets.append(args[i])
151                 i += 1
152
153         if not command:
154                 help(sys.stderr, args)
155                 sys.exit(1)
156
157         return command, params, targets
158
159 def parse_config(path):
160         if os.path.isabs(path):
161                 path = os.path.join(config.top_dir, path)
162
163         if os.path.exists(path):
164                 print 'Reading', path
165                 execfile(path, config.__dict__, config.__dict__)
166
167 def update_components():
168         targets = []
169         packages = {}
170
171         for c in config.components.itervalues():
172                 update_component_url(c)
173                 clone_metadata(c)
174                 update_component_packages(c, targets, packages)
175                 c.active_depends = []
176
177         update_component_depends(packages)
178
179         return targets
180
181 def update_component_url(c):
182         update_repository_url(c.repo)
183         update_repository_url(c.meta)
184
185 def update_repository_url(repo):
186         if git.contains_database(repo.path):
187                 url = git.getvar(repo.path, 'remote.origin.url')
188                 if url:
189                         repo.active_url = url
190                         if config.debug:
191                                 print 'Using', repo.active_url
192                         return
193
194         for root in config.roots:
195                 for suffix in ('.git', ''):
196                         url = '%s/%s%s' % (root, repo.name, suffix)
197
198                         if config.debug:
199                                 print 'Trying', url
200
201                         if not git.peek_remote(url, quiet=True):
202                                 continue
203
204                         repo.active_url = url
205                         if config.debug:
206                                 print 'Found', repo.active_url
207                         return
208
209         raise Error('Failed to locate repository: ' + repo.name)
210
211 def update_component_depends(packages):
212         for pkg in packages.itervalues():
213                 for spec in pkg.depends.split():
214                         depname = Depend(spec).name
215                         deppkg = packages.get(depname)
216
217                         if not deppkg:
218                                 print >>sys.stderr, 'Package', pkg.name, \
219                                         'depends on non-existent package', depname
220                                 continue
221
222                         if deppkg.component == pkg.component:
223                                 continue
224
225                         pkg.component.active_depends.append(deppkg.component)
226
227         fail = False
228         for pkg in packages.itervalues():
229                 for spec in pkg.depends.split():
230                         if not Depend(spec).check(packages):
231                                 fail = True
232                                 print >>sys.stderr, 'Dependency', spec, \
233                                         'failed for', pkg.name
234
235                 for spec in pkg.conflicts.split():
236                         if Depend(spec).check(packages):
237                                 fail = True
238                                 print >>sys.stderr, pkg.name, \
239                                         'conflicts with', spec
240
241         if fail:
242                 raise Error('Invalid component tree')
243
244 class Depend(object):
245         regex = re.compile(r'([@]?)([^\s:]+)[:]?([<>=]*)([^\s:]*)[:]?(.*)')
246
247         def __init__(self, spec):
248                 match = self.regex.match(spec)
249                 if not match:
250                         raise Error('Bad dependency specification: ' + spec)
251
252                 self.build, self.name, self.tag_op, self.tag, flags \
253                         = match.groups()
254                 self.flags = flags.split()
255
256         def check(self, packages):
257                 # TODO: check version and flags
258                 return self.name in packages
259
260 def execute(args):
261         if config.debug:
262                 print 'Executing:', ' '.join(args)
263
264         if os.spawnvp(os.P_WAIT, args[0], args) != 0:
265                 raise Error('Failed: ' + ' '.join(args))
266
267 def remove_tree(path):
268         execute(['rm', '-rf', path])
269
270 def clone_components(targets):
271         if not targets:
272                 targets = config.components.keys()
273
274         for name in targets:
275                 c = config.components[name]
276                 clone_component(c)
277
278 def clone_component(c, overwrite=False):
279         have_repo = git.contains_database(c.repo.path)
280         have_meta = git.contains_database(c.meta.path)
281
282         if not overwrite and have_repo and have_meta:
283                 return
284
285         if overwrite and os.path.exists(c.repo.path):
286                 print 'Removing', c.repo.path
287
288                 remove_tree(c.repo.path)
289                 have_repo = False
290                 have_meta = False
291
292         if not have_repo:
293                 clone_repository(c.repo)
294                 git.exclude(c.repo.path, 'meta')
295
296         if not have_meta:
297                 clone_repository(c.meta)
298
299 def clone_metadata(c):
300         if not git.contains_database(c.meta.path):
301                 clone_repository(c.meta)
302
303 def clone_repository(repo):
304         print 'Cloning', repo.path
305
306         if os.path.exists(repo.path):
307                 tmp = os.path.join(repo.path, 'tmp')
308                 git.clone(tmp, repo.active_url, checkout=False)
309                 try:
310                         tmpdb = os.path.join(tmp, '.git')
311                         repodb = os.path.join(repo.path, '.git')
312                         if config.debug:
313                                 print 'Renaming "%s" as "%s"' % (tmpdb, repodb)
314                         os.rename(tmpdb, repodb)
315                 finally:
316                         os.rmdir(tmp)
317                 git.checkout(repo.path)
318         else:
319                 git.clone(repo.path, repo.active_url, checkout=True)
320
321 def update_component_packages(c, targets, packages):
322         c.active_packages = {}
323
324         for path in glob.glob(os.path.join(c.meta.path, '*.package')):
325                 name = os.path.basename(path)[:-8]
326
327                 pkg = parse_package(name, c, path)
328                 if not pkg:
329                         continue
330
331                 c.active_packages[name] = pkg
332                 packages[name] = pkg
333
334                 if config.debug:
335                         print 'Component', c.name, 'provides', name
336
337         if c.active_packages:
338                 targets.append(c.name)
339         elif config.debug:
340                 print 'Component', c.name, 'does not provide any packages'
341
342 class Package(object):
343         def __init__(self, name, component):
344                 self.name = name
345                 self.component = component
346
347                 self.depends = []
348                 self.conflicts = []
349                 self.architectures = None
350
351 def parse_package(name, component, path):
352         pkg = Package(name, component)
353         execfile(path, pkg.__dict__, pkg.__dict__)
354
355         if pkg.architectures:
356                 arch = config.boards[config.board].arch
357                 if arch not in pkg.architectures:
358                         return None
359
360         return pkg
361
362 def build_components(targets, build_jobs=1, make_jobs=1):
363         selected = set([config.components[i] for i in targets])
364
365         depends = None
366         while depends is None or depends:
367                 depends = set()
368
369                 for c in selected:
370                         for dep in c.active_depends:
371                                 if dep not in selected:
372                                         depends.add(dep)
373
374                 selected |= depends
375
376         components = list(selected)
377         components.sort()
378
379         if config.debug:
380                 print 'Building components:'
381                 for c in components:
382                         print '\t' + c.name
383
384         jobs = {}
385
386         while True:
387                 found = False
388                 for c in components:
389                         if len(jobs) >= build_jobs:
390                                 break
391
392                         if component_buildable(c):
393                                 found = True
394
395                                 if not component_cached(c):
396                                         print 'Starting to build', c.name
397                                         start_job(c, jobs, make_jobs)
398                                 else:
399                                         print c.name, 'found from cache'
400                                         c.active_state = 'built'
401
402                 if not found and not jobs:
403                         for c in components:
404                                 if c.active_state != 'built':
405                                         raise Error('Internal error')
406                         break
407
408                 wait_for_job(jobs)
409
410 def component_buildable(c):
411         if c.active_state:
412                 if config.debug:
413                         print c.name, 'is in state:', c.active_state
414                 return False
415
416         for dep in c.active_depends:
417                 if config.components[dep.name].active_state != 'built':
418                         if config.debug:
419                                 print c.name, 'depends on unbuilt', dep.name
420                         return False
421
422         return True
423
424 def component_cached(c):
425         if c.from_platform:
426                 return True
427
428         for repo in (c.repo, c.meta):
429                 if git.ls_files(repo.path, ['-m', '-d']):
430                         if config.force:
431                                 return False
432                         else:
433                                 raise Error('Dirty files in ' + repo.path)
434
435         path = os.path.join(config.cache_dir, c.name)
436         if not os.path.exists(path):
437                 return False
438
439         file = open(path, 'r')
440         line = file.readline()
441         file.close()
442
443         match = re.match(r'([^\s]+)[\s]?([^\s]*)', line)
444         if not match:
445                 return False
446
447         hash, flagstring = match.groups()
448
449         if hash != get_component_hash(c):
450                 print 'Component has been changed:', c.name
451                 return False
452
453         if flagstring:
454                 flags = flagstring.split(',')
455         else:
456                 flags = []
457
458         if set(flags) != set(c.flags):
459                 print 'Component flags have been changed:', c.name
460                 return False
461
462         return True
463
464 def update_cache(c):
465         path = os.path.join(config.cache_dir, c.name)
466
467         dir = os.path.dirname(path)
468         if not os.path.exists(dir):
469                 os.makedirs(dir)
470
471         file = open(path, 'w')
472         done = False
473         try:
474                 print >>file, get_component_hash(c), ','.join(c.flags)
475                 done = True
476         finally:
477                 file.close()
478                 if not done:
479                         os.remove(path)
480
481 def get_component_hash(c):
482         return '%s+%s' % (get_repository_hash(c.repo), get_repository_hash(c.meta))
483
484 def get_repository_hash(repo):
485         if repo.active_hash is None:
486                 repo.active_hash = git.rev_parse(repo.path, 'HEAD')
487         return repo.active_hash
488
489 def start_job(c, jobs, make_jobs):
490         board = config.boards[config.board]
491
492         makefile = os.path.join(config.script_dir, 'matrix.mak')
493
494         workdir = os.path.join(config.top_dir, 'src', c.name)
495         args = ['make', '--no-print-directory', '-f', makefile, '-C', workdir,
496                 '-j', str(make_jobs), 'build_matrix_component',
497                 'MATRIX_TOPDIR='    + os.path.abspath(config.top_dir),
498                 'MATRIX_SCRIPTDIR=' + os.path.abspath(config.script_dir),
499                 'MATRIX_COMPONENT=' + c.name,
500                 'MATRIX_ARCH='      + board.arch,
501                 'MATRIX_GCC_MARCH=' + board.gcc_march,
502                 'MATRIX_GCC_MCPU='  + board.gcc_mcpu,
503                 'MATRIX_GCC_MFPU='  + board.gcc_mfpu,
504                 'MATRIX_GCC_OPTIONS=' + board.gcc_options,
505                 'MATRIX_GNU_HOST='  + board.gnu_host,
506                 'MATRIX_LIBC='      + config.libc]
507
508         for flag in config.flags:
509                 args.append(flag + '=1')
510
511         if config.verbose:
512                 args.append('MATRIX_VERBOSE=1')
513
514         if config.debug:
515                 print 'Executing:', ' '.join(args)
516
517         pid = os.spawnvp(os.P_NOWAIT, args[0], args)
518
519         jobs[pid] = c
520         c.active_state = 'running'
521
522 def wait_for_job(jobs):
523         if not jobs:
524                 return
525
526         pid, status = os.wait()
527
528         c = jobs.get(pid)
529         if not c:
530                 return
531
532         del jobs[pid]
533
534         if status == 0:
535                 c.active_state = 'built'
536                 update_cache(c)
537                 print c.name, 'completed'
538         else:
539                 c.active_state = 'error'
540                 print >>sys.stderr, c.name, 'failed with status:', status
541
542                 wait_for_job(jobs)
543                 raise Error('Build failed')
544
545 def clean_components(targets):
546         if not targets:
547                 targets = config.components.keys()
548
549         for name in targets:
550                 c = config.components[name]
551                 print 'Cleaning', c.repo.path
552
553                 files = git.ls_files(c.repo.path, ['-o'])
554                 paths = [os.path.join(c.repo.path, i) for i in files]
555                 paths.sort()
556                 paths.reverse()
557
558                 cache = os.path.join(config.cache_dir, name)
559                 if os.path.exists(cache):
560                         paths.append(cache)
561
562                 for path in paths:
563                         if git.contains_database(path):
564                                 continue
565
566                         if config.debug:
567                                 print 'Removing', path
568
569                         if os.path.islink(path) or not os.path.isdir(path):
570                                 os.remove(path)
571                         else:
572                                 remove_tree(path)
573
574                 for repo in (c.repo, c.meta):
575                         files = git.ls_files(repo.path, ['-m', '-d'])
576                         if files:
577                                 print len(files), 'dirty files left in', repo.path
578
579 def pull_components(targets):
580         if not targets:
581                 targets = config.components.keys()
582
583         for name in targets:
584                 print 'Pulling component:', name
585
586                 c = config.components[name]
587                 git.pull(c.repo.path)
588                 git.pull(c.meta.path)
589
590 def source_dist_components(targets):
591         if not targets:
592                 targets = config.components.keys()
593
594         for name in targets:
595                 c = config.components[name]
596                 generate_component_changes(c, 'dist')
597                 package_component_sources(c, 'dist')
598
599 def generate_component_changes(c, location):
600         print 'Generating change log for', c.repo.path
601
602         path = os.path.join(location, c.name) + '.changes'
603
604         pathdir = os.path.dirname(path)
605         if not os.path.exists(pathdir):
606                 os.makedirs(pathdir)
607
608         fd = os.open(path, os.O_WRONLY | os.O_CREAT, 0644)
609         git.log(c.repo.path, [c.get_active_tag()], fd=fd)
610         os.close(fd)
611
612 def package_component_sources(c, location):
613         print 'Archiving', c.repo.path
614
615         rev = git.describe(c.repo.path)
616         if rev:
617                 rev = '_' + rev
618         else:
619                 rev = ''
620
621         path = os.path.join(location, c.name) + rev + '.tar.bz2'
622         if os.path.exists(path):
623                 os.remove(path)
624
625         pathdir = os.path.dirname(path)
626         if not os.path.exists(pathdir):
627                 os.makedirs(pathdir)
628
629         git.archive(c.repo.path, path,
630                     prefix=os.path.basename(c.name) + '/',
631                     branch=c.get_active_tag())
632
633 def build_rootfs(clean=True, rootfs_only=False, jffs2_only=False, devrootfs_only=False):
634         parse_config('rootfs')
635
636         f = os.popen('sb-conf cu')
637         target = f.read()
638         f.close()
639
640         rootfs = RootFS(target.strip())
641         rootfs.include_paths(config.include_paths)
642         rootfs.include_files(config.include_files)
643         rootfs.filter_paths(config.exclude_paths)
644         rootfs.filter_files(config.exclude_files)
645         rootfs.filter_expressions(config.exclude_expressions)
646         rootfs.add_paths(config.created_paths)
647         rootfs.set_devices(config.devices)
648         rootfs.set_change_owner(config.change_owner)
649         rootfs.set_erase_size(config.boards[config.board].flash_erase_size)
650         rootfs.set_pad_size(config.boards[config.board].flash_pad_size)
651
652         if rootfs_only:
653                 rootfs.generate(clean, build_target="rootfs")
654         elif jffs2_only:
655                 rootfs.generate(clean, build_target="jffs2")
656         elif devrootfs_only:
657                 rootfs.generate(clean, build_target="devrootfs")
658         else:
659                 rootfs.generate(clean, build_target="all")