-k option builds as many components as possible
[matrix.git] / matrix / matrix.py
1 # Copyright (C) 2006-2008 Movial Oy
2 # Authors: Timo Savola <tsavola@movial.fi>
3 #          Toni Timonen
4 #          Kalle Vahlman <kalle.vahlman@movial.fi>
5 #          Tuomas Kulve <tuomas.kulve@movial.fi>
6
7 import os
8 import sys
9 from sets import Set as set
10
11 import cache
12 import components
13 import config
14 import git
15 import log
16 from build import build
17 from rootfs import RootFS
18
19 Error = RuntimeError
20
21 def main():
22         parse_config('config.local')
23
24         command, params, targets = parse_args(sys.argv)
25
26         parse_config('config')
27         parse_config('boards')
28         parse_config('components')
29
30         if 'global-cache' in config.flags:
31                 config.cache_dir = config.global_cache_dir
32
33         if not os.path.exists(config.cache_dir):
34                 os.makedirs(config.cache_dir)
35
36         all_targets = components.init()
37
38         for i in xrange(len(targets)):
39                 name = targets[i]
40                 if name.startswith('src' + os.path.sep):
41                         name = name.split(os.path.sep)[1]
42                         targets[i] = name
43
44                 if name not in config.components:
45                         raise Error('Component "%s" does not exist' % name)
46
47         if not targets:
48                 targets = all_targets
49
50         if command == 'install':
51                 build(targets, **params)
52         elif command == 'clone':
53                 clone(targets)
54         elif command == 'clean':
55                 clean(targets)
56         elif command == 'rebase':
57                 rebase(targets)
58         elif command == 'pull':
59                 pull(targets)
60         elif command == 'source-dist':
61                 source_dist(targets)
62         elif command == 'rootfs':
63                 build_rootfs(**params)
64         else:
65                 raise Error('Invalid command: ' + command)
66
67 def help(file, args):
68         print >>file, '''
69 Copyright (C) 2006-2008 Movial Oy
70
71 Usage: %(progname)s [<options>] <command> [<params>] [<components>]
72
73 If no components are specified, all of them will be targeted.  All component
74 metadata will be downloaded regardless of the specified command and components.
75
76 Options:
77         -v              Verbose output.
78         -d              Debug output.
79         -r URL          Specify the root for component git repos to clone from.
80                         If this option is not given, the roots specified in the
81                         components file will be used.  This option may be
82                         specified multiple times.
83         -f              Build components with dirty files.
84         -k              Build as much as possible after an error.
85         -h, --help      Print this help text.
86
87 Commands and parameters:
88         clone           Download the components' git repositories.
89         install         Download, build and install the components.
90                 -j N            Execute N build jobs in parallel.
91                 -mj N           Run component builds with make -j N.
92         clean           Remove all non-tracked files from the component git
93                         repository directories.
94         rebase          Update repositories from server by rebasing.
95         pull            Update repositories from server by merging.
96         source-dist     Download and package the component sources.
97         rootfs          Build a rootfs from the current target installation.
98                 -n              Do not clean env.faked and stripped directory.
99                 -r              Only generate a stripped rootfs.
100                 -j              Only generate a jffs2 image
101                 -d              Only generate a rootstrap (development rootfs).
102 ''' % {'progname': args[0]}
103
104 def parse_args(args):
105         def parse_jobs(arg):
106                 jobs = int(arg)
107                 if jobs <= 0:
108                         raise Error('Please specify a valid number of jobs')
109                 return jobs
110
111         command = None
112         params = {}
113         targets = []
114
115         i = 1
116         while i < len(args):
117                 if args[i].startswith('-'):
118                         if not command:
119                                 if args[i] == '-v':
120                                         config.verbose = True
121                                 elif args[i] == '-d':
122                                         config.debug = True
123                                 elif args[i] == '-r':
124                                         i += 1
125                                         config.roots.append(args[i])
126                                 elif args[i] == '-f':
127                                         config.force = True
128                                 elif args[i] == '-k':
129                                         config.keep_going = True
130                                 elif args[i] in ('-h', '--help'):
131                                         help(sys.stdout, args)
132                                         sys.exit(0)
133                                 else:
134                                         raise Error('Bad option: ' + args[i])
135                         elif command == 'install':
136                                 if args[i] == '-j':
137                                         i += 1
138                                         params['build_jobs'] = parse_jobs(args[i])
139                                 elif args[i] == '-mj':
140                                         i += 1
141                                         params['make_jobs'] = parse_jobs(args[i])
142                                 else:
143                                         raise Error('Bad parameter: ' + args[i])
144                         elif command == 'rootfs':
145                                 if args[i] == '-n':
146                                         i += 1
147                                         params['clean'] = False
148                                 elif args[i] == '-r':
149                                         i += 1
150                                         params['rootfs_only'] = True
151                                 elif args[i] == '-j':
152                                         i += 1
153                                         params['jffs2_only'] = True
154                                 elif args[i] == '-d':
155                                         i += 1
156                                         params['devrootfs_only'] = True
157                         else:
158                                 raise Error('Command takes no parameters')
159                 elif not command:
160                         command = args[i]
161                 else:
162                         targets.append(args[i])
163                 i += 1
164
165         if not command:
166                 help(sys.stderr, args)
167                 sys.exit(1)
168
169         return command, params, targets
170
171 def parse_config(path):
172         if os.path.isabs(path):
173                 path = os.path.join(config.top_dir, path)
174
175         if os.path.exists(path):
176                 if config.debug:
177                         print 'Reading', path
178
179                 execfile(path, config.__dict__, config.__dict__)
180
181 def execute(args):
182         if config.debug:
183                 print 'Executing:', ' '.join(args)
184
185         if os.spawnvp(os.P_WAIT, args[0], args) != 0:
186                 raise Error('Failed: ' + ' '.join(args))
187
188 def remove_tree(path):
189         execute(['rm', '-rf', path])
190
191 def clone(targets):
192         for name in targets:
193                 c = config.components[name]
194                 clone_component(c)
195
196 def clone_component(c, overwrite=False):
197         have_repo = c.repo.exists()
198         have_meta = c.meta.exists()
199
200         if not overwrite and have_repo and have_meta:
201                 return
202
203         if overwrite and os.path.exists(c.repo.path):
204                 print 'Removing', c.repo
205
206                 remove_tree(c.repo.path)
207                 have_repo = False
208                 have_meta = False
209
210         if not have_repo:
211                 c.repo.clone()
212         if not have_meta:
213                 c.meta.clone()
214
215 def clean(targets):
216         for name in targets:
217                 c = config.components[name]
218                 c.repo.clean()
219                 cache.remove(c)
220
221                 for repo in (c.repo, c.meta):
222                         if repo.dirty_files():
223                                 log.error('Dirty files left in %s' % repo.path)
224
225 def for_each_repository(func, targets=None):
226         for name in targets:
227                 c = config.components[name]
228                 if c.repo.exists():
229                         func(c.repo)
230                 func(c.meta)
231
232 def rebase(targets):
233         for_each_repository(lambda repo: repo.rebase(), targets)
234
235 def pull(targets):
236         for_each_repository(lambda repo: repo.pull(), targets)
237
238 def source_dist(targets):
239         location = 'dist'
240         if not os.path.exists(location):
241                 os.makedirs(location)
242
243         for name in targets:
244                 c = config.components[name]
245                 dist_changes(c, location)
246                 dist_sources(c, location)
247
248 def dist_changes(c, location):
249         path = os.path.join(location, c.name) + '.changes'
250         c.repo.dump_log(path)
251
252 def dist_sources(c, location):
253         rev = c.repo.describe()
254         if rev:
255                 rev = '_' + rev
256         else:
257                 rev = ''
258
259         name = c.name + rev
260         path = os.path.join(location, name) + '.tar.bz2'
261         if os.path.exists(path):
262                 os.remove(path)
263
264         c.repo.archive(name, path)
265
266 def build_rootfs(clean=True, rootfs_only=False, jffs2_only=False, devrootfs_only=False):
267         parse_config('rootfs')
268
269         f = os.popen('sb-conf cu')
270         target = f.read()
271         f.close()
272
273         rootfs = RootFS(target.strip())
274         rootfs.include_paths(config.include_paths)
275         rootfs.include_files(config.include_files)
276         rootfs.filter_paths(config.exclude_paths)
277         rootfs.filter_files(config.exclude_files)
278         rootfs.filter_expressions(config.exclude_expressions)
279         rootfs.add_paths(config.created_paths)
280         rootfs.set_devices(config.devices)
281         rootfs.set_change_owner(config.change_owner)
282         rootfs.set_erase_size(config.boards[config.board].flash_erase_size)
283         rootfs.set_pad_size(config.boards[config.board].flash_pad_size)
284
285         if rootfs_only:
286                 rootfs.generate(clean, build_target="rootfs")
287         elif jffs2_only:
288                 rootfs.generate(clean, build_target="jffs2")
289         elif devrootfs_only:
290                 rootfs.generate(clean, build_target="devrootfs")
291         else:
292                 rootfs.generate(clean, build_target="all")