primary repository is now called "source" instead of "repo"
[matrix.git] / matrix / matrix.py
1 # Copyright (C) 2006-2008 Movial Oy
2 # Authors: Timo Savola <tsavola@movial.fi>
3 #          Toni Timonen
4 #          Kalle Vahlman <kalle.vahlman@movial.fi>
5 #          Tuomas Kulve <tuomas.kulve@movial.fi>
6
7 import os
8 import sys
9 from sets import Set as set
10
11 import cache
12 import components
13 import config
14 import git
15 import log
16 from build import build, build_only
17 from rootfs import RootFS
18
19 Error = RuntimeError
20
21 def main():
22         config.parse('config.local')
23
24         command, params, targets = parse_args(sys.argv)
25
26         config.parse('config')
27         config.parse('boards')
28         config.parse('components')
29
30         if 'global-cache' in config.flags:
31                 config.cache_dir = config.global_cache_dir
32
33         if not os.path.exists(config.cache_dir):
34                 os.makedirs(config.cache_dir)
35
36         all_targets = components.init()
37
38         for i in xrange(len(targets)):
39                 name = targets[i]
40                 if name.startswith('src' + os.path.sep):
41                         name = name.split(os.path.sep)[1]
42                         targets[i] = name
43
44                 if name not in config.components:
45                         raise Error('Component "%s" does not exist' % name)
46
47         if not targets:
48                 targets = all_targets
49
50         if command == 'install':
51                 build(targets)
52         elif command == 'install-only':
53                 build_only(targets)
54         elif command == 'clone':
55                 clone(targets)
56         elif command == 'clean':
57                 clean(targets)
58         elif command == 'rebase':
59                 rebase(targets)
60         elif command == 'pull':
61                 pull(targets)
62         elif command == 'changes':
63                 changes(targets)
64         elif command == 'source-dist':
65                 source_dist(targets)
66         elif command == 'rootfs':
67                 build_rootfs(**params)
68         else:
69                 raise Error('Invalid command: ' + command)
70
71 def help(file, args):
72         print >>file, '''
73 Copyright (C) 2006-2008 Movial Oy
74
75 Usage: %(progname)s [<options>] <command> [<params>] [<components>]
76
77 If no components are specified, all of them will be targeted.  All component
78 metadata will be downloaded regardless of the specified command and components.
79
80 Options:
81         -v              Verbose output.
82         -d              Debug output.
83         -r URL          Specify the root for component git repos to clone from.
84                         If this option is not given, the roots specified in the
85                         components file will be used.  This option may be
86                         specified multiple times.
87         -f              Build (or be quiet about) components with dirty files.
88         -k              Build as much as possible after an error.
89         -j N            Execute N build jobs in parallel.
90         -mj N           Run component builds with make -j N.
91         -h, --help      Print this help text.
92
93 Commands and parameters:
94         clone           Download the components' git repositories.
95         install         Download, build and install the components.
96         install-only    Download, build and install the specified components
97                         without dependencies.
98         clean           Remove all non-tracked files from the component git
99                         repository directories.
100         rebase          Update repositories from server by rebasing.
101         pull            Update repositories from server by merging.
102         changes         Show commits which are not on the server.
103         source-dist     Download and package the component sources.
104         rootfs          Build a rootfs from the current target installation.
105                 -n              Do not clean env.faked and stripped directory.
106                 -r              Only generate a stripped rootfs.
107                 -j              Only generate a jffs2 image
108                 -d              Only generate a rootstrap (development rootfs).
109 ''' % {'progname': args[0]}
110
111 def parse_args(args):
112         def parse_jobs(arg):
113                 jobs = int(arg)
114                 if jobs <= 0:
115                         raise Error('Please specify a valid number of jobs')
116                 return jobs
117
118         command = None
119         params = {}
120         targets = []
121
122         i = 1
123         while i < len(args):
124                 if args[i].startswith('-'):
125                         if not command:
126                                 if args[i] == '-v':
127                                         config.verbose = True
128                                 elif args[i] == '-d':
129                                         config.debug = True
130                                 elif args[i] == '-r':
131                                         i += 1
132                                         config.roots.append(args[i])
133                                 elif args[i] == '-f':
134                                         config.force = True
135                                 elif args[i] == '-k':
136                                         config.keep_going = True
137                                 elif args[i] == '-j':
138                                         i += 1
139                                         config.jobs = parse_jobs(args[i])
140                                 elif args[i] == '-mj':
141                                         i += 1
142                                         config.make_jobs = parse_jobs(args[i])
143                                 elif args[i] in ('-h', '--help'):
144                                         help(sys.stdout, args)
145                                         sys.exit(0)
146                                 else:
147                                         raise Error('Bad option: ' + args[i])
148                         elif command == 'rootfs':
149                                 if args[i] == '-n':
150                                         i += 1
151                                         params['clean'] = False
152                                 elif args[i] == '-r':
153                                         i += 1
154                                         params['rootfs_only'] = True
155                                 elif args[i] == '-j':
156                                         i += 1
157                                         params['jffs2_only'] = True
158                                 elif args[i] == '-d':
159                                         i += 1
160                                         params['devrootfs_only'] = True
161                         else:
162                                 raise Error('Command takes no parameters')
163                 elif not command:
164                         command = args[i]
165                 else:
166                         targets.append(args[i])
167                 i += 1
168
169         if not command:
170                 help(sys.stderr, args)
171                 sys.exit(1)
172
173         return command, params, targets
174
175 def execute(args):
176         if config.debug:
177                 print 'Executing:', ' '.join(args)
178
179         if os.spawnvp(os.P_WAIT, args[0], args) != 0:
180                 raise Error('Failed: ' + ' '.join(args))
181
182 def remove_tree(path):
183         execute(['rm', '-rf', path])
184
185 def clone(targets):
186         for name in targets:
187                 c = config.components[name]
188                 clone_component(c)
189
190 def clone_component(c, overwrite=False):
191         have_source = c.source.exists()
192         have_meta = c.meta.exists()
193
194         if not overwrite and have_source and have_meta:
195                 return
196
197         if overwrite and os.path.exists(c.source.path):
198                 print 'Removing', c
199
200                 remove_tree(c.source.path)
201                 have_source = False
202                 have_meta = False
203
204         if not have_source:
205                 c.source.clone()
206         if not have_meta:
207                 c.meta.clone()
208
209 def clean(targets):
210         changed = False
211
212         for name in targets:
213                 c = config.components[name]
214                 c.source.clean()
215                 cache.remove(c)
216
217                 if not config.force:
218                         changed = check_dirty(c)
219
220         if changed:
221                 raise Error()
222
223 def for_each_repository(func, targets=None):
224         ret = None
225
226         for name in targets:
227                 c = config.components[name]
228                 if c.source.exists():
229                         value = func(c.source)
230                         if value:
231                                 ret = value
232                 value = func(c.meta)
233                 if value:
234                         ret = value
235
236         return ret
237
238 def rebase(targets):
239         for_each_repository(lambda repo: repo.rebase(), targets)
240
241 def pull(targets):
242         for_each_repository(lambda repo: repo.pull(), targets)
243
244 def changes(targets):
245         changed = False
246
247         for name in targets:
248                 c = config.components[name]
249
250                 if c.source.exists():
251                         if check_changes(c.source):
252                                 changed = True
253
254                 if check_changes(c.meta):
255                         changed = True
256
257                 if not config.force and check_dirty(c):
258                         changed = True
259
260         if changed:
261                 raise Error()
262
263 def check_changes(repo):
264         ret = repo.changes()
265         if ret:
266                 log.message('Changes in %s' % repo)
267         return ret
268
269 def check_dirty(c):
270         ret = c.is_dirty()
271         if ret:
272                 log.message('Dirty files in %s' % c.name)
273         return ret
274
275 def source_dist(targets):
276         location = 'dist'
277         if not os.path.exists(location):
278                 os.makedirs(location)
279
280         for name in targets:
281                 c = config.components[name]
282                 dist_changes(c, location)
283                 dist_sources(c, location)
284
285 def dist_changes(c, location):
286         path = os.path.join(location, c.name) + '.changes'
287         c.source.dump_log(path)
288
289 def dist_sources(c, location):
290         rev = c.source.describe()
291         if rev:
292                 rev = '_' + rev
293         else:
294                 rev = ''
295
296         name = c.name + rev
297         path = os.path.join(location, name) + '.tar.bz2'
298         if os.path.exists(path):
299                 os.remove(path)
300
301         c.source.archive(name, path)
302
303 def build_rootfs(clean=True, rootfs_only=False, jffs2_only=False, devrootfs_only=False):
304         config.parse('rootfs')
305
306         f = os.popen('sb-conf cu')
307         target = f.read()
308         f.close()
309
310         rootfs = RootFS(target.strip())
311         rootfs.include_paths(config.include_paths)
312         rootfs.include_files(config.include_files)
313         rootfs.filter_paths(config.exclude_paths)
314         rootfs.filter_files(config.exclude_files)
315         rootfs.filter_expressions(config.exclude_expressions)
316         rootfs.add_paths(config.created_paths)
317         rootfs.set_devices(config.devices)
318         rootfs.set_change_owner(config.change_owner)
319         rootfs.set_erase_size(config.boards[config.board].flash_erase_size)
320         rootfs.set_pad_size(config.boards[config.board].flash_pad_size)
321
322         if rootfs_only:
323                 rootfs.generate(clean, build_target="rootfs")
324         elif jffs2_only:
325                 rootfs.generate(clean, build_target="jffs2")
326         elif devrootfs_only:
327                 rootfs.generate(clean, build_target="devrootfs")
328         else:
329                 rootfs.generate(clean, build_target="all")