reorganized component and repository code into separate modules
[matrix.git] / matrix / matrix.py
1 # Copyright (C) 2006-2008 Movial Oy
2 # Authors: Timo Savola <tsavola@movial.fi>
3 #          Toni Timonen
4 #          Kalle Vahlman <kalle.vahlman@movial.fi>
5 #          Tuomas Kulve <tuomas.kulve@movial.fi>
6
7 import os
8 import sys
9 from sets import Set as set
10
11 import cache
12 import components
13 import config
14 import git
15 import log
16 from build import Builder
17 from rootfs import RootFS
18
19 Error = RuntimeError
20
21 def main():
22         parse_config('config.local')
23
24         command, params, targets = parse_args(sys.argv)
25
26         parse_config('config')
27         parse_config('boards')
28         parse_config('components')
29
30         if 'global-cache' in config.flags:
31                 config.cache_dir = config.global_cache_dir
32
33         if not os.path.exists(config.cache_dir):
34                 os.makedirs(config.cache_dir)
35
36         all_targets = components.init()
37
38         for name in targets:
39                 if name not in config.components:
40                         raise Error('Component "%s" does not exist' % name)
41
42         if not targets:
43                 targets = all_targets
44
45         if command == 'install':
46                 build(targets, **params)
47         elif command == 'clone':
48                 clone(targets)
49         elif command == 'clean':
50                 clean(targets)
51         elif command == 'rebase':
52                 rebase(targets)
53         elif command == 'pull':
54                 pull(targets)
55         elif command == 'source-dist':
56                 source_dist(targets)
57         elif command == 'rootfs':
58                 build_rootfs(**params)
59         else:
60                 raise Error('Invalid command: ' + command)
61
62 def help(file, args):
63         print >>file, '''
64 Copyright (C) 2006-2008 Movial Oy
65
66 Usage: %(progname)s [<options>] <command> [<params>] [<components>]
67
68 If no components are specified, all of them will be targeted.  All component
69 metadata will be downloaded regardless of the specified command and components.
70
71 Options:
72         -v              Verbose output.
73         -d              Debug output.
74         -r URL          Specify the root for component git repos to clone from.
75                         If this option is not given, the roots specified in the
76                         components file will be used.  This option may be
77                         specified multiple times.
78         -f              Build components with dirty files.
79         -h, --help      Print this help text.
80
81 Commands and parameters:
82         clone           Download the components' git repositories.
83         install         Download, build and install the components.
84                 -j N            Execute N build jobs in parallel.
85                 -mj N           Run component builds with make -j N.
86         clean           Remove all non-tracked files from the component git
87                         repository directories.
88         rebase          Update repositories from server by rebasing.
89         pull            Update repositories from server by merging.
90         source-dist     Download and package the component sources.
91         rootfs          Build a rootfs from the current target installation.
92                 -n              Do not clean env.faked and stripped directory.
93                 -r              Only generate a stripped rootfs.
94                 -j              Only generate a jffs2 image
95                 -d              Only generate a rootstrap (development rootfs).
96 ''' % {'progname': args[0]}
97
98 def parse_args(args):
99         def parse_jobs(arg):
100                 jobs = int(arg)
101                 if jobs <= 0:
102                         raise Error('Please specify a valid number of jobs')
103                 return jobs
104
105         command = None
106         params = {}
107         targets = []
108
109         i = 1
110         while i < len(args):
111                 if args[i].startswith('-'):
112                         if not command:
113                                 if args[i] == '-v':
114                                         config.verbose = True
115                                 elif args[i] == '-d':
116                                         config.debug = True
117                                 elif args[i] == '-r':
118                                         i += 1
119                                         config.roots.append(args[i])
120                                 elif args[i] == '-f':
121                                         config.force = True
122                                 elif args[i] in ('-h', '--help'):
123                                         help(sys.stdout, args)
124                                         sys.exit(0)
125                                 else:
126                                         raise Error('Bad option: ' + args[i])
127                         elif command == 'install':
128                                 if args[i] == '-j':
129                                         i += 1
130                                         params['build_jobs'] = parse_jobs(args[i])
131                                 elif args[i] == '-mj':
132                                         i += 1
133                                         params['make_jobs'] = parse_jobs(args[i])
134                                 else:
135                                         raise Error('Bad parameter: ' + args[i])
136                         elif command == 'rootfs':
137                                 if args[i] == '-n':
138                                         i += 1
139                                         params['clean'] = False
140                                 elif args[i] == '-r':
141                                         i += 1
142                                         params['rootfs_only'] = True
143                                 elif args[i] == '-j':
144                                         i += 1
145                                         params['jffs2_only'] = True
146                                 elif args[i] == '-d':
147                                         i += 1
148                                         params['devrootfs_only'] = True
149                         else:
150                                 raise Error('Command takes no parameters')
151                 elif not command:
152                         command = args[i]
153                 else:
154                         targets.append(args[i])
155                 i += 1
156
157         if not command:
158                 help(sys.stderr, args)
159                 sys.exit(1)
160
161         return command, params, targets
162
163 def parse_config(path):
164         if os.path.isabs(path):
165                 path = os.path.join(config.top_dir, path)
166
167         if os.path.exists(path):
168                 print 'Reading', path
169                 execfile(path, config.__dict__, config.__dict__)
170
171 def execute(args):
172         if config.debug:
173                 print 'Executing:', ' '.join(args)
174
175         if os.spawnvp(os.P_WAIT, args[0], args) != 0:
176                 raise Error('Failed: ' + ' '.join(args))
177
178 def remove_tree(path):
179         execute(['rm', '-rf', path])
180
181 def clone(targets):
182         for name in targets:
183                 c = config.components[name]
184                 clone_component(c)
185
186 def clone_component(c, overwrite=False):
187         have_repo = c.repo.exists()
188         have_meta = c.meta.exists()
189
190         if not overwrite and have_repo and have_meta:
191                 return
192
193         if overwrite and os.path.exists(c.repo.path):
194                 print 'Removing', c.repo.path
195
196                 remove_tree(c.repo.path)
197                 have_repo = False
198                 have_meta = False
199
200         if not have_repo:
201                 c.repo.clone()
202         if not have_meta:
203                 c.meta.clone()
204
205 def build(targets, build_jobs=1, make_jobs=1):
206         selected = set([config.components[i] for i in targets])
207
208         depends = None
209         while depends is None or depends:
210                 depends = set()
211
212                 for c in selected:
213                         for dep in c.depends:
214                                 if dep not in selected:
215                                         depends.add(dep)
216
217                 selected |= depends
218
219         if config.debug:
220                 print 'Building components:'
221
222                 sorted = list(selected)
223                 sorted.sort(key=lambda c: c.name)
224
225                 for c in sorted:
226                         print '\t' + c.name
227
228         builder = Builder(selected, build_jobs, make_jobs)
229         builder.run()
230
231 def clean(targets):
232         for name in targets:
233                 c = config.components[name]
234                 c.repo.clean()
235                 cache.remove(c)
236
237                 for repo in (c.repo, c.meta):
238                         if repo.dirty_files():
239                                 log.error('Dirty files left in %s' % repo.path)
240
241 def for_each_repository(func, targets=None):
242         for name in targets:
243                 c = config.components[name]
244                 if c.repo.exists():
245                         func(c.repo)
246                 func(c.meta)
247
248 def rebase(targets):
249         for_each_repository(lambda repo: repo.rebase(), targets)
250
251 def pull(targets):
252         for_each_repository(lambda repo: repo.pull(), targets)
253
254 def source_dist(targets):
255         location = 'dist'
256         if not os.path.exists(location):
257                 os.makedirs(location)
258
259         for name in targets:
260                 c = config.components[name]
261                 dist_changes(c, location)
262                 dist_sources(c, location)
263
264 def dist_changes(c, location):
265         path = os.path.join(location, c.name) + '.changes'
266         c.repo.dump_log(path)
267
268 def dist_sources(c, location):
269         rev = c.repo.describe()
270         if rev:
271                 rev = '_' + rev
272         else:
273                 rev = ''
274
275         name = c.name + rev
276         path = os.path.join(location, name) + '.tar.bz2'
277         if os.path.exists(path):
278                 os.remove(path)
279
280         c.repo.archive(name, path)
281
282 def build_rootfs(clean=True, rootfs_only=False, jffs2_only=False, devrootfs_only=False):
283         parse_config('rootfs')
284
285         f = os.popen('sb-conf cu')
286         target = f.read()
287         f.close()
288
289         rootfs = RootFS(target.strip())
290         rootfs.include_paths(config.include_paths)
291         rootfs.include_files(config.include_files)
292         rootfs.filter_paths(config.exclude_paths)
293         rootfs.filter_files(config.exclude_files)
294         rootfs.filter_expressions(config.exclude_expressions)
295         rootfs.add_paths(config.created_paths)
296         rootfs.set_devices(config.devices)
297         rootfs.set_change_owner(config.change_owner)
298         rootfs.set_erase_size(config.boards[config.board].flash_erase_size)
299         rootfs.set_pad_size(config.boards[config.board].flash_pad_size)
300
301         if rootfs_only:
302                 rootfs.generate(clean, build_target="rootfs")
303         elif jffs2_only:
304                 rootfs.generate(clean, build_target="jffs2")
305         elif devrootfs_only:
306                 rootfs.generate(clean, build_target="devrootfs")
307         else:
308                 rootfs.generate(clean, build_target="all")