move config parsing out of the matrix module (and fix the graph module)
[matrix.git] / matrix / matrix.py
1 # Copyright (C) 2006-2008 Movial Oy
2 # Authors: Timo Savola <tsavola@movial.fi>
3 #          Toni Timonen
4 #          Kalle Vahlman <kalle.vahlman@movial.fi>
5 #          Tuomas Kulve <tuomas.kulve@movial.fi>
6
7 import os
8 import sys
9 from sets import Set as set
10
11 import cache
12 import components
13 import config
14 import git
15 import log
16 from build import build
17 from rootfs import RootFS
18
19 Error = RuntimeError
20
21 def main():
22         config.parse('config.local')
23
24         command, params, targets = parse_args(sys.argv)
25
26         config.parse('config')
27         config.parse('boards')
28         config.parse('components')
29
30         if 'global-cache' in config.flags:
31                 config.cache_dir = config.global_cache_dir
32
33         if not os.path.exists(config.cache_dir):
34                 os.makedirs(config.cache_dir)
35
36         all_targets = components.init()
37
38         for i in xrange(len(targets)):
39                 name = targets[i]
40                 if name.startswith('src' + os.path.sep):
41                         name = name.split(os.path.sep)[1]
42                         targets[i] = name
43
44                 if name not in config.components:
45                         raise Error('Component "%s" does not exist' % name)
46
47         if not targets:
48                 targets = all_targets
49
50         if command == 'install':
51                 build(targets)
52         elif command == 'clone':
53                 clone(targets)
54         elif command == 'clean':
55                 clean(targets)
56         elif command == 'rebase':
57                 rebase(targets)
58         elif command == 'pull':
59                 pull(targets)
60         elif command == 'source-dist':
61                 source_dist(targets)
62         elif command == 'rootfs':
63                 build_rootfs(**params)
64         else:
65                 raise Error('Invalid command: ' + command)
66
67 def help(file, args):
68         print >>file, '''
69 Copyright (C) 2006-2008 Movial Oy
70
71 Usage: %(progname)s [<options>] <command> [<params>] [<components>]
72
73 If no components are specified, all of them will be targeted.  All component
74 metadata will be downloaded regardless of the specified command and components.
75
76 Options:
77         -v              Verbose output.
78         -d              Debug output.
79         -r URL          Specify the root for component git repos to clone from.
80                         If this option is not given, the roots specified in the
81                         components file will be used.  This option may be
82                         specified multiple times.
83         -f              Build components with dirty files.
84         -k              Build as much as possible after an error.
85         -j N            Execute N build jobs in parallel.
86         -mj N           Run component builds with make -j N.
87         -h, --help      Print this help text.
88
89 Commands and parameters:
90         clone           Download the components' git repositories.
91         install         Download, build and install the components.
92         clean           Remove all non-tracked files from the component git
93                         repository directories.
94         rebase          Update repositories from server by rebasing.
95         pull            Update repositories from server by merging.
96         source-dist     Download and package the component sources.
97         rootfs          Build a rootfs from the current target installation.
98                 -n              Do not clean env.faked and stripped directory.
99                 -r              Only generate a stripped rootfs.
100                 -j              Only generate a jffs2 image
101                 -d              Only generate a rootstrap (development rootfs).
102 ''' % {'progname': args[0]}
103
104 def parse_args(args):
105         def parse_jobs(arg):
106                 jobs = int(arg)
107                 if jobs <= 0:
108                         raise Error('Please specify a valid number of jobs')
109                 return jobs
110
111         command = None
112         params = {}
113         targets = []
114
115         i = 1
116         while i < len(args):
117                 if args[i].startswith('-'):
118                         if not command:
119                                 if args[i] == '-v':
120                                         config.verbose = True
121                                 elif args[i] == '-d':
122                                         config.debug = True
123                                 elif args[i] == '-r':
124                                         i += 1
125                                         config.roots.append(args[i])
126                                 elif args[i] == '-f':
127                                         config.force = True
128                                 elif args[i] == '-k':
129                                         config.keep_going = True
130                                 elif args[i] == '-j':
131                                         i += 1
132                                         config.jobs = parse_jobs(args[i])
133                                 elif args[i] == '-mj':
134                                         i += 1
135                                         config.make_jobs = parse_jobs(args[i])
136                                 elif args[i] in ('-h', '--help'):
137                                         help(sys.stdout, args)
138                                         sys.exit(0)
139                                 else:
140                                         raise Error('Bad option: ' + args[i])
141                         elif command == 'rootfs':
142                                 if args[i] == '-n':
143                                         i += 1
144                                         params['clean'] = False
145                                 elif args[i] == '-r':
146                                         i += 1
147                                         params['rootfs_only'] = True
148                                 elif args[i] == '-j':
149                                         i += 1
150                                         params['jffs2_only'] = True
151                                 elif args[i] == '-d':
152                                         i += 1
153                                         params['devrootfs_only'] = True
154                         else:
155                                 raise Error('Command takes no parameters')
156                 elif not command:
157                         command = args[i]
158                 else:
159                         targets.append(args[i])
160                 i += 1
161
162         if not command:
163                 help(sys.stderr, args)
164                 sys.exit(1)
165
166         return command, params, targets
167
168 def execute(args):
169         if config.debug:
170                 print 'Executing:', ' '.join(args)
171
172         if os.spawnvp(os.P_WAIT, args[0], args) != 0:
173                 raise Error('Failed: ' + ' '.join(args))
174
175 def remove_tree(path):
176         execute(['rm', '-rf', path])
177
178 def clone(targets):
179         for name in targets:
180                 c = config.components[name]
181                 clone_component(c)
182
183 def clone_component(c, overwrite=False):
184         have_repo = c.repo.exists()
185         have_meta = c.meta.exists()
186
187         if not overwrite and have_repo and have_meta:
188                 return
189
190         if overwrite and os.path.exists(c.repo.path):
191                 print 'Removing', c.repo
192
193                 remove_tree(c.repo.path)
194                 have_repo = False
195                 have_meta = False
196
197         if not have_repo:
198                 c.repo.clone()
199         if not have_meta:
200                 c.meta.clone()
201
202 def clean(targets):
203         for name in targets:
204                 c = config.components[name]
205                 c.repo.clean()
206                 cache.remove(c)
207
208                 for repo in (c.repo, c.meta):
209                         if repo.dirty_files():
210                                 log.message('Dirty files left in %s' % \
211                                             repo.path)
212
213 def for_each_repository(func, targets=None):
214         for name in targets:
215                 c = config.components[name]
216                 if c.repo.exists():
217                         func(c.repo)
218                 func(c.meta)
219
220 def rebase(targets):
221         for_each_repository(lambda repo: repo.rebase(), targets)
222
223 def pull(targets):
224         for_each_repository(lambda repo: repo.pull(), targets)
225
226 def source_dist(targets):
227         location = 'dist'
228         if not os.path.exists(location):
229                 os.makedirs(location)
230
231         for name in targets:
232                 c = config.components[name]
233                 dist_changes(c, location)
234                 dist_sources(c, location)
235
236 def dist_changes(c, location):
237         path = os.path.join(location, c.name) + '.changes'
238         c.repo.dump_log(path)
239
240 def dist_sources(c, location):
241         rev = c.repo.describe()
242         if rev:
243                 rev = '_' + rev
244         else:
245                 rev = ''
246
247         name = c.name + rev
248         path = os.path.join(location, name) + '.tar.bz2'
249         if os.path.exists(path):
250                 os.remove(path)
251
252         c.repo.archive(name, path)
253
254 def build_rootfs(clean=True, rootfs_only=False, jffs2_only=False, devrootfs_only=False):
255         config.parse('rootfs')
256
257         f = os.popen('sb-conf cu')
258         target = f.read()
259         f.close()
260
261         rootfs = RootFS(target.strip())
262         rootfs.include_paths(config.include_paths)
263         rootfs.include_files(config.include_files)
264         rootfs.filter_paths(config.exclude_paths)
265         rootfs.filter_files(config.exclude_files)
266         rootfs.filter_expressions(config.exclude_expressions)
267         rootfs.add_paths(config.created_paths)
268         rootfs.set_devices(config.devices)
269         rootfs.set_change_owner(config.change_owner)
270         rootfs.set_erase_size(config.boards[config.board].flash_erase_size)
271         rootfs.set_pad_size(config.boards[config.board].flash_pad_size)
272
273         if rootfs_only:
274                 rootfs.generate(clean, build_target="rootfs")
275         elif jffs2_only:
276                 rootfs.generate(clean, build_target="jffs2")
277         elif devrootfs_only:
278                 rootfs.generate(clean, build_target="devrootfs")
279         else:
280                 rootfs.generate(clean, build_target="all")