4817d652266d2d0ebc056e038e82767baae869dd
[matrix.git] / matrix / build.py
1 # Copyright (C) 2006-2008 Movial Oy
2 # Authors: Timo Savola <tsavola@movial.fi>
3
4 import os
5 import signal
6 import sys
7 from sets import Set as set
8
9 import cache
10 import components
11 import log
12 from config import config
13
14 Error = RuntimeError
15 InternalError = Exception
16
17 def build(targets):
18         try:
19                 selected = set([components.by_name[i] for i in targets])
20         except KeyError, e:
21                 raise Error("Component " + i + " not found")
22         components.fill_in_depends(selected)
23         components.init_rdepends(selected)
24
25         roots = initial_roots(selected)
26         roots = uncached_roots(roots)
27         init_weights(roots)
28
29         flat = flatten(roots)
30         count = len(flat)
31
32         if config.debug:
33                 l = list(flat)
34                 l.sort(compare_name)
35
36                 print 'Building components:'
37                 for c in l:
38                         print '\t' + c.name
39
40                 del l
41
42         del flat
43
44         Builder(roots, count).run()
45
46         unbuilt = []
47         for c in selected:
48                 if c.state != 'done' and not cache.contains(c):
49                         unbuilt.append(c.name)
50
51         if unbuilt:
52                 raise Error('Components not built (dependency cycles?): ' + \
53                             ' '.join(unbuilt))
54
55         if not count:
56                 print 'Nothing to build'
57
58 def build_only(targets):
59         try:
60                 all = [components.by_name[i] for i in targets]
61         except KeyError:
62                 raise Error('Component ' + i + ' not found')
63         count = len(all)
64
65         # Set weights based on user-specified order.  Weight recalculation
66         # won't happen because the reverse dependency tree is flat.
67         #
68         # NOTE: config.jobs can still cause parallel builds!
69         #
70         weight = count
71         for c in all:
72                 c.weight = weight
73                 weight -= 1
74
75         Builder(all, count).run()
76
77 def initial_roots(selected):
78         return [c for c in selected if not c.get_depends()]
79
80 def uncached_roots(old_roots):
81         roots = set()
82         count = [0]
83
84         def add_roots(c):
85                 if not config.debug:
86                         count[0] += 1
87                         print '\rChecking cache:', count[0],
88                         sys.stdout.flush()
89
90                 if cache.contains(c):
91                         for child in c.get_rdepends():
92                                 if child.remove_depend(c):
93                                         add_roots(child)
94
95                         c.clear_rdepends()
96                 else:
97                         roots.add(c)
98
99         try:
100                 for c in old_roots:
101                         add_roots(c)
102         finally:
103                 if not config.debug:
104                         print
105
106         return list(roots)
107
108 def init_weights(rdepends):
109         total = 0
110
111         for c in rdepends:
112                 c.weight = 1 + init_weights(c.get_rdepends())
113                 total += c.weight
114
115         return total
116
117 def flatten(rdepends, all=set()):
118         for c in rdepends:
119                 all.add(c)
120                 flatten(c.get_rdepends(), all)
121
122         return all
123
124 def compare_name(a, b):
125         return cmp(a.name, b.name)
126
127 def compare_weight(a, b):
128         return cmp(a.weight, b.weight)
129
130 class Builder(object):
131         def __init__(self, roots, count):
132                 self.jobs = {}
133
134                 self.wait_build = roots
135                 self.wait_build.sort(compare_weight)
136
137                 self.wait_install = []
138                 self.in_install = None
139
140                 self.progress = Progress(count)
141
142                 self.error = False
143
144         def run(self):
145                 try:
146                         sys.stdout = self.progress
147                         try:
148                                 while self.run_iteration():
149                                         pass
150                         finally:
151                                 sys.stdout = self.progress.file
152                 except:
153                         for pid in self.jobs:
154                                 c = self.jobs[pid]
155                                 c.state = 'killed'
156
157                                 print 'Aborting', c
158
159                                 try:
160                                         os.kill(pid, signal.SIGTERM)
161                                 except OSError, e:
162                                         log.error('kill: %s' % e)
163
164                         while self.jobs:
165                                 self.wait()
166
167                         raise
168
169                 if self.error:
170                         raise Error()
171
172         def run_iteration(self):
173                 if self.can_start_install():
174                         c = self.wait_install.pop(0)
175                         self.start_install(c)
176                         self.in_install = c
177
178                 while self.can_start_build():
179                         c = self.wait_build.pop()
180                         self.start_build(c)
181
182                 if self.jobs:
183                         self.wait()
184                         return True
185                 else:
186                         return False
187
188         def can_start_build(self):
189                 return (not self.error or config.keep_going) and \
190                        self.wait_build and self.can_start_job()
191
192         def can_start_install(self):
193                 return not self.in_install and self.wait_install and \
194                        self.can_start_job()
195
196         def can_start_job(self):
197                 return len(self.jobs) < config.jobs
198
199         def start_build(self, c):
200                 if not c.source.exists():
201                         c.source.clone()
202
203                 print 'Building', c
204                 self.start_job(c, 'build')
205
206         def start_install(self, c):
207                 print 'Installing', c
208                 self.start_job(c, 'install')
209
210         def start_job(self, c, action):
211                 c.state = 'in-' + action
212                 pid = spawn(c, action)
213                 self.jobs[pid] = c
214
215         def wait(self):
216                 pid, status = os.wait()
217
218                 if status:
219                         self.error = True
220                         if config.keep_going:
221                                 self.progress.clear_max()
222
223                 c = self.jobs[pid]
224                 del self.jobs[pid]
225
226                 if c.state == 'in-build':
227                         if status:
228                                 print 'Failed', log_path(c, 'build')
229                                 log.error('Failed to build %s' % c.name)
230                                 return
231
232                         if config.debug:
233                                 print 'Built', c
234
235                         c.state = 'wait-install'
236                         self.wait_install.append(c)
237
238                 elif c.state == 'in-install':
239                         assert c == self.in_install
240
241                         if status:
242                                 print 'Failed', log_path(c, 'install')
243                                 log.error('Failed to install %s' % c.name)
244                                 return
245
246                         self.progress.increment()
247                         print 'Done', c
248
249                         c.state = 'done'
250                         self.in_install = None
251
252                         cache.update(c)
253
254                         wait_build_changed = False
255                         for child in c.get_rdepends():
256                                 if child.remove_depend(c):
257                                         self.wait_build.append(child)
258                                         wait_build_changed = True
259
260                         c.clear_rdepends()
261
262                         if wait_build_changed:
263                                 self.wait_build.sort(compare_weight)
264
265                 elif c.state != 'killed':
266                         raise InternalError('Unexpected state: %s' % c.state)
267
268 class Progress(object):
269         def __init__(self, max):
270                 self.now = 0
271                 self.file = sys.stdout
272
273                 self.max_width = len(str(max))
274                 self.format = '%%%dd/%d ' % (self.max_width, max)
275
276                 self.newline = True
277
278         def clear_max(self):
279                 self.format = '%%%dd %s ' % (self.max_width, ' ' * self.max_width)
280
281         def increment(self):
282                 self.now += 1
283
284         def write(self, str):
285                 progress = self.format % self.now
286
287                 while str:
288                         if self.newline:
289                                 self.file.write(progress)
290
291                         index = str.find('\n')
292                         if index < 0:
293                                 if self.newline and str.find(' ') < 0:
294                                         str = '%-10s' % str
295
296                                 self.file.write(str)
297                                 self.newline = False
298                                 return
299
300                         self.file.write(str[:index+1])
301                         str = str[index+1:]
302                         self.newline = True
303
304 def spawn(c, action):
305         board = config.boards[config.board]
306
307         script = os.path.join(config.script_dir, 'run.sh')
308         workdir = os.path.join(config.top_dir, 'src', c.name)
309         log = log_path(c, action)
310
311         args = ['bash', script]
312
313         env = dict(
314                 MATRIX_TOPDIR      = os.path.abspath(config.top_dir),
315                 MATRIX_SCRIPTDIR   = os.path.abspath(config.script_dir),
316                 MATRIX_WORKDIR     = os.path.abspath(workdir),
317                 MATRIX_LOG         = os.path.abspath(log),
318                 MATRIX_JOBS        = str(config.make_jobs),
319                 MATRIX_ACTION      = action,
320                 MATRIX_COMPONENT   = c.name,
321                 MATRIX_ARCH        = board.arch,
322                 MATRIX_GCC_MARCH   = board.gcc_march,
323                 MATRIX_GCC_MCPU    = board.gcc_mcpu,
324                 MATRIX_GCC_MFPU    = board.gcc_mfpu,
325                 MATRIX_GCC_OPTIONS = ' '.join(board.gcc_options),
326                 MATRIX_GNU_HOST    = board.gnu_host,
327         )
328
329         if config.sb2_target:
330                 if not config.sb2_compiler:
331                         raise Error('SB2 compiler path not set')
332
333                 init_options = ' '.join(config.sb2_init_options)
334
335                 env.update(dict(
336                         MATRIX_SB2_TARGET       = config.sb2_target,
337                         MATRIX_SB2_COMPILER     = config.sb2_compiler,
338                         MATRIX_SB2_INIT_OPTIONS = init_options,
339                 ))
340         else:
341                 if config.sb2_compiler:
342                         raise Error('SB2 target name not set')
343
344         for flag in config.flags:
345                 env[flag] = '1'
346
347         if config.verbose:
348                 env['MATRIX_VERBOSE'] = '1'
349
350         env.update(os.environ)
351
352         if config.debug:
353                 print 'Executing:', ' '.join(args)
354                 print 'Variables:'
355                 l = [key for key in env if key.startswith('MATRIX_')]
356                 l.sort()
357                 for key in l:
358                         print '\t%-18s = "%s"' % (key, env[key])
359
360         return os.spawnvpe(os.P_NOWAIT, args[0], args, env)
361
362 def log_path(c, type):
363         return os.path.join(c.meta.path, '%s.log' % type)