51e23cf4c17b66c12551d75e23687239c349dcc8
[matrix.git] / matrix / build.py
1 # Copyright (C) 2006-2008 Movial Oy
2 # Authors: Timo Savola <tsavola@movial.fi>
3
4 import os
5 import signal
6 import sys
7 from sets import Set as set
8
9 import cache
10 import components
11 import log
12 from config import config
13
14 Error = RuntimeError
15 InternalError = Exception
16
17 def build(targets):
18         selected = set([components.by_name[i] for i in targets])
19         components.fill_in_depends(selected)
20         components.init_rdepends(selected)
21
22         roots = initial_roots(selected)
23         roots = uncached_roots(roots)
24         init_weights(roots)
25
26         flat = flatten(roots)
27         count = len(flat)
28
29         if config.debug:
30                 l = list(flat)
31                 l.sort(compare_name)
32
33                 print 'Building components:'
34                 for c in l:
35                         print '\t' + c.name
36
37                 del l
38
39         del flat
40
41         Builder(roots, count).run()
42
43         unbuilt = []
44         for c in selected:
45                 if c.state != 'done' and not cache.contains(c):
46                         unbuilt.append(c.name)
47
48         if unbuilt:
49                 raise Error('Components not built (dependency cycles?): ' + \
50                             ' '.join(unbuilt))
51
52         if not count:
53                 print 'Nothing to build'
54
55 def build_only(targets):
56         all = [components.by_name[i] for i in targets]
57         count = len(all)
58
59         # Set weights based on user-specified order.  Weight recalculation
60         # won't happen because the reverse dependency tree is flat.
61         #
62         # NOTE: config.jobs can still cause parallel builds!
63         #
64         weight = count
65         for c in all:
66                 c.weight = weight
67                 weight -= 1
68
69         Builder(all, count).run()
70
71 def initial_roots(selected):
72         return [c for c in selected if not c.get_depends()]
73
74 def uncached_roots(old_roots):
75         roots = set()
76         count = [0]
77
78         def add_roots(c):
79                 if not config.debug:
80                         count[0] += 1
81                         print '\rChecking cache:', count[0],
82                         sys.stdout.flush()
83
84                 if cache.contains(c):
85                         for child in c.get_rdepends():
86                                 if child.remove_depend(c):
87                                         add_roots(child)
88
89                         c.clear_rdepends()
90                 else:
91                         roots.add(c)
92
93         try:
94                 for c in old_roots:
95                         add_roots(c)
96         finally:
97                 if not config.debug:
98                         print
99
100         return list(roots)
101
102 def init_weights(rdepends):
103         total = 0
104
105         for c in rdepends:
106                 c.weight = 1 + init_weights(c.get_rdepends())
107                 total += c.weight
108
109         return total
110
111 def flatten(rdepends, all=set()):
112         for c in rdepends:
113                 all.add(c)
114                 flatten(c.get_rdepends(), all)
115
116         return all
117
118 def compare_name(a, b):
119         return cmp(a.name, b.name)
120
121 def compare_weight(a, b):
122         return cmp(a.weight, b.weight)
123
124 class Builder(object):
125         def __init__(self, roots, count):
126                 self.jobs = {}
127
128                 self.wait_build = roots
129                 self.wait_build.sort(compare_weight)
130
131                 self.wait_install = []
132                 self.in_install = None
133
134                 self.progress = Progress(count)
135
136                 self.error = False
137
138         def run(self):
139                 try:
140                         sys.stdout = self.progress
141                         try:
142                                 while self.run_iteration():
143                                         pass
144                         finally:
145                                 sys.stdout = self.progress.file
146                 except:
147                         for pid in self.jobs:
148                                 c = self.jobs[pid]
149                                 c.state = 'killed'
150
151                                 print 'Aborting', c
152
153                                 try:
154                                         os.kill(pid, signal.SIGTERM)
155                                 except OSError, e:
156                                         log.error('kill: %s' % e)
157
158                         while self.jobs:
159                                 self.wait()
160
161                         raise
162
163                 if self.error:
164                         raise Error()
165
166         def run_iteration(self):
167                 if self.can_start_install():
168                         c = self.wait_install.pop(0)
169                         self.start_install(c)
170                         self.in_install = c
171
172                 while self.can_start_build():
173                         c = self.wait_build.pop()
174                         self.start_build(c)
175
176                 if self.jobs:
177                         self.wait()
178                         return True
179                 else:
180                         return False
181
182         def can_start_build(self):
183                 return (not self.error or config.keep_going) and \
184                        self.wait_build and self.can_start_job()
185
186         def can_start_install(self):
187                 return not self.in_install and self.wait_install and \
188                        self.can_start_job()
189
190         def can_start_job(self):
191                 return len(self.jobs) < config.jobs
192
193         def start_build(self, c):
194                 if not c.source.exists():
195                         c.source.clone()
196
197                 print 'Building', c
198                 self.start_job(c, 'build')
199
200         def start_install(self, c):
201                 print 'Installing', c
202                 self.start_job(c, 'install')
203
204         def start_job(self, c, action):
205                 c.state = 'in-' + action
206                 pid = spawn(c, action)
207                 self.jobs[pid] = c
208
209         def wait(self):
210                 pid, status = os.wait()
211
212                 if status:
213                         self.error = True
214                         if config.keep_going:
215                                 self.progress.clear_max()
216
217                 c = self.jobs[pid]
218                 del self.jobs[pid]
219
220                 if c.state == 'in-build':
221                         if status:
222                                 print 'Failed', log_path(c, 'build')
223                                 log.error('Failed to build %s' % c.name)
224                                 return
225
226                         if config.debug:
227                                 print 'Built', c
228
229                         c.state = 'wait-install'
230                         self.wait_install.append(c)
231
232                 elif c.state == 'in-install':
233                         assert c == self.in_install
234
235                         if status:
236                                 print 'Failed', log_path(c, 'install')
237                                 log.error('Failed to install %s' % c.name)
238                                 return
239
240                         self.progress.increment()
241                         print 'Done', c
242
243                         c.state = 'done'
244                         self.in_install = None
245
246                         cache.update(c)
247
248                         wait_build_changed = False
249                         for child in c.get_rdepends():
250                                 if child.remove_depend(c):
251                                         self.wait_build.append(child)
252                                         wait_build_changed = True
253
254                         c.clear_rdepends()
255
256                         if wait_build_changed:
257                                 self.wait_build.sort(compare_weight)
258
259                 elif c.state != 'killed':
260                         raise InternalError('Unexpected state: %s' % c.state)
261
262 class Progress(object):
263         def __init__(self, max):
264                 self.now = 0
265                 self.file = sys.stdout
266
267                 self.max_width = len(str(max))
268                 self.format = '%%%dd/%d ' % (self.max_width, max)
269
270                 self.newline = True
271
272         def clear_max(self):
273                 self.format = '%%%dd %s ' % (self.max_width, ' ' * self.max_width)
274
275         def increment(self):
276                 self.now += 1
277
278         def write(self, str):
279                 progress = self.format % self.now
280
281                 while str:
282                         if self.newline:
283                                 self.file.write(progress)
284
285                         index = str.find('\n')
286                         if index < 0:
287                                 if self.newline and str.find(' ') < 0:
288                                         str = '%-10s' % str
289
290                                 self.file.write(str)
291                                 self.newline = False
292                                 return
293
294                         self.file.write(str[:index+1])
295                         str = str[index+1:]
296                         self.newline = True
297
298 def spawn(c, action):
299         board = config.boards[config.board]
300
301         script = os.path.join(config.script_dir, 'run.sh')
302         workdir = os.path.join(config.top_dir, 'src', c.name)
303         log = log_path(c, action)
304
305         args = ['bash', script]
306
307         env = dict(
308                 MATRIX_TOPDIR      = os.path.abspath(config.top_dir),
309                 MATRIX_SCRIPTDIR   = os.path.abspath(config.script_dir),
310                 MATRIX_WORKDIR     = os.path.abspath(workdir),
311                 MATRIX_LOG         = os.path.abspath(log),
312                 MATRIX_JOBS        = str(config.make_jobs),
313                 MATRIX_ACTION      = action,
314                 MATRIX_COMPONENT   = c.name,
315                 MATRIX_ARCH        = board.arch,
316                 MATRIX_GCC_MARCH   = board.gcc_march,
317                 MATRIX_GCC_MCPU    = board.gcc_mcpu,
318                 MATRIX_GCC_MFPU    = board.gcc_mfpu,
319                 MATRIX_GCC_OPTIONS = ' '.join(board.gcc_options),
320                 MATRIX_GNU_HOST    = board.gnu_host,
321         )
322
323         if config.sb2_target:
324                 if not config.sb2_compiler:
325                         raise Error('SB2 compiler path not set')
326
327                 init_options = ' '.join(config.sb2_init_options)
328
329                 env.update(dict(
330                         MATRIX_SB2_TARGET       = config.sb2_target,
331                         MATRIX_SB2_COMPILER     = config.sb2_compiler,
332                         MATRIX_SB2_INIT_OPTIONS = init_options,
333                 ))
334         else:
335                 if config.sb2_compiler:
336                         raise Error('SB2 target name not set')
337
338         for flag in config.flags:
339                 env[flag] = '1'
340
341         if config.verbose:
342                 env['MATRIX_VERBOSE'] = '1'
343
344         env.update(os.environ)
345
346         if config.debug:
347                 print 'Executing:', ' '.join(args)
348                 print 'Variables:'
349                 l = [key for key in env if key.startswith('MATRIX_')]
350                 l.sort()
351                 for key in l:
352                         print '\t%-18s = "%s"' % (key, env[key])
353
354         return os.spawnvpe(os.P_NOWAIT, args[0], args, env)
355
356 def log_path(c, type):
357         return os.path.join(c.meta.path, '%s.log' % type)