Reimplement dependency order handling to solve a performance issue.
[matrix.git] / matrix / build.py
1 # Copyright (C) 2006-2008 Movial Oy
2 # Authors: Timo Savola <tsavola@movial.fi>
3
4 import os
5 import signal
6 import sys
7 from sets import Set as set
8
9 import cache
10 import components
11 import log
12 from config import config
13
14 Error = RuntimeError
15 InternalError = Exception
16
17 def build(targets):
18         try:
19                 selected = set([components.by_name[i] for i in targets])
20         except KeyError, e:
21                 raise Error("Component " + i + " not found")
22
23         components.fill_in_depends(selected)
24
25         print 'Calculating dependencies...',
26         sys.stdout.flush()
27         count = analyze_build(selected)
28         print 'done, rebuilding', count, 'components'
29
30         Builder(selected, count).run()
31
32         unbuilt = []
33         for c in selected:
34                 if c.state != 'done' and not cache.contains(c):
35                         unbuilt.append(c.name)
36
37         if unbuilt:
38                 raise Error('Components not built (dependency cycles?): ' + \
39                             ' '.join(unbuilt))
40
41         if not count:
42                 print 'Nothing to build'
43
44 def build_only(targets):
45         try:
46                 all = [components.by_name[i] for i in targets]
47         except KeyError:
48                 raise Error('Component ' + i + ' not found')
49         count = len(all)
50
51         # Set weights based on user-specified order.  Weight recalculation
52         # won't happen because the reverse dependency tree is flat.
53         #
54         # NOTE: config.jobs can still cause parallel builds!
55         #
56         weight = count
57         for c in all:
58                 c.weight = weight
59                 weight -= 1
60
61         Builder(all, count).run()
62
63 def analyze_build(targets):
64         count = 0
65
66         for c in targets:
67                 if c.rebuild_checked:
68                         continue
69
70                 count += analyze_build(c.get_depends())
71
72                 if count or not cache.contains(c):
73                         c.needs_rebuild = True
74                         count += 1
75         
76                 c.rebuild_checked = True
77         
78         return count
79
80 class Builder(object):
81         def __init__(self, components, count):
82                 self.jobs = {}
83
84                 self.components = components
85
86                 self.wait_install = []
87                 self.in_install = None
88
89                 self.progress = Progress(count)
90
91                 self.error = False
92
93         def run(self):
94                 try:
95                         sys.stdout = self.progress
96                         try:
97                                 while self.run_iteration():
98                                         pass
99                         finally:
100                                 sys.stdout = self.progress.file
101                 except:
102                         for pid in self.jobs:
103                                 c = self.jobs[pid]
104                                 c.state = 'killed'
105
106                                 print 'Aborting', c
107
108                                 try:
109                                         os.kill(pid, signal.SIGTERM)
110                                 except OSError, e:
111                                         log.error('kill: %s' % e)
112
113                         while self.jobs:
114                                 self.wait()
115
116                         raise
117
118                 if self.error:
119                         raise Error()
120
121         def run_iteration(self):
122                 if self.can_start_install():
123                         c = self.wait_install.pop(0)
124                         self.start_install(c)
125                         self.in_install = c
126
127                 while self.can_start_build():
128                         c = self.find_build_job(self.components)
129                         if not c:
130                                 break
131                         self.start_build(c)
132
133                 if self.jobs:
134                         self.wait()
135                         return True
136                 else:
137                         return False
138
139         def can_start_build(self):
140                 return (not self.error or config.keep_going) and self.can_start_job()
141
142         def can_start_install(self):
143                 return not self.in_install and self.wait_install
144
145         def can_start_job(self):
146                 return len(self.jobs) < config.jobs
147
148         def find_build_job(self, components):
149                 for c in components:
150                         if c.state or not c.needs_rebuild:
151                                 continue
152
153                         dep_job = self.find_build_job(c.get_depends())
154
155                         if dep_job:
156                                 return dep_job
157
158                         return c
159
160                 return None
161                                 
162
163         def start_build(self, c):
164                 if not c.source.exists():
165                         c.source.clone()
166
167                 print 'Building', c
168                 self.start_job(c, 'build')
169
170         def start_install(self, c):
171                 print 'Installing', c
172                 self.start_job(c, 'install')
173
174         def start_job(self, c, action):
175                 c.state = 'in-' + action
176                 pid = spawn(c, action)
177                 self.jobs[pid] = c
178
179         def wait(self):
180                 pid, status = os.wait()
181
182                 if status:
183                         self.error = True
184                         if config.keep_going:
185                                 self.progress.clear_max()
186
187                 c = self.jobs[pid]
188                 del self.jobs[pid]
189
190                 if c.state == 'in-build':
191                         if status:
192                                 print 'Failed', log_path(c, 'build')
193                                 log.error('Failed to build %s' % c.name)
194                                 return
195
196                         if config.debug:
197                                 print 'Built', c
198
199                         c.state = 'wait-install'
200                         self.wait_install.append(c)
201
202                 elif c.state == 'in-install':
203                         assert c == self.in_install
204
205                         if status:
206                                 print 'Failed', log_path(c, 'install')
207                                 log.error('Failed to install %s' % c.name)
208                                 return
209
210                         self.progress.increment()
211                         print 'Done', c
212
213                         c.state = 'done'
214                         self.in_install = None
215
216                         cache.update(c)
217
218                 elif c.state != 'killed':
219                         raise InternalError('Unexpected state: %s' % c.state)
220
221 class Progress(object):
222         def __init__(self, max):
223                 self.now = 0
224                 self.file = sys.stdout
225
226                 self.max_width = len(str(max))
227                 self.format = '%%%dd/%d ' % (self.max_width, max)
228
229                 self.newline = True
230
231         def clear_max(self):
232                 self.format = '%%%dd %s ' % (self.max_width, ' ' * self.max_width)
233
234         def increment(self):
235                 self.now += 1
236
237         def write(self, str):
238                 progress = self.format % self.now
239
240                 while str:
241                         if self.newline:
242                                 self.file.write(progress)
243
244                         index = str.find('\n')
245                         if index < 0:
246                                 if self.newline and str.find(' ') < 0:
247                                         str = '%-10s' % str
248
249                                 self.file.write(str)
250                                 self.newline = False
251                                 return
252
253                         self.file.write(str[:index+1])
254                         str = str[index+1:]
255                         self.newline = True
256
257 def spawn(c, action):
258         board = config.boards[config.board]
259
260         script = os.path.join(config.script_dir, 'run.sh')
261         workdir = os.path.join(config.top_dir, 'src', c.name)
262         log = log_path(c, action)
263
264         args = ['bash', script]
265
266         env = dict(
267                 MATRIX_TOPDIR      = os.path.abspath(config.top_dir),
268                 MATRIX_SCRIPTDIR   = os.path.abspath(config.script_dir),
269                 MATRIX_WORKDIR     = os.path.abspath(workdir),
270                 MATRIX_LOG         = os.path.abspath(log),
271                 MATRIX_JOBS        = str(config.make_jobs),
272                 MATRIX_ACTION      = action,
273                 MATRIX_COMPONENT   = c.name,
274                 MATRIX_ARCH        = board.arch,
275                 MATRIX_GCC_MARCH   = board.gcc_march,
276                 MATRIX_GCC_MCPU    = board.gcc_mcpu,
277                 MATRIX_GCC_MFPU    = board.gcc_mfpu,
278                 MATRIX_GCC_OPTIONS = ' '.join(board.gcc_options),
279                 MATRIX_GNU_HOST    = board.gnu_host,
280         )
281
282         if config.sb2_target:
283                 if not config.sb2_compiler:
284                         raise Error('SB2 compiler path not set')
285
286                 init_options = ' '.join(config.sb2_init_options)
287
288                 env.update(dict(
289                         MATRIX_SB2_TARGET       = config.sb2_target,
290                         MATRIX_SB2_COMPILER     = config.sb2_compiler,
291                         MATRIX_SB2_INIT_OPTIONS = init_options,
292                 ))
293         else:
294                 if config.sb2_compiler:
295                         raise Error('SB2 target name not set')
296
297         for flag in config.flags:
298                 env[flag] = '1'
299
300         if config.verbose:
301                 env['MATRIX_VERBOSE'] = '1'
302
303         env.update(os.environ)
304
305         if config.debug:
306                 print 'Executing:', ' '.join(args)
307                 print 'Variables:'
308                 l = [key for key in env if key.startswith('MATRIX_')]
309                 l.sort()
310                 for key in l:
311                         print '\t%-18s = "%s"' % (key, env[key])
312
313         return os.spawnvpe(os.P_NOWAIT, args[0], args, env)
314
315 def log_path(c, type):
316         return os.path.join(c.meta.path, '%s.log' % type)