9df152e2541ed2e91193689ea9bbf5f6f8ba69da
[matrix.git] / matrix / build.py
1 # Copyright (C) 2006-2008 Movial Creative Technologies Inc.
2 # Authors: Timo Savola
3
4 import os
5 import signal
6 import sys
7 from sets import Set as set
8
9 import cache
10 import components
11 import log
12 from config import config
13
14 Error = RuntimeError
15 InternalError = Exception
16
17 def build(targets):
18         try:
19                 selected = set([components.by_name[i] for i in targets])
20         except KeyError, e:
21                 raise Error("Component " + i + " not found")
22
23         components.fill_in_depends(selected)
24
25         count = 0
26
27         print 'Calculating dependencies...',
28         sys.stdout.flush()
29
30         for c in selected:
31                 if analyze_build(c):
32                         count += 1
33         
34         print 'done, rebuilding', count, 'components'
35
36         Builder(selected, count).run()
37
38         unbuilt = []
39         for c in selected:
40                 if c.state != 'done' and not cache.contains(c):
41                         unbuilt.append(c.name)
42
43         if unbuilt:
44                 raise Error('Components not built (dependency cycles?): ' + \
45                             ' '.join(unbuilt))
46
47         if not count:
48                 print 'Nothing to build'
49
50 def build_only(targets):
51         try:
52                 all = [components.by_name[i] for i in targets]
53         except KeyError:
54                 raise Error('Component ' + i + ' not found')
55         count = len(all)
56
57         for c in all:
58                 c.needs_rebuild = True
59
60         Builder(all, count).run()
61
62 def analyze_build(c):
63         if not c.rebuild_checked:
64                 if not cache.contains(c):
65                         c.needs_rebuild = True
66                 else:
67                         for d in c.get_depends():
68                                 if analyze_build(d):
69                                         c.needs_rebuild = True
70
71                 c.rebuild_checked = True
72
73         return c.needs_rebuild
74
75 class Builder(object):
76         def __init__(self, components, count):
77                 self.jobs = {}
78
79                 self.components = components
80
81                 self.wait_install = []
82                 self.in_install = None
83
84                 self.progress = Progress(count)
85
86                 self.error = False
87
88         def run(self):
89                 try:
90                         sys.stdout = self.progress
91                         try:
92                                 while self.run_iteration():
93                                         pass
94                         finally:
95                                 sys.stdout = self.progress.file
96                 except:
97                         for pid in self.jobs:
98                                 c = self.jobs[pid]
99                                 c.state = 'killed'
100
101                                 print 'Aborting', c
102
103                                 try:
104                                         os.kill(pid, signal.SIGTERM)
105                                 except OSError, e:
106                                         log.error('kill: %s' % e)
107
108                         while self.jobs:
109                                 self.wait()
110
111                         raise
112
113                 if self.error:
114                         raise Error()
115
116         def run_iteration(self):
117                 if self.can_start_install():
118                         c = self.wait_install.pop(0)
119                         self.start_install(c)
120                         self.in_install = c
121
122                 while self.can_start_build():
123                         c = self.find_build_job(self.components)
124                         if not c:
125                                 break
126                         self.start_build(c)
127
128                 if self.jobs:
129                         self.wait()
130                         return True
131                 else:
132                         return False
133
134         def can_start_build(self):
135                 return (not self.error or config.keep_going) and self.can_start_job()
136
137         def can_start_install(self):
138                 return not self.in_install and self.wait_install
139
140         def can_start_job(self):
141                 return len(self.jobs) < config.jobs
142
143         def check_depends(self, components):
144                 for c in components:
145                         if c.needs_rebuild and c.state != 'done':
146                                 return False
147
148                 return True
149
150
151         def find_build_job(self, components):
152                 for c in components:
153                         if not c.needs_rebuild or c.state:
154                                 continue
155
156                         if not self.check_depends(c.get_depends()):
157                                 continue
158
159                         return c
160
161                 return None
162
163         def start_build(self, c):
164                 if not c.source.exists():
165                         c.source.clone()
166
167                 print 'Building', c
168                 self.start_job(c, 'build')
169
170         def start_install(self, c):
171                 print 'Installing', c
172                 self.start_job(c, 'install')
173
174         def start_job(self, c, action):
175                 c.state = 'in-' + action
176                 pid = spawn(c, action)
177                 self.jobs[pid] = c
178
179         def wait(self):
180                 pid, status = os.wait()
181
182                 if status:
183                         self.error = True
184                         if config.keep_going:
185                                 self.progress.clear_max()
186
187                 c = self.jobs[pid]
188                 del self.jobs[pid]
189
190                 if c.state == 'in-build':
191                         if status:
192                                 print 'Failed', log_path(c, 'build')
193                                 log.error('Failed to build %s' % c.name)
194                                 return
195
196                         if config.debug:
197                                 print 'Built', c
198
199                         c.state = 'wait-install'
200                         self.wait_install.append(c)
201
202                 elif c.state == 'in-install':
203                         assert c == self.in_install
204
205                         if status:
206                                 print 'Failed', log_path(c, 'install')
207                                 log.error('Failed to install %s' % c.name)
208                                 return
209
210                         self.progress.increment()
211                         print 'Done', c
212
213                         c.state = 'done'
214                         self.in_install = None
215
216                         cache.update(c)
217
218                 elif c.state != 'killed':
219                         raise InternalError('Unexpected state: %s' % c.state)
220
221 class Progress(object):
222         def __init__(self, max):
223                 self.now = 0
224                 self.file = sys.stdout
225
226                 self.max_width = len(str(max))
227                 self.format = '%%%dd/%d ' % (self.max_width, max)
228
229                 self.newline = True
230
231         def clear_max(self):
232                 self.format = '%%%dd %s ' % (self.max_width, ' ' * self.max_width)
233
234         def increment(self):
235                 self.now += 1
236
237         def write(self, str):
238                 progress = self.format % self.now
239
240                 while str:
241                         if self.newline:
242                                 self.file.write(progress)
243
244                         index = str.find('\n')
245                         if index < 0:
246                                 if self.newline and str.find(' ') < 0:
247                                         str = '%-10s' % str
248
249                                 self.file.write(str)
250                                 self.newline = False
251                                 return
252
253                         self.file.write(str[:index+1])
254                         str = str[index+1:]
255                         self.newline = True
256
257 def spawn(c, action):
258         board = config.boards[config.board]
259
260         script = os.path.join(config.script_dir, 'run.sh')
261         workdir = os.path.join(config.top_dir, 'src', c.name)
262         log = log_path(c, action)
263
264         args = ['bash', script]
265
266         env = dict(
267                 MATRIX_TOPDIR      = os.path.abspath(config.top_dir),
268                 MATRIX_SCRIPTDIR   = os.path.abspath(config.script_dir),
269                 MATRIX_WORKDIR     = os.path.abspath(workdir),
270                 MATRIX_LOG         = os.path.abspath(log),
271                 MATRIX_JOBS        = str(config.make_jobs),
272                 MATRIX_ACTION      = action,
273                 MATRIX_COMPONENT   = c.name,
274                 MATRIX_ARCH        = board.arch,
275                 MATRIX_GCC_MARCH   = board.gcc_march,
276                 MATRIX_GCC_MCPU    = board.gcc_mcpu,
277                 MATRIX_GCC_MFPU    = board.gcc_mfpu,
278                 MATRIX_GCC_OPTIONS = ' '.join(board.gcc_options),
279                 MATRIX_GNU_HOST    = board.gnu_host,
280         )
281
282         if config.sb2_target:
283                 if not config.sb2_compiler:
284                         raise Error('SB2 compiler path not set')
285
286                 init_options = ' '.join(config.sb2_init_options)
287
288                 env.update(dict(
289                         MATRIX_SB2_TARGET       = config.sb2_target,
290                         MATRIX_SB2_COMPILER     = config.sb2_compiler,
291                         MATRIX_SB2_INIT_OPTIONS = init_options,
292                 ))
293         else:
294                 if config.sb2_compiler:
295                         raise Error('SB2 target name not set')
296
297         for flag in config.flags:
298                 env[flag] = '1'
299
300         if config.verbose:
301                 env['MATRIX_VERBOSE'] = '1'
302
303         env.update(os.environ)
304
305         if config.debug:
306                 print 'Executing:', ' '.join(args)
307                 print 'Variables:'
308                 l = [key for key in env if key.startswith('MATRIX_')]
309                 l.sort()
310                 for key in l:
311                         print '\t%-18s = "%s"' % (key, env[key])
312
313         return os.spawnvpe(os.P_NOWAIT, args[0], args, env)
314
315 def log_path(c, type):
316         return os.path.join(c.meta.path, '%s.log' % type)