use Repository.__str__() in messages
[matrix.git] / matrix / build.py
1 # Copyright (C) 2006-2008 Movial Oy
2 # Authors: Timo Savola <tsavola@movial.fi>
3
4 import os
5 import signal
6 import sys
7 from sets import Set as set
8
9 import cache
10 import components
11 import config
12 import log
13
14 Error = RuntimeError
15 InternalError = Exception
16
17 def build(targets, build_jobs=1, make_jobs=1):
18         selected = set([config.components[i] for i in targets])
19         components.fill_in_depends(selected)
20         components.init_rdepends(selected)
21
22         roots = initial_roots(selected)
23         roots = uncached_roots(roots)
24         init_weights(roots)
25
26         flat = flatten(roots)
27         count = len(flat)
28
29         if config.debug:
30                 l = list(flat)
31                 l.sort(key=lambda c: c.name)
32
33                 print 'Building components:'
34                 for c in l:
35                         print '\t' + c.name
36
37                 del l
38
39         del flat
40
41         Builder(roots, count, build_jobs, make_jobs).run()
42
43         unbuilt = []
44         for c in selected:
45                 if c.state != 'done' and not cache.contains(c):
46                         unbuilt.append(c.name)
47
48         if unbuilt:
49                 raise Error('Components not built (dependency cycles?): ' + \
50                             ' '.join(unbuilt))
51
52         if not count:
53                 print 'Nothing to build'
54
55 def initial_roots(selected):
56         return [c for c in selected if not c.get_depends()]
57
58 def uncached_roots(old_roots):
59         roots = set()
60
61         def add_roots(c):
62                 if cache.contains(c):
63                         for child in c.get_rdepends():
64                                 if child.remove_depend(c):
65                                         add_roots(child)
66
67                         c.clear_rdepends()
68                 else:
69                         roots.add(c)
70
71         for c in old_roots:
72                 add_roots(c)
73
74         return list(roots)
75
76 def init_weights(rdepends):
77         total = 0
78
79         for c in rdepends:
80                 c.weight = 1 + init_weights(c.get_rdepends())
81                 total += c.weight
82
83         return total
84
85 def flatten(rdepends, all=set()):
86         for c in rdepends:
87                 all.add(c)
88                 flatten(c.get_rdepends(), all)
89
90         return all
91
92 class Builder(object):
93         def __init__(self, roots, count, max_jobs, max_make_jobs):
94                 self.max_jobs = max_jobs
95                 self.max_make_jobs = max_make_jobs
96
97                 self.jobs = {}
98
99                 self.wait_build = roots
100                 self.wait_install = []
101                 self.in_install = None
102
103                 self.progress = Progress(count * 2)
104
105                 self.error = False
106
107         def run(self):
108                 try:
109                         sys.stdout = self.progress
110                         try:
111                                 while self.run_iteration():
112                                         pass
113                         finally:
114                                 sys.stdout = self.progress.file
115                 except:
116                         for pid in self.jobs:
117                                 try:
118                                         os.kill(pid, signal.SIGTERM)
119                                 except OSError, e:
120                                         log.error('kill: %s' % e)
121
122                                 c = self.jobs[pid]
123                                 c.state = 'killed'
124
125                         while self.jobs:
126                                 self.wait()
127
128                         raise
129
130                 if self.error:
131                         raise Error()
132
133         def run_iteration(self):
134                 if self.can_start_install():
135                         c = self.wait_install.pop(0)
136                         self.start_install(c)
137                         self.in_install = c
138
139                 while self.can_start_build():
140                         c = self.wait_build.pop()
141                         self.start_build(c)
142
143                 if self.jobs:
144                         self.wait()
145                         return True
146                 else:
147                         return False
148
149         def can_start_build(self):
150                 return self.wait_build and self.can_start_job()
151
152         def can_start_install(self):
153                 return not self.in_install and self.wait_install and \
154                        self.can_start_job()
155
156         def can_start_job(self):
157                 return not self.error and len(self.jobs) < self.max_jobs
158
159         def start_build(self, c):
160                 if not c.repo.exists():
161                         c.repo.clone()
162
163                 print 'Building', c.repo
164                 self.start_job(c, 'build_matrix_component', 'in-build')
165
166         def start_install(self, c):
167                 print 'Installing', c.repo
168                 self.start_job(c, 'install_matrix_component', 'in-install')
169
170         def start_job(self, c, make_target, state):
171                 c.state = state
172                 pid = spawn_make(c, self.max_make_jobs, make_target)
173                 self.jobs[pid] = c
174
175         def wait(self):
176                 pid, status = os.wait()
177
178                 if status:
179                         self.error = True
180
181                 c = self.jobs[pid]
182                 del self.jobs[pid]
183
184                 if c.state == 'in-build':
185                         if status:
186                                 log.error('Failed to build %s' % c.name)
187                                 return
188
189                         self.progress.increment()
190                         if config.debug:
191                                 print 'Built', c.repo
192
193                         c.state = 'wait-install'
194                         self.wait_install.append(c)
195
196                         wait_build_changed = False
197                         for child in c.get_rdepends():
198                                 if child.remove_depend(c):
199                                         self.wait_build.append(child)
200                                         wait_build_changed = True
201
202                         c.clear_rdepends()
203
204                         if wait_build_changed:
205                                 self.wait_build.sort(key=lambda c: c.weight)
206
207                 elif c.state == 'in-install':
208                         assert c == self.in_install
209
210                         if status:
211                                 log.error('Failed to install %s' % c.name)
212                                 return
213
214                         self.progress.increment()
215                         print 'Finished', c.repo
216
217                         c.state = 'done'
218                         self.in_install = None
219
220                         cache.update(c)
221
222                 elif c.state != 'killed':
223                         raise InternalError('Unexpected state: %s' % c.state)
224
225 class Progress(object):
226         def __init__(self, max):
227                 self.now = 0
228                 self.max = max
229                 self.file = sys.stdout
230
231                 self.newline = True
232
233         def increment(self):
234                 self.now += 1
235
236         def write(self, str):
237                 progress = '%3d%% ' % (self.now * 100 / self.max)
238
239                 while str:
240                         if self.newline:
241                                 self.file.write(progress)
242
243                         index = str.find('\n')
244                         if index < 0:
245                                 if self.newline and str.find(' ') < 0:
246                                         str = '%-10s' % str
247
248                                 self.file.write(str)
249                                 self.newline = False
250                                 return
251
252                         self.file.write(str[:index+1])
253                         str = str[index+1:]
254                         self.newline = True
255
256 def spawn_make(c, jobs, target):
257         board = config.boards[config.board]
258
259         make = os.getenv('MAKE', 'make')
260         makefile = os.path.join(config.script_dir, 'matrix.mak')
261         workdir = os.path.join(config.top_dir, 'src', c.name)
262         args = [make, '--no-print-directory', '-f', makefile, '-C', workdir,
263                 '-j', str(jobs), target,
264                 'MATRIX_TOPDIR='    + os.path.abspath(config.top_dir),
265                 'MATRIX_SCRIPTDIR=' + os.path.abspath(config.script_dir),
266                 'MATRIX_COMPONENT=' + c.name,
267                 'MATRIX_ARCH='      + board.arch,
268                 'MATRIX_GCC_MARCH=' + board.gcc_march,
269                 'MATRIX_GCC_MCPU='  + board.gcc_mcpu,
270                 'MATRIX_GCC_MFPU='  + board.gcc_mfpu,
271                 'MATRIX_GCC_OPTIONS=' + board.gcc_options,
272                 'MATRIX_GNU_HOST='  + board.gnu_host,
273                 'MATRIX_LIBC='      + config.libc]
274
275         for flag in config.flags:
276                 args.append(flag + '=1')
277
278         if config.verbose:
279                 args.append('MATRIX_VERBOSE=1')
280
281         if config.debug:
282                 print 'Executing:', ' '.join(args)
283
284         return os.spawnvp(os.P_NOWAIT, args[0], args)