reorganized component and repository code into separate modules
[matrix.git] / matrix / build.py
1 # Copyright (C) 2006-2008 Movial Oy
2 # Authors: Timo Savola <tsavola@movial.fi>
3
4 import os
5 import signal
6 from sets import Set as set
7
8 import cache
9 import config
10 import log
11 import matrix
12
13 Error = RuntimeError
14
15 class Builder(object):
16         def __init__(self, components, max_jobs, max_make_jobs):
17                 self.max_jobs = max_jobs
18                 self.max_make_jobs = max_make_jobs
19
20                 self.jobs = {}
21
22                 self.wait_build = set(components)
23                 self.wait_install = []
24                 self.in_install = None
25
26                 self.progress_now = 0
27                 self.progress_total = len(components) * 2
28
29                 self.error = False
30
31         def run(self):
32                 try:
33                         while self.run_iteration():
34                                 pass
35                 except:
36                         for pid in self.jobs:
37                                 try:
38                                         os.kill(pid, signal.SIGTERM)
39                                 except OSError, e:
40                                         log.error('kill: %s' % e)
41
42                                 c = self.jobs[pid]
43                                 c.state = 'killed'
44
45                         while self.jobs:
46                                 self.wait()
47
48                         raise
49
50                 if self.error:
51                         raise Error()
52
53         def run_iteration(self):
54                 if self.can_start_install():
55                         c = self.wait_install.pop(0)
56                         self.start_install(c)
57                         self.in_install = c
58
59                 while self.can_start_build():
60                         found = False
61
62                         for c in self.wait_build:
63                                 if self.is_buildable(c):
64                                         found = True
65
66                                         if not self.try_cache(c):
67                                                 matrix.clone_component(c)
68                                                 self.start_build(c)
69
70                                         self.wait_build.remove(c)
71                                         break
72
73                         if not found:
74                                 break
75
76                 if self.jobs:
77                         self.wait()
78                         return True
79                 else:
80                         if self.wait_build and not self.error:
81                                 raise Error('Circular dependencies?')
82                         return False
83
84         def progress(self, increment=1):
85                 self.progress_now += increment
86
87         def message(self, arg1, arg2=None):
88                 if arg2 is None:
89                         message = arg1
90                 else:
91                         message = '%-10s %s' % (arg1, arg2)
92
93                 progress = '%3d%%' % \
94                            (self.progress_now *100 / self.progress_total)
95
96                 print progress, message
97
98         def is_buildable(self, c):
99                 if c.state:
100                         if config.debug:
101                                 self.message('Component %s is in state: %s' % \
102                                         (c.name, c.state))
103                         return False
104
105                 for dep in c.depends:
106                         if config.components[dep.name].state != 'done':
107                                 if config.debug:
108                                         self.message('Component %s depends ' \
109                                                 'on uninstalled %s' % \
110                                                 (c.name, dep.name))
111                                 return False
112
113                 return True
114
115         def try_cache(self, c):
116                 if not cache.contains(c):
117                         return False
118
119                 c.state = 'done'
120                 self.progress(2)
121
122                 if config.debug:
123                         self.message('Cached', c.repo.path)
124
125                 return True
126
127         def can_start_build(self):
128                 return self.can_start_job()
129
130         def can_start_install(self):
131                 return not self.in_install and self.wait_install and \
132                        self.can_start_job()
133
134         def can_start_job(self):
135                 return not self.error and len(self.jobs) < self.max_jobs
136
137         def start_build(self, c):
138                 self.message('Building', c.repo.path)
139                 self.start_job(c, 'build_matrix_component', 'in-build')
140
141         def start_install(self, c):
142                 self.message('Installing', c.repo.path)
143                 self.start_job(c, 'install_matrix_component', 'in-install')
144
145         def start_job(self, c, make_target, state):
146                 pid = spawn_make(c, self.max_make_jobs, make_target)
147                 self.jobs[pid] = c
148                 c.state = state
149
150         def wait(self):
151                 pid, status = os.wait()
152
153                 if status:
154                         self.error = True
155
156                 c = self.jobs[pid]
157                 del self.jobs[pid]
158
159                 if c.state == 'in-build':
160                         if status:
161                                 log.error('Failed to build %s' % c.name)
162                                 return
163
164                         self.progress()
165                         if config.debug:
166                                 self.message('Built', c.repo.path)
167
168                         c.state = 'wait-install'
169                         self.wait_install.append(c)
170
171                 elif c.state == 'in-install':
172                         assert c == self.in_install
173
174                         if status:
175                                 log.error('Failed to install %s' % c.name)
176                                 return
177
178                         self.progress()
179                         self.message('Finished', c.repo.path)
180
181                         c.state = 'done'
182                         self.in_install = None
183                         cache.update(c)
184
185                 elif c.state != 'killed':
186                         raise Error('Unexpected state: %s' % c.state)
187
188 def spawn_make(c, jobs, target):
189         board = config.boards[config.board]
190
191         make = os.getenv('MAKE', 'make')
192         makefile = os.path.join(config.script_dir, 'matrix.mak')
193         workdir = os.path.join(config.top_dir, 'src', c.name)
194         args = [make, '--no-print-directory', '-f', makefile, '-C', workdir,
195                 '-j', str(jobs), target,
196                 'MATRIX_TOPDIR='    + os.path.abspath(config.top_dir),
197                 'MATRIX_SCRIPTDIR=' + os.path.abspath(config.script_dir),
198                 'MATRIX_COMPONENT=' + c.name,
199                 'MATRIX_ARCH='      + board.arch,
200                 'MATRIX_GCC_MARCH=' + board.gcc_march,
201                 'MATRIX_GCC_MCPU='  + board.gcc_mcpu,
202                 'MATRIX_GCC_MFPU='  + board.gcc_mfpu,
203                 'MATRIX_GCC_OPTIONS=' + board.gcc_options,
204                 'MATRIX_GNU_HOST='  + board.gnu_host,
205                 'MATRIX_LIBC='      + config.libc]
206
207         for flag in config.flags:
208                 args.append(flag + '=1')
209
210         if config.verbose:
211                 args.append('MATRIX_VERBOSE=1')
212
213         if config.debug:
214                 print 'Executing:', ' '.join(args)
215
216         return os.spawnvp(os.P_NOWAIT, args[0], args)