move build logic and state into a class
authorTimo Savola <tsavola@movial.fi>
Sat, 5 Apr 2008 21:57:45 +0000 (00:57 +0300)
committerTimo Savola <tsavola@movial.fi>
Sat, 5 Apr 2008 21:59:32 +0000 (00:59 +0300)
matrix/matrix.py

index 34f924b..133ae36 100644 (file)
@@ -7,6 +7,7 @@
 import glob
 import os
 import re
+import signal
 import sys
 import tarfile
 from sets import Set as set
@@ -404,57 +405,177 @@ def build_components(targets, build_jobs=1, make_jobs=1):
                for c in components:
                        print '\t' + c.name
 
-       jobs = {}
-       built = []
-       installing = None
+       builder = Builder(components, build_jobs, make_jobs)
+       builder.run()
 
-       while True:
-               if not installing and built and len(jobs) < build_jobs:
-                       c = built.pop(0)
-                       print 'Installing', c.name
-                       start_install_job(c, jobs)
-                       installing = c
+class Builder(object):
+       def __init__(self, components, max_jobs, max_make_jobs):
+               self.max_jobs = max_jobs
+               self.max_make_jobs = max_make_jobs
 
-               found = False
-               for c in components:
-                       if len(jobs) >= build_jobs:
+               self.jobs = {}
+
+               self.wait_build = set(components)
+               self.wait_install = []
+               self.in_install = None
+
+               self.progress_now = 0
+               self.progress_total = len(components) * 2
+
+               self.error = False
+
+       def run(self):
+               try:
+                       while self.run_iteration():
+                               pass
+               except:
+                       for pid in self.jobs:
+                               try:
+                                       os.kill(pid, signal.SIGTERM)
+                               except OSError, e:
+                                       log_error('kill: %s' % e)
+
+                               c = self.jobs[pid]
+                               c.active_state = 'killed'
+
+                       while self.jobs:
+                               self.wait()
+
+                       raise
+
+       def run_iteration(self):
+               if self.can_start_install():
+                       c = self.wait_install.pop(0)
+                       self.start_install(c)
+                       self.in_install = c
+
+               while self.can_start_build():
+                       found = False
+
+                       for c in self.wait_build:
+                               if self.is_buildable(c):
+                                       found = True
+
+                                       if not self.try_cache(c):
+                                               self.start_build(c)
+
+                                       self.wait_build.remove(c)
+                                       break
+
+                       if not found:
                                break
 
-                       if component_buildable(c):
-                               found = True
-
-                               if not component_cached(c):
-                                       print 'Building', c.name
-                                       start_build_job(c, jobs, make_jobs)
-                               elif config.debug:
-                                       print 'Component', c.name, \
-                                             'found from cache'
-                                       c.active_state = 'installed'
-
-               if not found and not jobs:
-                       for c in components:
-                               if c.active_state != 'installed':
-                                       raise Error('Internal error')
-                       break
-
-               c = wait_for_job(jobs, built)
-               if c is not None and c == installing:
-                       installing = None
-
-def component_buildable(c):
-       if c.active_state:
-               if config.debug:
-                       print 'Component', c.name, 'is in state:', c.active_state
-               return False
+               if self.jobs:
+                       self.wait()
+                       return True
+               else:
+                       if self.wait_build and not self.error:
+                               raise Error('Circular dependencies?')
+                       return False
 
-       for dep in c.active_depends:
-               if config.components[dep.name].active_state != 'installed':
+       def progress(self, increment=1):
+               self.progress_now += increment
+
+       def message(self, arg1, arg2=None):
+               if arg2 is None:
+                       message = arg1
+               else:
+                       message = '%-10s %s' % (arg1, arg2)
+
+               progress = '%3d%%' % \
+                          (self.progress_now *100 / self.progress_total)
+
+               print progress, message
+
+       def is_buildable(self, c):
+               if c.active_state:
                        if config.debug:
-                               print 'Component', c.name, \
-                                     'depends on uninstalled', dep.name
+                               self.message('Component %s is in state: %s' % \
+                                       (c.name, c.active_state))
                        return False
 
-       return True
+               for dep in c.active_depends:
+                       if config.components[dep.name].active_state != 'done':
+                               if config.debug:
+                                       self.message('Component %s depends ' \
+                                               'on uninstalled %s' % \
+                                               (c.name, dep.name))
+                               return False
+
+               return True
+
+       def try_cache(self, c):
+               if not component_cached(c):
+                       return False
+
+               c.active_state = 'done'
+               self.progress(2)
+
+               if config.debug:
+                       self.message('Cached', c.repo.path)
+
+               return True
+
+       def can_start_build(self):
+               return self.can_start_job()
+
+       def can_start_install(self):
+               return not self.in_install and self.wait_install and \
+                      self.can_start_job()
+
+       def can_start_job(self):
+               return not self.error and len(self.jobs) < self.max_jobs
+
+       def start_build(self, c):
+               self.message('Building', c.repo.path)
+               self.start_job(c, 'build_matrix_component', 'in-build')
+
+       def start_install(self, c):
+               self.message('Installing', c.repo.path)
+               self.start_job(c, 'install_matrix_component', 'in-install')
+
+       def start_job(self, c, make_target, state):
+               pid = spawn_make(c, self.max_make_jobs, make_target)
+               self.jobs[pid] = c
+               c.active_state = state
+
+       def wait(self):
+               pid, status = os.wait()
+
+               if status:
+                       self.error = True
+
+               c = self.jobs[pid]
+               del self.jobs[pid]
+
+               if c.active_state == 'in-build':
+                       if status:
+                               log_error('Failed to build %s' % c.name)
+                               return
+
+                       self.progress()
+                       if config.debug:
+                               self.message('Built', c.repo.path)
+
+                       c.active_state = 'wait-install'
+                       self.wait_install.append(c)
+
+               elif c.active_state == 'in-install':
+                       assert c == self.in_install
+
+                       if status:
+                               log_error('Failed to install %s' % c.name)
+                               return
+
+                       self.progress()
+                       self.message('Finished', c.repo.path)
+
+                       c.active_state = 'done'
+                       self.in_install = None
+                       update_cache(c)
+
+               elif c.active_state != 'killed':
+                       raise Error('Unexpected state: %s' % c.active_state)
 
 def component_cached(c):
        if c.from_platform:
@@ -482,7 +603,8 @@ def component_cached(c):
        hash, flagstring = match.groups()
 
        if hash != get_component_hash(c):
-               print 'Component has been changed:', c.name
+               if config.debug:
+                       print 'Component has been changed:', c.name
                return False
 
        if flagstring:
@@ -491,7 +613,8 @@ def component_cached(c):
                flags = []
 
        if set(flags) != set(c.flags):
-               print 'Component flags have been changed:', c.name
+               if config.debug:
+                       print 'Component flags have been changed:', c.name
                return False
 
        return True
@@ -521,14 +644,14 @@ def get_repository_hash(repo):
                repo.active_hash = git.rev_parse(repo.path, 'HEAD')
        return repo.active_hash
 
-def spawn_job(c, make_jobs, target):
+def spawn_make(c, jobs, target):
        board = config.boards[config.board]
 
+       make = os.getenv('MAKE', 'make')
        makefile = os.path.join(config.script_dir, 'matrix.mak')
-
        workdir = os.path.join(config.top_dir, 'src', c.name)
-       args = ['make', '--no-print-directory', '-f', makefile, '-C', workdir,
-               '-j', str(make_jobs), target,
+       args = [make, '--no-print-directory', '-f', makefile, '-C', workdir,
+               '-j', str(jobs), target,
                'MATRIX_TOPDIR='    + os.path.abspath(config.top_dir),
                'MATRIX_SCRIPTDIR=' + os.path.abspath(config.script_dir),
                'MATRIX_COMPONENT=' + c.name,
@@ -551,50 +674,6 @@ def spawn_job(c, make_jobs, target):
 
        return os.spawnvp(os.P_NOWAIT, args[0], args)
 
-def start_build_job(c, jobs, make_jobs):
-       pid = spawn_job(c, make_jobs, 'build_matrix_component')
-       c.active_state = 'building'
-       jobs[pid] = c
-
-def start_install_job(c, jobs):
-       pid = spawn_job(c, 1, 'install_matrix_component')
-       c.active_state = 'installing'
-       jobs[pid] = c
-
-def wait_for_job(jobs, built):
-       if not jobs:
-               return None
-
-       pid, status = os.wait()
-
-       c = jobs.get(pid)
-       if not c:
-               return None
-
-       del jobs[pid]
-
-       if status == 0:
-               if c.active_state == 'building':
-                       print 'Built', c.repo.path
-                       c.active_state = 'built'
-                       built.append(c)
-               elif c.active_state == 'installing':
-                       print 'Installed', c.repo.path
-                       c.active_state = 'installed'
-                       update_cache(c)
-               else:
-                       raise Error('Internal error')
-       else:
-               c.active_state = 'error'
-               print >>sys.stderr, c.name, 'failed with status:', status
-
-               while jobs:
-                       wait_for_job(jobs, built)
-
-               raise Error('Build failed')
-
-       return c
-
 def clean_components(targets):
        if not targets:
                targets = config.components.keys()