remove flock dependency by invoking make separately for installation
authorTimo Savola <tsavola@movial.fi>
Fri, 4 Apr 2008 16:44:04 +0000 (19:44 +0300)
committerTimo Savola <tsavola@movial.fi>
Fri, 4 Apr 2008 16:44:04 +0000 (19:44 +0300)
install targets are serialized by the build loop logic instead of blocking
on a lockfile.

matrix/matrix.py
scripts/matrix-autotools.mak
scripts/matrix-common.mak [deleted file]
scripts/matrix-custom.mak
scripts/matrix.mak

index 345f00b..4b5f772 100644 (file)
@@ -385,8 +385,16 @@ def build_components(targets, build_jobs=1, make_jobs=1):
                        print '\t' + c.name
 
        jobs = {}
+       built = []
+       installing = None
 
        while True:
+               if not installing and built and len(jobs) < build_jobs:
+                       c = built.pop(0)
+                       print 'Installing', c.name
+                       start_install_job(c, jobs)
+                       installing = c
+
                found = False
                for c in components:
                        if len(jobs) >= build_jobs:
@@ -397,19 +405,21 @@ def build_components(targets, build_jobs=1, make_jobs=1):
 
                                if not component_cached(c):
                                        print 'Building', c.name
-                                       start_job(c, jobs, make_jobs)
+                                       start_build_job(c, jobs, make_jobs)
                                elif config.debug:
                                        print 'Component', c.name, \
                                              'found from cache'
-                                       c.active_state = 'built'
+                                       c.active_state = 'installed'
 
                if not found and not jobs:
                        for c in components:
-                               if c.active_state != 'built':
+                               if c.active_state != 'installed':
                                        raise Error('Internal error')
                        break
 
-               wait_for_job(jobs)
+               c = wait_for_job(jobs, built)
+               if c is not None and c == installing:
+                       installing = None
 
 def component_buildable(c):
        if c.active_state:
@@ -418,10 +428,10 @@ def component_buildable(c):
                return False
 
        for dep in c.active_depends:
-               if config.components[dep.name].active_state != 'built':
+               if config.components[dep.name].active_state != 'installed':
                        if config.debug:
                                print 'Component', c.name, \
-                                     'depends on unbuilt', dep.name
+                                     'depends on uninstalled', dep.name
                        return False
 
        return True
@@ -491,14 +501,14 @@ def get_repository_hash(repo):
                repo.active_hash = git.rev_parse(repo.path, 'HEAD')
        return repo.active_hash
 
-def start_job(c, jobs, make_jobs):
+def spawn_job(c, make_jobs, target):
        board = config.boards[config.board]
 
        makefile = os.path.join(config.script_dir, 'matrix.mak')
 
        workdir = os.path.join(config.top_dir, 'src', c.name)
        args = ['make', '--no-print-directory', '-f', makefile, '-C', workdir,
-               '-j', str(make_jobs), 'build_matrix_component',
+               '-j', str(make_jobs), target,
                'MATRIX_TOPDIR='    + os.path.abspath(config.top_dir),
                'MATRIX_SCRIPTDIR=' + os.path.abspath(config.script_dir),
                'MATRIX_COMPONENT=' + c.name,
@@ -519,34 +529,52 @@ def start_job(c, jobs, make_jobs):
        if config.debug:
                print 'Executing:', ' '.join(args)
 
-       pid = os.spawnvp(os.P_NOWAIT, args[0], args)
+       return os.spawnvp(os.P_NOWAIT, args[0], args)
+
+def start_build_job(c, jobs, make_jobs):
+       pid = spawn_job(c, make_jobs, 'build_matrix_component')
+       c.active_state = 'building'
+       jobs[pid] = c
 
+def start_install_job(c, jobs):
+       pid = spawn_job(c, 1, 'install_matrix_component')
+       c.active_state = 'installing'
        jobs[pid] = c
-       c.active_state = 'running'
 
-def wait_for_job(jobs):
+def wait_for_job(jobs, built):
        if not jobs:
-               return
+               return None
 
        pid, status = os.wait()
 
        c = jobs.get(pid)
        if not c:
-               return
+               return None
 
        del jobs[pid]
 
        if status == 0:
-               c.active_state = 'built'
-               update_cache(c)
-               print c.name, 'completed'
+               if c.active_state == 'building':
+                       print 'Built', c.repo.path
+                       c.active_state = 'built'
+                       built.append(c)
+               elif c.active_state == 'installing':
+                       print 'Installed', c.repo.path
+                       c.active_state = 'installed'
+                       update_cache(c)
+               else:
+                       raise Error('Internal error')
        else:
                c.active_state = 'error'
                print >>sys.stderr, c.name, 'failed with status:', status
 
-               wait_for_job(jobs)
+               while jobs:
+                       wait_for_job(jobs, built)
+
                raise Error('Build failed')
 
+       return c
+
 def clean_components(targets):
        if not targets:
                targets = config.components.keys()
index 0fcd54b..aa0f8f6 100644 (file)
@@ -8,8 +8,6 @@
 
 __MATRIX_BUILD_STYLE_LOADED=1
 
-include $(MATRIX_SCRIPTDIR)/matrix-common.mak
-
 t := $(shell if [ -d /usr/share/aclocal ]; then echo yes; else echo no; fi)
 ifeq (yes,$(t))
 ACLOCAL_ARGS := -I /usr/share/aclocal $(ACLOCAL_ARGS)
@@ -52,9 +50,12 @@ else
 MATRIX_AUTOGEN = 
 endif
 
-__matrix_build: $(MATRIX_PREPARE_TARGET) $(MATRIX_AUTOGEN) __configure __build  __install_pre_lock
+__matrix_build: $(MATRIX_PREPARE_TARGET) $(MATRIX_AUTOGEN) __configure __build
        @echo "[$(MATRIX_COMPONENT)] Build completed"
 
+__matrix_install: __install
+       @echo "[$(MATRIX_COMPONENT)] Install completed"
+
 __autogen: $(MATRIX_PREPARE_TARGET)
        @$(MATRIX_AUTOGEN_ENV) ./autogen.sh $(MATRIX_AUTOGEN_ARGS)
 
@@ -76,9 +77,8 @@ endif
 
 
 # this can't have any requisites as that would trigger a rebuild
-# __install_pre_lock depends on __build so we're safe here
 
-__install_post_lock:
+__install:
 ifdef MATRIX_OBJ_DIR
        @cd $(MATRIX_OBJ_DIR) && $(MATRIX_INSTALL_ENV) $(CCTOOL_INSTALL) $(MAKE) install $(MATRIX_INSTALL_ARGS)
 else
diff --git a/scripts/matrix-common.mak b/scripts/matrix-common.mak
deleted file mode 100644 (file)
index dfc14c1..0000000
+++ /dev/null
@@ -1,19 +0,0 @@
-# Copyright (C) 2006-2008 Movial Oy
-# Authors: Lauri Leukkunen
-#
-# This provides the necessary stuff to guarantee synchronization
-# of component "install" target execution.
-
-MATRIX_INSTALL_LOCK_FILE = $(MATRIX_TOPDIR)/.matrix-install.lock
-
-# re-run make with install lock
-__install_pre_lock: __build
-       if [ ! -e $(MATRIX_INSTALL_LOCK_FILE) ]; then touch $(MATRIX_INSTALL_LOCK_FILE); fi
-       flock $(MATRIX_INSTALL_LOCK_FILE) $(MAKE) -f $(MATRIX_SCRIPTDIR)/matrix.mak __locked_install
-
-__locked_install:
-       touch $(MATRIX_INSTALL_LOCK_FILE)
-       $(MAKE) -f $(MATRIX_SCRIPTDIR)/matrix.mak __install_post_lock
-       # generate installed files list
-       #find /targets/$$(sb-conf current) -cnewer $(MATRIX_INSTALL_LOCK_FILE) | sed "s/\/targets\/$$(sb-conf current)//" > .matrix/installed_files
-
index b82ac70..6ee2b36 100644 (file)
@@ -4,11 +4,10 @@
 
 __MATRIX_BUILD_STYLE_LOADED=1
 
-.PHONY: configure build install __configure __build
+.PHONY: configure build install __configure __build __install
 
-include $(MATRIX_SCRIPTDIR)/matrix-common.mak
-
-__matrix_build: __configure __build __install_pre_lock
+__matrix_build: __configure __build
+__matrix_install: __install
 
 __configure: configure
        @$(CCTOOL_BUILD) $(MAKE) -f meta/plug configure
@@ -17,6 +16,5 @@ __build: __configure build
        @$(CCTOOL_BUILD) $(MAKE) -f meta/plug build
 
 # this can't have any requisites as that would trigger a rebuild
-# __install_pre_lock depends on __build so we're safe here
-__install_post_lock:
-       @$(matrix_install_lock) $(CCTOOL_INSTALL) $(MAKE) -f meta/plug install
+__install:
+       @$(CCTOOL_INSTALL) $(MAKE) -f meta/plug install
index d9b8de6..0d916a9 100644 (file)
@@ -68,6 +68,14 @@ else
        @$(MAKE) $(MATRIX_MAK_DEBUG) -f $(MATRIX_SCRIPTDIR)/matrix.mak __matrix_build > meta/build.log 2>&1
 endif
 
+# __matrix_install is defined in the included build style
+install_matrix_component:
+ifdef MATRIX_VERBOSE
+       @$(MAKE) $(MATRIX_MAK_DEBUG) -f $(MATRIX_SCRIPTDIR)/matrix.mak __matrix_install
+else
+       @$(MAKE) $(MATRIX_MAK_DEBUG) -f $(MATRIX_SCRIPTDIR)/matrix.mak __matrix_install > meta/install.log 2>&1
+endif
+
 
 matrix_mak_debug:
        @echo MATRIX_TOPDIR=$(MATRIX_TOPDIR)