find dependencies automatically
authorTimo Savola <tsavola@movial.fi>
Wed, 6 Aug 2008 14:31:14 +0000 (17:31 +0300)
committerTimo Savola <tsavola@movial.fi>
Wed, 6 Aug 2008 14:31:14 +0000 (17:31 +0300)
    * all components don't have to be declared in config via Component()

    * the automatic dependency resolution only works for components which
      provide the default package (= single package with the same name as
      the component)

    * custom settings (such as component-specific branch or rank) still
      need to be declared explicitly using Component()

matrix/build.py
matrix/components.py
matrix/graph.py
matrix/matrix.py
matrix/rootfs.py

index 0621240..aa07ec1 100644 (file)
@@ -15,7 +15,7 @@ Error = RuntimeError
 InternalError = Exception
 
 def build(targets):
-       selected = set([config.components[i] for i in targets])
+       selected = set([components.by_name[i] for i in targets])
        components.fill_in_depends(selected)
        components.init_rdepends(selected)
 
@@ -53,7 +53,7 @@ def build(targets):
                print 'Nothing to build'
 
 def build_only(targets):
-       all = [config.components[i] for i in targets]
+       all = [components.by_name[i] for i in targets]
        count = len(all)
 
        # Set weights based on user-specified order.  Weight recalculation
index cc11c0c..7f67037 100644 (file)
@@ -86,21 +86,145 @@ class PlatformProvidedComponent(Component):
 
        cached = True
 
-def init():
-       targets = []
-       packages = {}
+by_name = None
 
-       for c in config.components.itervalues():
-               if not c.meta.exists():
-                       c.meta.clone()
+def init(targets=[]):
+       global by_name
+       by_name = Resolver().resolve(targets)
+
+class Resolver(object):
+       def __init__(self):
+               self.components = {}
+               self.packages = {}
+
+       def resolve(self, targets):
+               # add all manually specified components before dependencies so
+               # that all custom Component() arguments will come into effect
+               #
+               for c in config.components.itervalues():
+                       self.add_component(c)
+
+               # iterate over a _copy_ of the _current_ package list;
+               # automatic components' dependencies will be initialized
+               # recursively as we go.
+               #
+               for p in self.packages.values():
+                       self.init_package_depends(p)
+
+               for name in targets:
+                       if name not in self.components:
+                               self.add_automatic_component(name)
+
+               self.init_rank_depends()
+               self.check_depends()
 
-               init_packages(c, targets, packages)
+               return self.components
 
-       init_depends(packages)
-       init_rank_depends()
+       def add_component(self, c):
+               if not c.meta.exists():
+                       c.meta.clone()
 
-       targets.sort()
-       return targets
+               # custom packages?
+               #
+               for path in glob(os.path.join(c.meta.path, '*.package')):
+                       name = os.path.basename(path)[:-8]
+                       self.add_package(c, name, path)
+
+                       if config.debug:
+                               print 'Component', c.name, 'provides', name
+
+               # default package?
+               #
+               if not c.packages:
+                       path = os.path.join(c.meta.path, 'info')
+                       if os.path.exists(path):
+                               self.add_package(c, c.name, path)
+                       else:
+                               self.add_package(c, c.name)
+
+               self.components[c.name] = c
+
+       def add_package(self, c, name, path=None):
+               p = parse_package(name, c, path)
+               if p:
+                       c.packages[name] = p
+                       self.packages[name] = p
+
+       def init_package_depends(self, p):
+               for spec in to_seq(p.depends):
+                       depname = Dependency(spec).name
+                       deppkg = self.packages.get(depname)
+
+                       depcomp = None
+                       if deppkg:
+                               if deppkg.component != p.component:
+                                       depcomp = deppkg.component
+                       else:
+                               depcomp = self.add_automatic_component(depname)
+                               if not depcomp:
+                                       log.error('Package %s depends on ' \
+                                                 'non-existent package %s' % \
+                                                 (p.name, depname))
+
+                       if depcomp:
+                               p.component.add_depend(depcomp)
+
+       def add_automatic_component(self, name):
+               if config.debug:
+                       print 'Looking for automatic component:', name
+
+               c = Component(name)
+               if not c.meta.exists():
+                       try:
+                               c.meta.clone()
+                       except Error, e:
+                               log.error(str(e))
+                               return None
+
+               self.add_component(c)
+
+               for p in c.packages.values():
+                       self.init_package_depends(p)
+
+               return c
+
+       def init_rank_depends(self):
+               by_rank = {}
+               for c in self.components.itervalues():
+                       l = by_rank.get(c.rank)
+                       if l is None:
+                               l = []
+                               by_rank[c.rank] = l
+                       l.append(c)
+
+               ranks = by_rank.keys()
+               ranks.sort()
+
+               for i in xrange(1, len(ranks)):
+                       curr_rank = ranks[i]
+                       for curr_comp in by_rank[curr_rank]:
+                               for j in xrange(i):
+                                       prev_rank = ranks[j]
+                                       for prev_comp in by_rank[prev_rank]:
+                                               curr_comp.add_depend(prev_comp)
+
+       def check_depends(self):
+               fail = False
+               for p in self.packages.itervalues():
+                       for spec in to_seq(p.depends):
+                               if not Dependency(spec).check(self.packages):
+                                       fail = True
+                                       log.error('Dependency %s failed for %s' % \
+                                                 (spec, p.name))
+
+                       for spec in to_seq(p.conflicts):
+                               if Dependency(spec).check(self.packages):
+                                       fail = True
+                                       log.error('Package %s conflicts with %s' % \
+                                                 (p.name, spec))
+
+               if fail:
+                       raise Error('Invalid component tree')
 
 class Package(object):
        def __init__(self, name, component):
@@ -118,41 +242,18 @@ def to_seq(value):
                return value.split()
        return value
 
-def init_packages(c, targets, packages):
-       # custom/multiple packages
-       for path in glob(os.path.join(c.meta.path, '*.package')):
-               name = os.path.basename(path)[:-8]
-               add_package(c, packages, name, path)
-
-       if not c.packages:
-               # default package
-               path = os.path.join(c.meta.path, 'info')
-               if os.path.exists(path):
-                       add_package(c, packages, c.name, path)
-       elif config.debug:
-               print 'Component', c.name, 'provides:', ' '.join(c.packages)
-
-       if c.packages:
-               targets.append(c.name)
-       elif config.debug:
-               print 'Component', c.name, 'does not provide any packages'
-
-def add_package(c, packages, name, path):
-       pkg = parse_package(name, c, path)
-       if pkg:
-               c.packages[name] = pkg
-               packages[name] = pkg
-
-def parse_package(name, component, path):
-       pkg = Package(name, component)
-       execfile(path, pkg.__dict__, pkg.__dict__)
-
-       if pkg.architectures:
+def parse_package(name, component, path=None):
+       p = Package(name, component)
+
+       if path:
+               execfile(path, p.__dict__, p.__dict__)
+
+       if p.architectures:
                arch = config.boards[config.board].arch
-               if arch not in pkg.architectures:
+               if arch not in p.architectures:
                        return None
 
-       return pkg
+       return p
 
 class Dependency(object):
        regex = re.compile(r'([@]?)([^\s:]+)[:]?([<>=]*)([^\s:]*)[:]?(.*)')
@@ -170,60 +271,6 @@ class Dependency(object):
                # TODO: check version and flags
                return self.name in packages
 
-def init_depends(packages):
-       for pkg in packages.itervalues():
-               for spec in to_seq(pkg.depends):
-                       depname = Dependency(spec).name
-                       deppkg = packages.get(depname)
-
-                       if not deppkg:
-                               log.error('Package %s depends on ' \
-                                         'non-existent package %s' % \
-                                         (pkg.name, depname))
-                               continue
-
-                       if deppkg.component == pkg.component:
-                               continue
-
-                       pkg.component.add_depend(deppkg.component)
-
-       fail = False
-       for pkg in packages.itervalues():
-               for spec in to_seq(pkg.depends):
-                       if not Dependency(spec).check(packages):
-                               fail = True
-                               log.error('Dependency %s failed for %s' % \
-                                         (spec, pkg.name))
-
-               for spec in to_seq(pkg.conflicts):
-                       if Dependency(spec).check(packages):
-                               fail = True
-                               log.error('Package %s conflicts with %s' % \
-                                         (pkg.name, spec))
-
-       if fail:
-               raise Error('Invalid component tree')
-
-def init_rank_depends():
-       components_by_rank = {}
-       for c in config.components.itervalues():
-               l = components_by_rank.get(c.rank)
-               if l is None:
-                       l = []
-                       components_by_rank[c.rank] = l
-               l.append(c)
-
-       ranks = list(components_by_rank.keys())
-       ranks.sort()
-
-       for i in xrange(1, len(ranks)):
-               curr_rank = ranks[i]
-               for curr_comp in components_by_rank[curr_rank]:
-                       for j in xrange(i):
-                               prev_rank = ranks[j]
-                               for prev_comp in components_by_rank[prev_rank]:
-                                       curr_comp.add_depend(prev_comp)
-
 def fill_in_depends(components):
        depends = True
        while depends:
index f77d282..54bbdc5 100644 (file)
@@ -80,7 +80,7 @@ class Graph(object):
                roots = self.get_roots()
 
                depth = 1
-               for component in config.components.values():
+               for component in components.by_name.values():
                        if roots is None or component.name in roots:
 
                                self.add_component(component, self.maxdepth, depth)
@@ -195,7 +195,7 @@ class ComponentGraph(Graph):
                Graph.__init__(self, arguments)
 
                if self.roots is not None:
-                       for component in config.components.values():
+                       for component in components.by_name.values():
                                if component.name in self.roots:
                                        self.all_deps.append(self.depends(None, component))
                                        break
index 33e25f5..6869879 100644 (file)
@@ -37,7 +37,7 @@ def main():
                merge_config(name, options[name])
 
        cache.init()
-       all_targets = components.init()
+       components.init(targets)
 
        for i in xrange(len(targets)):
                name = targets[i]
@@ -45,11 +45,12 @@ def main():
                        name = name.split(os.path.sep)[1]
                        targets[i] = name
 
-               if name not in config.components:
+               if name not in components.by_name:
                        raise Error('Component "%s" does not exist' % name)
 
        if not targets:
-               targets = all_targets
+               targets = components.by_name.keys()
+               targets.sort()
 
        command_funcs = {
                'install':      build,
@@ -188,13 +189,13 @@ def remove_tree(path):
 
 def meta(targets):
        for name in targets:
-               c = config.components[name]
+               c = components.by_name[name]
                if not c.meta.exists():
                        c.meta.clone()
        
 def clone(targets):
        for name in targets:
-               c = config.components[name]
+               c = components.by_name[name]
                clone_component(c)
 
 def clone_component(c, overwrite=False):
@@ -220,7 +221,7 @@ def clean(targets):
        changed = False
 
        for name in targets:
-               c = config.components[name]
+               c = components.by_name[name]
                c.source.clean()
                cache.remove(c)
 
@@ -234,7 +235,7 @@ def for_each_repository(func, targets=None):
        ret = None
 
        for name in targets:
-               c = config.components[name]
+               c = components.by_name[name]
                if c.source.exists():
                        value = func(c.source)
                        if value:
@@ -265,7 +266,7 @@ def changes(targets):
        changed = False
 
        for name in targets:
-               c = config.components[name]
+               c = components.by_name[name]
 
                if c.source.exists():
                        if check_changes(c.source):
@@ -298,7 +299,7 @@ def source_dist(targets):
                os.makedirs(location)
 
        for name in targets:
-               c = config.components[name]
+               c = components.by_name[name]
                dist_changes(c, location)
                dist_sources(c, location)
 
index e6c7d73..0dc9a2f 100644 (file)
@@ -324,10 +324,7 @@ class Builder(object):
                        os.remove("/tmp/env.faked")
 
                fakedbs = []
-               for c in config.components.values():
-                       path = os.path.join(c.meta.path, 'fakedb')
-                       if os.path.exists(path):
-                               fakedbs.append(path)
+               # TODO: find fakedbs
 
                if fakedbs:
                        os.system('cat >/tmp/env.faked ' + ' '.join(fakedbs))