Implement dependency resolver for Homebrew "tapped" tool shed packages (highly beta).

Homebrew/Linuxbrew can be configured the same way the more vanilla Homebrew resolver works (added in 6b7782f).

When this resolver is active, if the tool has been installed via the Tool Shed and the dependencies also installed it will use the database information to determine the dependent packages - otherwise it will just try to find the tool_dependencies.xml file on disk and recover the dependencies from that.

This is for dependencies installed via Platform Brew (https://github.com/jmchilton/platform-brew) and converted from Tool Shed repositories usiing the shed2tap WIP code (e.g. https://github.com/jmchilton/homebrew-toolshed).

lib/galaxy/tools/__init__.py

@@ -1843,7 +1843,8 @@ class Tool( object, Dictifiable ):
         """Return a list of commands to be run to populate the current environment to include this tools requirements."""
         return self.app.toolbox.dependency_manager.dependency_shell_commands(
             self.requirements,
-            installed_tool_dependencies=self.installed_tool_dependencies
+            installed_tool_dependencies=self.installed_tool_dependencies,
+            tool_dir=self.tool_dir,
         )
 
     @property

lib/galaxy/tools/deps/resolvers/brewed_tool_shed_packages.py

@@ -0,0 +1,150 @@
+"""
+This dependency resolver resolves tool shed dependencies (those defined
+tool_dependencies.xml) installed using Platform Homebrew and converted
+via shed2tap (e.g. https://github.com/jmchilton/homebrew-toolshed).
+"""
+import logging
+import os
+from xml.etree import ElementTree as ET
+
+from .resolver_mixins import (
+    UsesHomebrewMixin,
+    UsesToolDependencyDirMixin,
+    UsesInstalledRepositoriesMixin,
+)
+from ..resolvers import DependencyResolver, INDETERMINATE_DEPENDENCY
+
+log = logging.getLogger(__name__)
+
+
+class HomebrewToolShedDependencyResolver(
+    DependencyResolver,
+    UsesHomebrewMixin,
+    UsesToolDependencyDirMixin,
+    UsesInstalledRepositoriesMixin,
+):
+    resolver_type = "tool_shed_tap"
+
+    def __init__(self, dependency_manager, **kwds):
+        self._init_homebrew(**kwds)
+        self._init_base_path(dependency_manager, **kwds)
+
+    def resolve(self, name, version, type, **kwds):
+        if type != "package":
+            return INDETERMINATE_DEPENDENCY
+
+        if version is None:
+            return INDETERMINATE_DEPENDENCY
+
+        return self._find_tool_dependencies(name, version, type, **kwds)
+
+    def _find_tool_dependencies(self, name, version, type, **kwds):
+        installed_tool_dependency = self._get_installed_dependency(name, type, version=version, **kwds)
+        if installed_tool_dependency:
+            return self._resolve_from_installed_tool_dependency(name, version, installed_tool_dependency)
+
+        if "tool_dir" in kwds:
+            tool_directory = os.path.abspath(kwds["tool_dir"])
+            tool_depenedencies_path = os.path.join(tool_directory, "tool_dependencies.xml")
+            if os.path.exists(tool_depenedencies_path):
+                return self._resolve_from_tool_dependencies_path(name, version, tool_depenedencies_path)
+
+        return INDETERMINATE_DEPENDENCY
+
+    def _resolve_from_installed_tool_dependency(self, name, version, installed_tool_dependency):
+        tool_shed_repository = installed_tool_dependency.tool_shed_repository
+        recipe_name = build_recipe_name(
+            package_name=name,
+            package_version=version,
+            repository_owner=tool_shed_repository.owner,
+            repository_name=tool_shed_repository.name,
+        )
+        return self._find_dep_default(recipe_name, None)
+
+    def _resolve_from_tool_dependencies_path(self, name, version, tool_dependencies_path):
+        try:
+            raw_dependencies = RawDependencies(tool_dependencies_path)
+        except Exception:
+            log.debug("Failed to parse dependencies in file %s" % tool_dependencies_path)
+            return INDETERMINATE_DEPENDENCY
+
+        raw_dependency = raw_dependencies.find(name, version)
+        if not raw_dependency:
+            return INDETERMINATE_DEPENDENCY
+
+        recipe_name = build_recipe_name(
+            package_name=name,
+            package_version=version,
+            repository_owner=raw_dependency.repository_owner,
+            repository_name=raw_dependency.repository_name
+        )
+        dep = self._find_dep_default(recipe_name, None)
+        return dep
+
+
+class RawDependencies(object):
+
+    def __init__(self, dependencies_file):
+        self.root = ET.parse(dependencies_file).getroot()
+        dependencies = []
+        package_els = self.root.findall("package") or []
+        for package_el in package_els:
+            repository_el = package_el.find("repository")
+            if repository_el is None:
+                continue
+            dependency = RawDependency(self, package_el, repository_el)
+            dependencies.append(dependency)
+        self.dependencies = dependencies
+
+    def find(self, package_name, package_version):
+        target_dependency = None
+
+        for dependency in self.dependencies:
+            if dependency.package_name == package_name and dependency.package_version == package_version:
+                target_dependency = dependency
+                break
+        return target_dependency
+
+
+class RawDependency(object):
+
+    def __init__(self, dependencies, package_el, repository_el):
+        self.dependencies = dependencies
+        self.package_el = package_el
+        self.repository_el = repository_el
+
+    def __repr__(self):
+        temp = "Dependency[package_name=%s,version=%s,dependent_package=%s]"
+        return temp % (
+            self.package_el.attrib["name"],
+            self.package_el.attrib["version"],
+            self.repository_el.attrib["name"]
+        )
+
+    @property
+    def repository_owner(self):
+        return self.repository_el.attrib["owner"]
+
+    @property
+    def repository_name(self):
+        return self.repository_el.attrib["name"]
+
+    @property
+    def package_name(self):
+        return self.package_el.attrib["name"]
+
+    @property
+    def package_version(self):
+        return self.package_el.attrib["version"]
+
+
+def build_recipe_name(package_name, package_version, repository_owner, repository_name):
+    # TODO: Consider baking package_name and package_version into name? (would be more "correct")
+    owner = repository_owner.replace("-", "")
+    name = repository_name
+    name = name.replace("_", "").replace("-", "")
+    base = "%s_%s" % (owner, name)
+    return base
+
+
+__all__ = [HomebrewToolShedDependencyResolver]

lib/galaxy/tools/deps/resolvers/galaxy_packages.py

@@ -1,12 +1,13 @@
-from os.path import join, islink, realpath, basename, exists, abspath
+from os.path import join, islink, realpath, basename, exists
 
 from ..resolvers import DependencyResolver, INDETERMINATE_DEPENDENCY, Dependency
+from .resolver_mixins import UsesToolDependencyDirMixin
 
 import logging
 log = logging.getLogger( __name__ )
 
 
-class GalaxyPackageDependencyResolver(DependencyResolver):
+class GalaxyPackageDependencyResolver(DependencyResolver, UsesToolDependencyDirMixin):
     resolver_type = "galaxy_packages"
 
     def __init__(self, dependency_manager, **kwds):
@@ -16,7 +17,7 @@ class GalaxyPackageDependencyResolver(DependencyResolver):
         ## resolver that will just grab 'default' version of exact version
         ## unavailable.
         self.versionless = str(kwds.get('versionless', "false")).lower() == "true"
-        self.base_path = abspath( kwds.get('base_path', dependency_manager.default_base_path) )
+        self._init_base_path( dependency_manager, **kwds )
 
     def resolve( self, name, version, type, **kwds ):
         """

lib/galaxy/tools/deps/resolvers/homebrew.py

@@ -12,20 +12,19 @@ This is still an experimental module and there will almost certainly be backward
 incompatible changes coming.
 """
 
-import os
 
-from ..brew_exts import DEFAULT_HOMEBREW_ROOT, recipe_cellar_path, build_env_statements
-from ..resolvers import DependencyResolver, INDETERMINATE_DEPENDENCY, Dependency
+from .resolver_mixins import UsesHomebrewMixin
+from ..resolvers import DependencyResolver, INDETERMINATE_DEPENDENCY
 
 # TODO: Implement prefer version linked...
 PREFER_VERSION_LINKED = 'linked'
 PREFER_VERSION_LATEST = 'latest'
-UNKNOWN_PREFER_VERSION_MESSAGE_TEMPLATE = "HomebrewDependencyResolver prefer_version must be latest %s"
+UNKNOWN_PREFER_VERSION_MESSAGE_TEMPLATE = "HomebrewDependencyResolver prefer_version must be %s"
 UNKNOWN_PREFER_VERSION_MESSAGE = UNKNOWN_PREFER_VERSION_MESSAGE_TEMPLATE % (PREFER_VERSION_LATEST)
 DEFAULT_PREFER_VERSION = PREFER_VERSION_LATEST
 
 
-class HomebrewDependencyResolver(DependencyResolver):
+class HomebrewDependencyResolver(DependencyResolver, UsesHomebrewMixin):
     resolver_type = "homebrew"
 
     def __init__(self, dependency_manager, **kwds):
@@ -38,11 +37,7 @@ class HomebrewDependencyResolver(DependencyResolver):
         if self.versionless and self.prefer_version not in [PREFER_VERSION_LATEST]:
             raise Exception(UNKNOWN_PREFER_VERSION_MESSAGE)
 
-        cellar_root = kwds.get('cellar', None)
-        if cellar_root is None:
-            cellar_root = os.path.join(DEFAULT_HOMEBREW_ROOT, "Cellar")
-
-        self.cellar_root = cellar_root
+        self._init_homebrew(**kwds)
 
     def resolve(self, name, version, type, **kwds):
         if type != "package":
@@ -53,41 +48,6 @@ class HomebrewDependencyResolver(DependencyResolver):
         else:
             return self._find_dep_versioned(name, version)
 
-    def _find_dep_versioned(self, name, version):
-        recipe_path = recipe_cellar_path(self.cellar_root, name, version)
-        if not os.path.exists(recipe_path) or not os.path.isdir(recipe_path):
-            return INDETERMINATE_DEPENDENCY
-
-        commands = build_env_statements(self.cellar_root, recipe_path, relaxed=True)
-        return HomebrewDependency(commands)
-
-    def _find_dep_default(self, name, version):
-        installed_versions = self._installed_versions(name)
-        if not installed_versions:
-            return INDETERMINATE_DEPENDENCY
-
-        # Just grab newest installed version - may make sense some day to find
-        # the linked version instead.
-        default_version = sorted(installed_versions, reverse=True)[0]
-        return self._find_dep_versioned(name, default_version)
-
-    def _installed_versions(self, recipe):
-        recipe_base_path = os.path.join(self.cellar_root, recipe)
-        if not os.path.exists(recipe_base_path):
-            return []
-
-        names = os.listdir(recipe_base_path)
-        return filter(lambda n: os.path.isdir(os.path.join(recipe_base_path, n)), names)
-
-
-class HomebrewDependency(Dependency):
-
-    def __init__(self, commands):
-        self.commands = commands
-
-    def shell_commands(self, requirement):
-        return self.commands.replace("\n", ";") + "\n"
-
 
 def _string_as_bool( value ):
     return str( value ).lower() == "true"

lib/galaxy/tools/deps/resolvers/resolver_mixins.py

@@ -0,0 +1,73 @@
+import os
+from ..brew_exts import DEFAULT_HOMEBREW_ROOT, recipe_cellar_path, build_env_statements
+from ..resolvers import INDETERMINATE_DEPENDENCY, Dependency
+
+
+class UsesHomebrewMixin:
+
+    def _init_homebrew(self, **kwds):
+        cellar_root = kwds.get('cellar', None)
+        if cellar_root is None:
+            cellar_root = os.path.join(DEFAULT_HOMEBREW_ROOT, "Cellar")
+
+        self.cellar_root = cellar_root
+
+    def _find_dep_versioned(self, name, version):
+        recipe_path = recipe_cellar_path(self.cellar_root, name, version)
+        if not os.path.exists(recipe_path) or not os.path.isdir(recipe_path):
+            return INDETERMINATE_DEPENDENCY
+
+        commands = build_env_statements(self.cellar_root, recipe_path, relaxed=True)
+        return HomebrewDependency(commands)
+
+    def _find_dep_default(self, name, version):
+        installed_versions = self._installed_versions(name)
+        if not installed_versions:
+            return INDETERMINATE_DEPENDENCY
+
+        # Just grab newest installed version - may make sense some day to find
+        # the linked version instead.
+        default_version = sorted(installed_versions, reverse=True)[0]
+        return self._find_dep_versioned(name, default_version)
+
+    def _installed_versions(self, recipe):
+        recipe_base_path = os.path.join(self.cellar_root, recipe)
+        if not os.path.exists(recipe_base_path):
+            return []
+
+        names = os.listdir(recipe_base_path)
+        return filter(lambda n: os.path.isdir(os.path.join(recipe_base_path, n)), names)
+
+
+class UsesToolDependencyDirMixin:
+
+    def _init_base_path(self, dependency_manager, **kwds):
+        self.base_path = os.path.abspath( kwds.get('base_path', dependency_manager.default_base_path) )
+
+
+class UsesInstalledRepositoriesMixin:
+
+    def _get_installed_dependency( self, name, type, version=None, **kwds ):
+        installed_tool_dependencies = kwds.get("installed_tool_dependencies", [])
+        for installed_tool_dependency in (installed_tool_dependencies or []):
+            name_and_type_equal = installed_tool_dependency.name == name and installed_tool_dependency.type == type
+            if version:
+                if name_and_type_equal and installed_tool_dependency.version == version:
+                    return installed_tool_dependency
+            else:
+                if name_and_type_equal:
+                    return installed_tool_dependency
+        return None
+
+
+class HomebrewDependency(Dependency):
+
+    def __init__(self, commands):
+        self.commands = commands
+
+    def shell_commands(self, requirement):
+        raw_commands = self.commands.replace("\n", ";")
+        return raw_commands
+
+    def __repr__(self):
+        return "PlatformBrewDependency[commands=%s]" % self.commands

lib/galaxy/tools/deps/resolvers/tool_shed_packages.py

@@ -1,10 +1,11 @@
 from os.path import abspath, join, exists
 
+from .resolver_mixins import UsesInstalledRepositoriesMixin
 from .galaxy_packages import GalaxyPackageDependencyResolver, GalaxyPackageDependency
 from ..resolvers import INDETERMINATE_DEPENDENCY
 
 
-class ToolShedPackageDependencyResolver(GalaxyPackageDependencyResolver):
+class ToolShedPackageDependencyResolver(GalaxyPackageDependencyResolver, UsesInstalledRepositoriesMixin):
     resolver_type = "tool_shed_packages"
 
     def __init__(self, dependency_manager, **kwds):
@@ -12,9 +13,8 @@ class ToolShedPackageDependencyResolver(GalaxyPackageDependencyResolver):
 
     def _find_dep_versioned( self, name, version, type='package', **kwds ):
         installed_tool_dependency = self._get_installed_dependency( name, type, version=version, **kwds )
-        base_path = self.base_path
         if installed_tool_dependency:
-            path = self._get_package_installed_dependency_path( installed_tool_dependency, base_path, name, version )
+            path = self._get_package_installed_dependency_path( installed_tool_dependency, name, version )
             return self._galaxy_package_dep(path, version)
         else:
             return INDETERMINATE_DEPENDENCY
@@ -29,26 +29,17 @@ class ToolShedPackageDependencyResolver(GalaxyPackageDependencyResolver):
                     return GalaxyPackageDependency(dependency.script, dependency.path, None)
         return INDETERMINATE_DEPENDENCY
 
-    def _get_installed_dependency( self, name, type, version=None, **kwds ):
-        installed_tool_dependencies = kwds.get("installed_tool_dependencies", [])
-        for installed_tool_dependency in (installed_tool_dependencies or []):
-            name_and_type_equal = installed_tool_dependency.name == name and installed_tool_dependency.type == type
-            if version:
-                if name_and_type_equal and installed_tool_dependency.version == version:
-                    return installed_tool_dependency
-            else:
-                if name_and_type_equal:
-                    return installed_tool_dependency
-        return None
-
-    def _get_package_installed_dependency_path( self, installed_tool_dependency, base_path, name, version ):
+    def _get_package_installed_dependency_path( self, installed_tool_dependency, name, version ):
         tool_shed_repository = installed_tool_dependency.tool_shed_repository
-        return join( base_path,
-                             name,
-                             version,
-                             tool_shed_repository.owner,
-                             tool_shed_repository.name,
-                             tool_shed_repository.installed_changeset_revision )
+        base_path = self.base_path
+        return join(
+            base_path,
+            name,
+            version,
+            tool_shed_repository.owner,
+            tool_shed_repository.name,
+            tool_shed_repository.installed_changeset_revision
+        )
 
     def _get_set_environment_installed_dependency_script_path( self, installed_tool_dependency, name ):
         tool_shed_repository = installed_tool_dependency.tool_shed_repository
@@ -64,4 +55,5 @@ class ToolShedPackageDependencyResolver(GalaxyPackageDependencyResolver):
             return GalaxyPackageDependency(script, path, None)
         return INDETERMINATE_DEPENDENCY
 
+
 __all__ = [ToolShedPackageDependencyResolver]

lib/galaxy/util/plugin_config.py

@@ -24,7 +24,7 @@ def plugins_dict(module, plugin_type_identifier):
     for plugin_module in submodules( module ):
         # FIXME: this is not how one is suppose to use __all__ why did you do
         # this past John?
-        for clazz in plugin_module.__all__:
+        for clazz in getattr( plugin_module, "__all__", [] ):
             plugin_type = getattr( clazz, plugin_type_identifier, None )
             if plugin_type:
                 plugin_dict[ plugin_type ] = clazz