From f83ff130f2170aa4a1ec9edff0a420175aa7c154 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Wed, 7 Jan 2015 16:33:11 -0500 Subject: [PATCH] Implement dependency resolver for Homebrew "tapped" tool shed packages (highly beta). Homebrew/Linuxbrew can be configured the same way the more vanilla Homebrew resolver works (added in 6b7782f). When this resolver is active, if the tool has been installed via the Tool Shed and the dependencies also installed it will use the database information to determine the dependent packages - otherwise it will just try to find the tool_dependencies.xml file on disk and recover the dependencies from that. This is for dependencies installed via Platform Brew (https://github.com/jmchilton/platform-brew) and converted from Tool Shed repositories usiing the shed2tap WIP code (e.g. https://github.com/jmchilton/homebrew-toolshed). --- lib/galaxy/tools/__init__.py | 3 +- .../resolvers/brewed_tool_shed_packages.py | 150 ++++++++++++++++++ .../tools/deps/resolvers/galaxy_packages.py | 7 +- lib/galaxy/tools/deps/resolvers/homebrew.py | 50 +----- .../tools/deps/resolvers/resolver_mixins.py | 73 +++++++++ .../deps/resolvers/tool_shed_packages.py | 36 ++--- lib/galaxy/util/plugin_config.py | 2 +- 7 files changed, 249 insertions(+), 72 deletions(-) create mode 100644 lib/galaxy/tools/deps/resolvers/brewed_tool_shed_packages.py create mode 100644 lib/galaxy/tools/deps/resolvers/resolver_mixins.py diff --git a/lib/galaxy/tools/__init__.py b/lib/galaxy/tools/__init__.py index 63409c2191b..2c44b60b5bf 100755 --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -1843,7 +1843,8 @@ class Tool( object, Dictifiable ): """Return a list of commands to be run to populate the current environment to include this tools requirements.""" return self.app.toolbox.dependency_manager.dependency_shell_commands( self.requirements, - installed_tool_dependencies=self.installed_tool_dependencies + installed_tool_dependencies=self.installed_tool_dependencies, + tool_dir=self.tool_dir, ) @property diff --git a/lib/galaxy/tools/deps/resolvers/brewed_tool_shed_packages.py b/lib/galaxy/tools/deps/resolvers/brewed_tool_shed_packages.py new file mode 100644 index 00000000000..6a4cb989809 --- /dev/null +++ b/lib/galaxy/tools/deps/resolvers/brewed_tool_shed_packages.py @@ -0,0 +1,150 @@ +""" +This dependency resolver resolves tool shed dependencies (those defined +tool_dependencies.xml) installed using Platform Homebrew and converted +via shed2tap (e.g. https://github.com/jmchilton/homebrew-toolshed). +""" +import logging +import os +from xml.etree import ElementTree as ET + +from .resolver_mixins import ( + UsesHomebrewMixin, + UsesToolDependencyDirMixin, + UsesInstalledRepositoriesMixin, +) +from ..resolvers import DependencyResolver, INDETERMINATE_DEPENDENCY + +log = logging.getLogger(__name__) + + +class HomebrewToolShedDependencyResolver( + DependencyResolver, + UsesHomebrewMixin, + UsesToolDependencyDirMixin, + UsesInstalledRepositoriesMixin, +): + resolver_type = "tool_shed_tap" + + def __init__(self, dependency_manager, **kwds): + self._init_homebrew(**kwds) + self._init_base_path(dependency_manager, **kwds) + + def resolve(self, name, version, type, **kwds): + if type != "package": + return INDETERMINATE_DEPENDENCY + + if version is None: + return INDETERMINATE_DEPENDENCY + + return self._find_tool_dependencies(name, version, type, **kwds) + + def _find_tool_dependencies(self, name, version, type, **kwds): + installed_tool_dependency = self._get_installed_dependency(name, type, version=version, **kwds) + if installed_tool_dependency: + return self._resolve_from_installed_tool_dependency(name, version, installed_tool_dependency) + + if "tool_dir" in kwds: + tool_directory = os.path.abspath(kwds["tool_dir"]) + tool_depenedencies_path = os.path.join(tool_directory, "tool_dependencies.xml") + if os.path.exists(tool_depenedencies_path): + return self._resolve_from_tool_dependencies_path(name, version, tool_depenedencies_path) + + return INDETERMINATE_DEPENDENCY + + def _resolve_from_installed_tool_dependency(self, name, version, installed_tool_dependency): + tool_shed_repository = installed_tool_dependency.tool_shed_repository + recipe_name = build_recipe_name( + package_name=name, + package_version=version, + repository_owner=tool_shed_repository.owner, + repository_name=tool_shed_repository.name, + ) + return self._find_dep_default(recipe_name, None) + + def _resolve_from_tool_dependencies_path(self, name, version, tool_dependencies_path): + try: + raw_dependencies = RawDependencies(tool_dependencies_path) + except Exception: + log.debug("Failed to parse dependencies in file %s" % tool_dependencies_path) + return INDETERMINATE_DEPENDENCY + + raw_dependency = raw_dependencies.find(name, version) + if not raw_dependency: + return INDETERMINATE_DEPENDENCY + + recipe_name = build_recipe_name( + package_name=name, + package_version=version, + repository_owner=raw_dependency.repository_owner, + repository_name=raw_dependency.repository_name + ) + dep = self._find_dep_default(recipe_name, None) + return dep + + +class RawDependencies(object): + + def __init__(self, dependencies_file): + self.root = ET.parse(dependencies_file).getroot() + dependencies = [] + package_els = self.root.findall("package") or [] + for package_el in package_els: + repository_el = package_el.find("repository") + if repository_el is None: + continue + dependency = RawDependency(self, package_el, repository_el) + dependencies.append(dependency) + self.dependencies = dependencies + + def find(self, package_name, package_version): + target_dependency = None + + for dependency in self.dependencies: + if dependency.package_name == package_name and dependency.package_version == package_version: + target_dependency = dependency + break + return target_dependency + + +class RawDependency(object): + + def __init__(self, dependencies, package_el, repository_el): + self.dependencies = dependencies + self.package_el = package_el + self.repository_el = repository_el + + def __repr__(self): + temp = "Dependency[package_name=%s,version=%s,dependent_package=%s]" + return temp % ( + self.package_el.attrib["name"], + self.package_el.attrib["version"], + self.repository_el.attrib["name"] + ) + + @property + def repository_owner(self): + return self.repository_el.attrib["owner"] + + @property + def repository_name(self): + return self.repository_el.attrib["name"] + + @property + def package_name(self): + return self.package_el.attrib["name"] + + @property + def package_version(self): + return self.package_el.attrib["version"] + + +def build_recipe_name(package_name, package_version, repository_owner, repository_name): + # TODO: Consider baking package_name and package_version into name? (would be more "correct") + owner = repository_owner.replace("-", "") + name = repository_name + name = name.replace("_", "").replace("-", "") + base = "%s_%s" % (owner, name) + return base + + +__all__ = [HomebrewToolShedDependencyResolver] diff --git a/lib/galaxy/tools/deps/resolvers/galaxy_packages.py b/lib/galaxy/tools/deps/resolvers/galaxy_packages.py index 7f250479315..99eab3a6939 100644 --- a/lib/galaxy/tools/deps/resolvers/galaxy_packages.py +++ b/lib/galaxy/tools/deps/resolvers/galaxy_packages.py @@ -1,12 +1,13 @@ -from os.path import join, islink, realpath, basename, exists, abspath +from os.path import join, islink, realpath, basename, exists from ..resolvers import DependencyResolver, INDETERMINATE_DEPENDENCY, Dependency +from .resolver_mixins import UsesToolDependencyDirMixin import logging log = logging.getLogger( __name__ ) -class GalaxyPackageDependencyResolver(DependencyResolver): +class GalaxyPackageDependencyResolver(DependencyResolver, UsesToolDependencyDirMixin): resolver_type = "galaxy_packages" def __init__(self, dependency_manager, **kwds): @@ -16,7 +17,7 @@ class GalaxyPackageDependencyResolver(DependencyResolver): ## resolver that will just grab 'default' version of exact version ## unavailable. self.versionless = str(kwds.get('versionless', "false")).lower() == "true" - self.base_path = abspath( kwds.get('base_path', dependency_manager.default_base_path) ) + self._init_base_path( dependency_manager, **kwds ) def resolve( self, name, version, type, **kwds ): """ diff --git a/lib/galaxy/tools/deps/resolvers/homebrew.py b/lib/galaxy/tools/deps/resolvers/homebrew.py index 20f180a2928..53885041ebd 100644 --- a/lib/galaxy/tools/deps/resolvers/homebrew.py +++ b/lib/galaxy/tools/deps/resolvers/homebrew.py @@ -12,20 +12,19 @@ This is still an experimental module and there will almost certainly be backward incompatible changes coming. """ -import os -from ..brew_exts import DEFAULT_HOMEBREW_ROOT, recipe_cellar_path, build_env_statements -from ..resolvers import DependencyResolver, INDETERMINATE_DEPENDENCY, Dependency +from .resolver_mixins import UsesHomebrewMixin +from ..resolvers import DependencyResolver, INDETERMINATE_DEPENDENCY # TODO: Implement prefer version linked... PREFER_VERSION_LINKED = 'linked' PREFER_VERSION_LATEST = 'latest' -UNKNOWN_PREFER_VERSION_MESSAGE_TEMPLATE = "HomebrewDependencyResolver prefer_version must be latest %s" +UNKNOWN_PREFER_VERSION_MESSAGE_TEMPLATE = "HomebrewDependencyResolver prefer_version must be %s" UNKNOWN_PREFER_VERSION_MESSAGE = UNKNOWN_PREFER_VERSION_MESSAGE_TEMPLATE % (PREFER_VERSION_LATEST) DEFAULT_PREFER_VERSION = PREFER_VERSION_LATEST -class HomebrewDependencyResolver(DependencyResolver): +class HomebrewDependencyResolver(DependencyResolver, UsesHomebrewMixin): resolver_type = "homebrew" def __init__(self, dependency_manager, **kwds): @@ -38,11 +37,7 @@ class HomebrewDependencyResolver(DependencyResolver): if self.versionless and self.prefer_version not in [PREFER_VERSION_LATEST]: raise Exception(UNKNOWN_PREFER_VERSION_MESSAGE) - cellar_root = kwds.get('cellar', None) - if cellar_root is None: - cellar_root = os.path.join(DEFAULT_HOMEBREW_ROOT, "Cellar") - - self.cellar_root = cellar_root + self._init_homebrew(**kwds) def resolve(self, name, version, type, **kwds): if type != "package": @@ -53,41 +48,6 @@ class HomebrewDependencyResolver(DependencyResolver): else: return self._find_dep_versioned(name, version) - def _find_dep_versioned(self, name, version): - recipe_path = recipe_cellar_path(self.cellar_root, name, version) - if not os.path.exists(recipe_path) or not os.path.isdir(recipe_path): - return INDETERMINATE_DEPENDENCY - - commands = build_env_statements(self.cellar_root, recipe_path, relaxed=True) - return HomebrewDependency(commands) - - def _find_dep_default(self, name, version): - installed_versions = self._installed_versions(name) - if not installed_versions: - return INDETERMINATE_DEPENDENCY - - # Just grab newest installed version - may make sense some day to find - # the linked version instead. - default_version = sorted(installed_versions, reverse=True)[0] - return self._find_dep_versioned(name, default_version) - - def _installed_versions(self, recipe): - recipe_base_path = os.path.join(self.cellar_root, recipe) - if not os.path.exists(recipe_base_path): - return [] - - names = os.listdir(recipe_base_path) - return filter(lambda n: os.path.isdir(os.path.join(recipe_base_path, n)), names) - - -class HomebrewDependency(Dependency): - - def __init__(self, commands): - self.commands = commands - - def shell_commands(self, requirement): - return self.commands.replace("\n", ";") + "\n" - def _string_as_bool( value ): return str( value ).lower() == "true" diff --git a/lib/galaxy/tools/deps/resolvers/resolver_mixins.py b/lib/galaxy/tools/deps/resolvers/resolver_mixins.py new file mode 100644 index 00000000000..661d87ecaee --- /dev/null +++ b/lib/galaxy/tools/deps/resolvers/resolver_mixins.py @@ -0,0 +1,73 @@ +import os +from ..brew_exts import DEFAULT_HOMEBREW_ROOT, recipe_cellar_path, build_env_statements +from ..resolvers import INDETERMINATE_DEPENDENCY, Dependency + + +class UsesHomebrewMixin: + + def _init_homebrew(self, **kwds): + cellar_root = kwds.get('cellar', None) + if cellar_root is None: + cellar_root = os.path.join(DEFAULT_HOMEBREW_ROOT, "Cellar") + + self.cellar_root = cellar_root + + def _find_dep_versioned(self, name, version): + recipe_path = recipe_cellar_path(self.cellar_root, name, version) + if not os.path.exists(recipe_path) or not os.path.isdir(recipe_path): + return INDETERMINATE_DEPENDENCY + + commands = build_env_statements(self.cellar_root, recipe_path, relaxed=True) + return HomebrewDependency(commands) + + def _find_dep_default(self, name, version): + installed_versions = self._installed_versions(name) + if not installed_versions: + return INDETERMINATE_DEPENDENCY + + # Just grab newest installed version - may make sense some day to find + # the linked version instead. + default_version = sorted(installed_versions, reverse=True)[0] + return self._find_dep_versioned(name, default_version) + + def _installed_versions(self, recipe): + recipe_base_path = os.path.join(self.cellar_root, recipe) + if not os.path.exists(recipe_base_path): + return [] + + names = os.listdir(recipe_base_path) + return filter(lambda n: os.path.isdir(os.path.join(recipe_base_path, n)), names) + + +class UsesToolDependencyDirMixin: + + def _init_base_path(self, dependency_manager, **kwds): + self.base_path = os.path.abspath( kwds.get('base_path', dependency_manager.default_base_path) ) + + +class UsesInstalledRepositoriesMixin: + + def _get_installed_dependency( self, name, type, version=None, **kwds ): + installed_tool_dependencies = kwds.get("installed_tool_dependencies", []) + for installed_tool_dependency in (installed_tool_dependencies or []): + name_and_type_equal = installed_tool_dependency.name == name and installed_tool_dependency.type == type + if version: + if name_and_type_equal and installed_tool_dependency.version == version: + return installed_tool_dependency + else: + if name_and_type_equal: + return installed_tool_dependency + return None + + +class HomebrewDependency(Dependency): + + def __init__(self, commands): + self.commands = commands + + def shell_commands(self, requirement): + raw_commands = self.commands.replace("\n", ";") + return raw_commands + + def __repr__(self): + return "PlatformBrewDependency[commands=%s]" % self.commands diff --git a/lib/galaxy/tools/deps/resolvers/tool_shed_packages.py b/lib/galaxy/tools/deps/resolvers/tool_shed_packages.py index d72b4d6a20b..a58703bd6fb 100644 --- a/lib/galaxy/tools/deps/resolvers/tool_shed_packages.py +++ b/lib/galaxy/tools/deps/resolvers/tool_shed_packages.py @@ -1,10 +1,11 @@ from os.path import abspath, join, exists +from .resolver_mixins import UsesInstalledRepositoriesMixin from .galaxy_packages import GalaxyPackageDependencyResolver, GalaxyPackageDependency from ..resolvers import INDETERMINATE_DEPENDENCY -class ToolShedPackageDependencyResolver(GalaxyPackageDependencyResolver): +class ToolShedPackageDependencyResolver(GalaxyPackageDependencyResolver, UsesInstalledRepositoriesMixin): resolver_type = "tool_shed_packages" def __init__(self, dependency_manager, **kwds): @@ -12,9 +13,8 @@ class ToolShedPackageDependencyResolver(GalaxyPackageDependencyResolver): def _find_dep_versioned( self, name, version, type='package', **kwds ): installed_tool_dependency = self._get_installed_dependency( name, type, version=version, **kwds ) - base_path = self.base_path if installed_tool_dependency: - path = self._get_package_installed_dependency_path( installed_tool_dependency, base_path, name, version ) + path = self._get_package_installed_dependency_path( installed_tool_dependency, name, version ) return self._galaxy_package_dep(path, version) else: return INDETERMINATE_DEPENDENCY @@ -29,26 +29,17 @@ class ToolShedPackageDependencyResolver(GalaxyPackageDependencyResolver): return GalaxyPackageDependency(dependency.script, dependency.path, None) return INDETERMINATE_DEPENDENCY - def _get_installed_dependency( self, name, type, version=None, **kwds ): - installed_tool_dependencies = kwds.get("installed_tool_dependencies", []) - for installed_tool_dependency in (installed_tool_dependencies or []): - name_and_type_equal = installed_tool_dependency.name == name and installed_tool_dependency.type == type - if version: - if name_and_type_equal and installed_tool_dependency.version == version: - return installed_tool_dependency - else: - if name_and_type_equal: - return installed_tool_dependency - return None - - def _get_package_installed_dependency_path( self, installed_tool_dependency, base_path, name, version ): + def _get_package_installed_dependency_path( self, installed_tool_dependency, name, version ): tool_shed_repository = installed_tool_dependency.tool_shed_repository - return join( base_path, - name, - version, - tool_shed_repository.owner, - tool_shed_repository.name, - tool_shed_repository.installed_changeset_revision ) + base_path = self.base_path + return join( + base_path, + name, + version, + tool_shed_repository.owner, + tool_shed_repository.name, + tool_shed_repository.installed_changeset_revision + ) def _get_set_environment_installed_dependency_script_path( self, installed_tool_dependency, name ): tool_shed_repository = installed_tool_dependency.tool_shed_repository @@ -64,4 +55,5 @@ class ToolShedPackageDependencyResolver(GalaxyPackageDependencyResolver): return GalaxyPackageDependency(script, path, None) return INDETERMINATE_DEPENDENCY + __all__ = [ToolShedPackageDependencyResolver] diff --git a/lib/galaxy/util/plugin_config.py b/lib/galaxy/util/plugin_config.py index 8cff759d534..a3b6b9fb78b 100644 --- a/lib/galaxy/util/plugin_config.py +++ b/lib/galaxy/util/plugin_config.py @@ -24,7 +24,7 @@ def plugins_dict(module, plugin_type_identifier): for plugin_module in submodules( module ): # FIXME: this is not how one is suppose to use __all__ why did you do # this past John? - for clazz in plugin_module.__all__: + for clazz in getattr( plugin_module, "__all__", [] ): plugin_type = getattr( clazz, plugin_type_identifier, None ) if plugin_type: plugin_dict[ plugin_type ] = clazz