From f597b08b878f478b9beee32c59b5e4218458c4fe Mon Sep 17 00:00:00 2001 From: John Chilton Date: Sat, 26 Dec 2015 15:47:04 +0000 Subject: [PATCH 1/8] Refactor ToolBox functionality. This moves in two directions toward a better design. It moves some Galaxy tool specific logic out of the AbstractToolBox (which should just be worried about panels, etc... IMO) and it moves more galaxy-lib functionality (dependency resolution) into a galaxy-lib module (galaxy.tools.toolbox). It does this by creating a BaseGalaxyToolBox that can reason about which files look like tools and what dependency resoution should be - without dependening on Galaxy models, job running, etc... that is in galaxy.tools.__init__. --- lib/galaxy/tools/__init__.py | 9 ++---- lib/galaxy/tools/toolbox/__init__.py | 2 ++ lib/galaxy/tools/toolbox/base.py | 42 +++++++++++++++++++++------- 3 files changed, 36 insertions(+), 17 deletions(-) diff --git a/lib/galaxy/tools/__init__.py b/lib/galaxy/tools/__init__.py index b39caa58297..32a3e799965 100755 --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -25,7 +25,6 @@ from galaxy.tools.actions import DefaultToolAction from galaxy.tools.actions.upload import UploadToolAction from galaxy.tools.actions.data_source import DataSourceToolAction from galaxy.tools.actions.data_manager import DataManagerToolAction -from galaxy.tools.deps import build_dependency_manager from galaxy.tools.parameters import params_to_incoming, check_param, params_from_strings, params_to_strings, visit_input_values from galaxy.tools.parameters import output_collect from galaxy.tools.parameters.basic import (BaseURLToolParameter, @@ -39,7 +38,7 @@ from galaxy.tools.test import parse_tests from galaxy.tools.parser import get_tool_source from galaxy.tools.parser.xml import XmlPageSource from galaxy.tools.parser import ToolOutputCollectionPart -from galaxy.tools.toolbox import AbstractToolBox +from galaxy.tools.toolbox import BaseGalaxyToolBox from galaxy.util import rst_to_html, string_as_bool from galaxy.util import ExecutionTimer from galaxy.util import listify @@ -98,7 +97,7 @@ class ToolNotFoundException( Exception ): pass -class ToolBox( AbstractToolBox ): +class ToolBox( BaseGalaxyToolBox ): """ A derivative of AbstractToolBox with knowledge about Tool internals - how to construct them, action types, dependency management, etc.... """ @@ -109,7 +108,6 @@ class ToolBox( AbstractToolBox ): tool_root_dir=tool_root_dir, app=app, ) - self._init_dependency_manager() @property def tools_by_id( self ): @@ -158,9 +156,6 @@ class ToolBox( AbstractToolBox ): tool = ToolClass( config_file, tool_source, self.app, guid=guid, repository_id=repository_id, **kwds ) return tool - def _init_dependency_manager( self ): - self.dependency_manager = build_dependency_manager( self.app.config ) - def handle_datatypes_changed( self ): """ Refresh upload tools when new datatypes are added. """ for tool_id in self._tools_by_id: diff --git a/lib/galaxy/tools/toolbox/__init__.py b/lib/galaxy/tools/toolbox/__init__.py index d33eb14d589..39f26b87a7b 100644 --- a/lib/galaxy/tools/toolbox/__init__.py +++ b/lib/galaxy/tools/toolbox/__init__.py @@ -6,10 +6,12 @@ from .panel import ToolSection from .panel import ToolSectionLabel from .base import AbstractToolBox +from .base import BaseGalaxyToolBox __all__ = [ "ToolSection", "ToolSectionLabel", "panel_item_types", "AbstractToolBox", + "BaseGalaxyToolBox" ] diff --git a/lib/galaxy/tools/toolbox/base.py b/lib/galaxy/tools/toolbox/base.py index 49ba91ef773..99d01527a08 100644 --- a/lib/galaxy/tools/toolbox/base.py +++ b/lib/galaxy/tools/toolbox/base.py @@ -23,14 +23,17 @@ from .panel import ToolSection from .panel import panel_item_types from .integrated_panel import ManagesIntegratedToolPanelMixin -from galaxy.tools.loader_directory import looks_like_a_tool - from .lineages import LineageMap from .tags import tool_tag_manager from .filters import FilterFactory from .watcher import get_watcher +# Extra tool dependency not used by AbstractToolBox but by +# BaseGalaxyToolBox +from galaxy.tools.loader_directory import looks_like_a_tool +from galaxy.tools.deps import build_dependency_manager + log = logging.getLogger( __name__ ) @@ -693,7 +696,7 @@ class AbstractToolBox( Dictifiable, ManagesIntegratedToolPanelMixin, object ): child_path = os.path.join(directory, name) if os.path.isdir(child_path) and recursive: self.__watch_directory(child_path, elems, integrated_elems, load_panel_dict, recursive) - elif looks_like_a_tool(child_path, enable_beta_formats=getattr(self.app.config, "enable_beta_tool_formats", False)): + elif self._looks_like_a_tool(child_path): quick_load( child_path, async=False ) tool_loaded = True if tool_loaded or force_watch: @@ -957,13 +960,6 @@ class AbstractToolBox( Dictifiable, ManagesIntegratedToolPanelMixin, object ): stored = self.app.model.context.query( self.app.model.StoredWorkflow ).get( id ) return stored.latest_workflow - @property - def sa_session( self ): - """ - Returns a SQLAlchemy session - """ - return self.app.model.context - def tool_panel_contents( self, trans, **kwds ): """ Filter tool_panel contents for displaying for user. """ @@ -1096,3 +1092,29 @@ def _filter_for_panel( item, item_type, filters, context ): return copy return None + + +class BaseGalaxyToolBox(AbstractToolBox): + """ + Extend the AbstractToolBox with more Galaxy tooling-specific + functionality. Adds dependencies on dependency resolution and + tool loading modules, that an abstract description of panels + shouldn't really depend on. + """ + + def __init__(self, config_filenames, tool_root_dir, app): + super(BaseGalaxyToolBox, self).__init__(config_filenames, tool_root_dir, app) + self._init_dependency_manager() + + @property + def sa_session( self ): + """ + Returns a SQLAlchemy session + """ + return self.app.model.context + + def _looks_like_a_tool(self, path): + return looks_like_a_tool(path, enable_beta_formats=getattr(self.app.config, "enable_beta_tool_formats", False)) + + def _init_dependency_manager( self ): + self.dependency_manager = build_dependency_manager( self.app.config ) From 204aa96206a7fdaff69d22fde6387a9956c1a0bc Mon Sep 17 00:00:00 2001 From: John Chilton Date: Sat, 26 Dec 2015 17:12:51 +0000 Subject: [PATCH 2/8] API-ify tool dependency resolution. --- lib/galaxy/tools/deps/__init__.py | 7 +- lib/galaxy/tools/deps/resolvers/__init__.py | 14 ++- lib/galaxy/tools/deps/resolvers/conda.py | 8 ++ .../tools/deps/resolvers/galaxy_packages.py | 3 + lib/galaxy/tools/deps/resolvers/modules.py | 4 + .../deps/resolvers/tool_shed_packages.py | 4 + .../resolvers/unlinked_tool_shed_packages.py | 7 +- lib/galaxy/tools/deps/views.py | 57 ++++++++++++ lib/galaxy/tools/toolbox/base.py | 3 + .../webapps/galaxy/api/tool_dependencies.py | 91 +++++++++++++++++++ lib/galaxy/webapps/galaxy/buildapp.py | 4 + test/unit/test_routes.py | 24 +++++ 12 files changed, 221 insertions(+), 5 deletions(-) create mode 100644 lib/galaxy/tools/deps/views.py create mode 100644 lib/galaxy/webapps/galaxy/api/tool_dependencies.py diff --git a/lib/galaxy/tools/deps/__init__.py b/lib/galaxy/tools/deps/__init__.py index dab30db6512..8b549eb3ed6 100644 --- a/lib/galaxy/tools/deps/__init__.py +++ b/lib/galaxy/tools/deps/__init__.py @@ -49,6 +49,7 @@ def build_dependency_manager( config ): class NullDependencyManager( object ): + dependency_resolvers = [] def uses_tool_shed_dependencies(self): return False @@ -107,7 +108,11 @@ class DependencyManager( object ): def find_dep( self, name, version=None, type='package', **kwds ): log.debug('Find dependency %s version %s' % (name, version)) - for resolver in self.dependency_resolvers: + index = kwds.get('index', None) + for i, resolver in enumerate(self.dependency_resolvers): + if index is not None and i != index: + continue + dependency = resolver.resolve( name, version, type, **kwds ) log.debug('Resolver %s returned %s (isnull? %s)' % (resolver.resolver_type, dependency, dependency == INDETERMINATE_DEPENDENCY)) diff --git a/lib/galaxy/tools/deps/resolvers/__init__.py b/lib/galaxy/tools/deps/resolvers/__init__.py index 2f5e1e6da04..f4ca06a90de 100644 --- a/lib/galaxy/tools/deps/resolvers/__init__.py +++ b/lib/galaxy/tools/deps/resolvers/__init__.py @@ -1,7 +1,15 @@ +from galaxy.util.dictifiable import Dictifiable + from abc import ABCMeta, abstractmethod -class DependencyResolver( object ): +class DependencyResolver(Dictifiable, object): + # Keys for dictification. + dict_collection_visible_keys = ['resolver_type', 'resolves_simple_dependencies'] + # A "simple" dependency is one that does not depend on the the tool + # resolving the dependency. Classic tool shed dependencies are non-simple + # because the tool shed install. + resolves_simple_dependencies = True __metaclass__ = ABCMeta @abstractmethod @@ -28,7 +36,8 @@ class DependencyResolver( object ): return default -class Dependency( object ): +class Dependency(Dictifiable, object): + dict_collection_visible_keys = ['dependency_type'] __metaclass__ = ABCMeta @abstractmethod @@ -39,6 +48,7 @@ class Dependency( object ): class NullDependency( Dependency ): + dependency_type = None def shell_commands( self, requirement ): return None diff --git a/lib/galaxy/tools/deps/resolvers/conda.py b/lib/galaxy/tools/deps/resolvers/conda.py index 58d6ddc8ce2..48691a0c12e 100644 --- a/lib/galaxy/tools/deps/resolvers/conda.py +++ b/lib/galaxy/tools/deps/resolvers/conda.py @@ -28,6 +28,7 @@ log = logging.getLogger(__name__) class CondaDependencyResolver(DependencyResolver): + dict_collection_visible_keys = DependencyResolver.dict_collection_visible_keys + ['conda_prefix', 'versionless', 'ensure_channels', 'auto_install'] resolver_type = "conda" def __init__(self, dependency_manager, **kwds): @@ -55,6 +56,7 @@ class CondaDependencyResolver(DependencyResolver): debug=debug, ensure_channels=ensure_channels, ) + self.ensure_channels = ensure_channels # Conda operations options (these define how resolution will occur) auto_init = _string_as_bool(get_option("auto_init")) @@ -117,8 +119,14 @@ class CondaDependencyResolver(DependencyResolver): else: raise Exception("Conda dependency seemingly installed but failed to build job environment.") + @property + def prefix(self): + return self.conda_context.conda_prefix + class CondaDepenency(Dependency): + dict_collection_visible_keys = Dependency.dict_collection_visible_keys + ['environment_path'] + dependency_type = 'conda' def __init__(self, activate, environment_path): self.activate = activate diff --git a/lib/galaxy/tools/deps/resolvers/galaxy_packages.py b/lib/galaxy/tools/deps/resolvers/galaxy_packages.py index 3d190f85bd6..2e9f5527aae 100644 --- a/lib/galaxy/tools/deps/resolvers/galaxy_packages.py +++ b/lib/galaxy/tools/deps/resolvers/galaxy_packages.py @@ -8,6 +8,7 @@ log = logging.getLogger( __name__ ) class GalaxyPackageDependencyResolver(DependencyResolver, UsesToolDependencyDirMixin): + dict_collection_visible_keys = DependencyResolver.dict_collection_visible_keys + ['base_path', 'versionless'] resolver_type = "galaxy_packages" def __init__(self, dependency_manager, **kwds): @@ -54,6 +55,8 @@ class GalaxyPackageDependencyResolver(DependencyResolver, UsesToolDependencyDirM class GalaxyPackageDependency(Dependency): + dict_collection_visible_keys = Dependency.dict_collection_visible_keys + ['script', 'path', 'version'] + dependency_type = 'galaxy_package' def __init__( self, script, path, version ): self.script = script diff --git a/lib/galaxy/tools/deps/resolvers/modules.py b/lib/galaxy/tools/deps/resolvers/modules.py index b155aa88f9f..a3fa551f35e 100644 --- a/lib/galaxy/tools/deps/resolvers/modules.py +++ b/lib/galaxy/tools/deps/resolvers/modules.py @@ -24,6 +24,7 @@ UNKNOWN_FIND_BY_MESSAGE = "ModuleDependencyResolver does not know how to find mo class ModuleDependencyResolver(DependencyResolver): + dict_collection_visible_keys = DependencyResolver.dict_collection_visible_keys + ['base_path', 'modulepath'] resolver_type = "modules" def __init__(self, dependency_manager, **kwds): @@ -151,6 +152,9 @@ class ModuleDependency(Dependency): Using Environment Modules' 'modulecmd' (specifically 'modulecmd sh load') to convert module specifications into shell expressions for inclusion in the script used to run a tool in Galaxy.""" + dict_collection_visible_keys = Dependency.dict_collection_visible_keys + ['module_name', 'module_version'] + dependency_type = 'module' + def __init__(self, module_dependency_resolver, module_name, module_version=None): self.module_dependency_resolver = module_dependency_resolver self.module_name = module_name diff --git a/lib/galaxy/tools/deps/resolvers/tool_shed_packages.py b/lib/galaxy/tools/deps/resolvers/tool_shed_packages.py index bcd89f1a260..d23f9f632bb 100644 --- a/lib/galaxy/tools/deps/resolvers/tool_shed_packages.py +++ b/lib/galaxy/tools/deps/resolvers/tool_shed_packages.py @@ -7,6 +7,10 @@ from ..resolvers import INDETERMINATE_DEPENDENCY class ToolShedPackageDependencyResolver(GalaxyPackageDependencyResolver, UsesInstalledRepositoriesMixin): resolver_type = "tool_shed_packages" + # Resolution of these dependencies depends on more than just the requirement + # tag, it depends on the tool installation context - therefore these are + # non-simple. + resolves_simple_dependencies = False def __init__(self, dependency_manager, **kwds): super(ToolShedPackageDependencyResolver, self).__init__(dependency_manager, **kwds) diff --git a/lib/galaxy/tools/deps/resolvers/unlinked_tool_shed_packages.py b/lib/galaxy/tools/deps/resolvers/unlinked_tool_shed_packages.py index 7074034bc7c..57571ee228b 100644 --- a/lib/galaxy/tools/deps/resolvers/unlinked_tool_shed_packages.py +++ b/lib/galaxy/tools/deps/resolvers/unlinked_tool_shed_packages.py @@ -23,7 +23,7 @@ from os import listdir from os.path import join, exists, getmtime from .galaxy_packages import GalaxyPackageDependencyResolver -from ..resolvers import INDETERMINATE_DEPENDENCY +from ..resolvers import INDETERMINATE_DEPENDENCY, Dependency import logging log = logging.getLogger( __name__ ) @@ -33,6 +33,7 @@ PREFERRED_OWNERS = MANUAL + ",iuc,devteam" class UnlinkedToolShedPackageDependencyResolver(GalaxyPackageDependencyResolver): + dict_collection_visible_keys = GalaxyPackageDependencyResolver.dict_collection_visible_keys + ['preferred_owners', 'select_by_owner'] resolver_type = "unlinked_tool_shed_packages" def __init__(self, dependency_manager, **kwds): @@ -132,7 +133,9 @@ class UnlinkedToolShedPackageDependencyResolver(GalaxyPackageDependencyResolver) """ -class CandidateDepenency(): +class CandidateDepenency(Dependency): + dict_collection_visible_keys = Dependency.dict_collection_visible_keys + ['dependency', 'path', 'owner'] + dependency_type = 'unlinked_tool_shed_package' def __init__(self, dependency, path, owner=MANUAL): self.dependency = dependency diff --git a/lib/galaxy/tools/deps/views.py b/lib/galaxy/tools/deps/views.py new file mode 100644 index 00000000000..acc465658f0 --- /dev/null +++ b/lib/galaxy/tools/deps/views.py @@ -0,0 +1,57 @@ +from galaxy.exceptions import RequestParameterMissingException + + +class DependencyResolversView(object): + """ Provide a RESTfulish/JSONy interface to a galaxy.tools.deps.DependencyResolver + object. This can be adapted by the Galaxy web framework or other web apps. + """ + + def __init__(self, app): + self._app = app + + def index(self): + return map(lambda r: r.to_dict(), self._dependency_resolvers) + + def show(self, index): + return self._dependency_resolver(index).to_dict() + + def reload(self): + self.toolbox.reload_dependency_manager() + + def manager_dependency(self, **kwds): + return self._dependency(**kwds) + + def resolver_dependency(self, index, **kwds): + return self._dependency(**kwds) + + def _dependency(self, index=None, **kwds): + if index is not None: + index = int(index) + + name = kwds.get("name", None) + if name is None: + raise RequestParameterMissingException("Missing 'name' parameter required for resolution.") + version = kwds.get("version", None) + type = kwds.get("type", "package") + resolve_kwds = dict( + job_directory="/path/to/example/job_directory", + index=index, + ) + dependency = self._dependency_manager.find_dep( + name, version=version, type=type, **resolve_kwds + ) + return dependency.to_dict() + + def _dependency_resolver(self, index): + index = int(index) + return self._dependency_resolvers[index] + + @property + def _dependency_manager(self): + return self._app.toolbox.dependency_manager + + @property + def _dependency_resolvers(self): + dependency_manager = self._dependency_manager + dependency_resolvers = dependency_manager.dependency_resolvers + return dependency_resolvers diff --git a/lib/galaxy/tools/toolbox/base.py b/lib/galaxy/tools/toolbox/base.py index 99d01527a08..402dfe9de62 100644 --- a/lib/galaxy/tools/toolbox/base.py +++ b/lib/galaxy/tools/toolbox/base.py @@ -1118,3 +1118,6 @@ class BaseGalaxyToolBox(AbstractToolBox): def _init_dependency_manager( self ): self.dependency_manager = build_dependency_manager( self.app.config ) + + def reload_dependency_manager(self): + self._init_dependency_manager() diff --git a/lib/galaxy/webapps/galaxy/api/tool_dependencies.py b/lib/galaxy/webapps/galaxy/api/tool_dependencies.py new file mode 100644 index 00000000000..4fe0741832a --- /dev/null +++ b/lib/galaxy/webapps/galaxy/api/tool_dependencies.py @@ -0,0 +1,91 @@ +""" +API operations allowing clients to determine datatype supported by Galaxy. +""" + +from galaxy.web import _future_expose_api as expose_api +from galaxy.web import require_admin +from galaxy.web.base.controller import BaseAPIController + +from galaxy.tools.deps import views + + +import logging +log = logging.getLogger( __name__ ) + + +class ToolDependenciesAPIController( BaseAPIController ): + + def __init__(self, app): + super(ToolDependenciesAPIController, self).__init__(app) + self._view = views.DependencyResolversView(app) + + @expose_api + @require_admin + def index(self, trans): + """ + GET /api/dependencies_resolvers + """ + return self._view.index() + + @expose_api + @require_admin + def show(self, trans, id): + """ + GET /api/dependencies_resolver/ + """ + return self._view.show(id) + + @expose_api + @require_admin + def update(self, trans): + """ + PUT /api/dependencies_resolvers + + Reload tool dependency resolution configuration. + """ + return self._view.reload() + + @expose_api + @require_admin + def resolver_dependency(self, trans, id, **kwds): + """ + GET /api/dependencies_resolver/{index}/dependency + + Resolve described requirement against specified dependency resolver. + + :type index: int + :param index: index of the dependncy resolver + :type kwds: dict + :param kwds: dictionary structure containing extra parameters + :type name: str + :param name: name of the requirement to find a dependency for (required) + :type version: str + :param version: version of the requirement to find a dependency for (required) + + :rtype: dict + :returns: a dictified description of the dependency, with type: null + if no match was found. + """ + return self._view.resolver_dependency(id, **kwds) + + def manager_dependency(self, trans, **kwds): + """ + GET /api/dependencies_resolvers/dependency + + Resolve described requirement against all dependency resolvers, returning + the match with highest priority. + + :type index: int + :param index: index of the dependncy resolver + :type kwds: dict + :param kwds: dictionary structure containing extra parameters + :type name: str + :param name: name of the requirement to find a dependency for (required) + :type version: str + :param version: version of the requirement to find a dependency for (required) + + :rtype: dict + :returns: a dictified description of the dependency, with type: null + if no match was found. + """ + return self._view.manager_dependency(**kwds) diff --git a/lib/galaxy/webapps/galaxy/buildapp.py b/lib/galaxy/webapps/galaxy/buildapp.py index 7895d1da5d7..ef4f10b29b1 100644 --- a/lib/galaxy/webapps/galaxy/buildapp.py +++ b/lib/galaxy/webapps/galaxy/buildapp.py @@ -251,6 +251,10 @@ def populate_api_routes( webapp, app ): webapp.mapper.connect( '/api/tools/{id:.+?}', action='show', controller="tools" ) webapp.mapper.resource( 'tool', 'tools', path_prefix='/api' ) + webapp.mapper.connect( '/api/dependency_resolvers/dependency', action="manager_dependency", controller="tool_dependencies" ) + webapp.mapper.connect( '/api/dependency_resolvers/{id}/dependency', action="resolver_dependency", controller="tool_dependencies" ) + webapp.mapper.resource( 'dependency_resolver', 'dependency_resolvers', controller="tool_dependencies", path_prefix='api' ) + webapp.mapper.resource_with_deleted( 'user', 'users', path_prefix='/api' ) webapp.mapper.resource( 'genome', 'genomes', path_prefix='/api' ) webapp.mapper.resource( 'visualization', 'visualizations', path_prefix='/api' ) diff --git a/test/unit/test_routes.py b/test/unit/test_routes.py index 8a0caea17a6..9d960eea620 100644 --- a/test/unit/test_routes.py +++ b/test/unit/test_routes.py @@ -84,6 +84,30 @@ def test_galaxy_routes( ): "/api/histories/123/contents/datasets/456" ) + test_webapp.assert_maps( + "/api/dependency_resolvers", + controller="tool_dependencies", + action="index" + ) + + test_webapp.assert_maps( + "/api/dependency_resolvers/dependency", + controller="tool_dependencies", + action="manager_dependency" + ) + + test_webapp.assert_maps( + "/api/dependency_resolvers/0", + controller="tool_dependencies", + action="show" + ) + + test_webapp.assert_maps( + "/api/dependency_resolvers/0/dependency", + controller="tool_dependencies", + action="resolver_dependency" + ) + def assert_url_is( actual, expected ): assert actual == expected, "Expected URL [%s] but obtained [%s]" % ( expected, actual ) From b167a741a444c3988447b0d63a1ba3dc5e4e62f5 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Sat, 26 Dec 2015 20:39:58 +0000 Subject: [PATCH 3/8] Include information about whether matches are dependencies are exact. --- lib/galaxy/tools/deps/__init__.py | 3 +++ lib/galaxy/tools/deps/resolvers/__init__.py | 12 ++++++++-- lib/galaxy/tools/deps/resolvers/conda.py | 11 ++++++++-- .../tools/deps/resolvers/galaxy_packages.py | 22 ++++++++++++------- lib/galaxy/tools/deps/resolvers/modules.py | 11 +++++++--- .../tools/deps/resolvers/resolver_mixins.py | 9 ++++++-- .../deps/resolvers/tool_shed_packages.py | 6 ++--- .../resolvers/unlinked_tool_shed_packages.py | 5 +++++ .../webapps/galaxy/api/tool_dependencies.py | 6 +++++ 9 files changed, 65 insertions(+), 20 deletions(-) diff --git a/lib/galaxy/tools/deps/__init__.py b/lib/galaxy/tools/deps/__init__.py index 8b549eb3ed6..d27fe3ee3df 100644 --- a/lib/galaxy/tools/deps/__init__.py +++ b/lib/galaxy/tools/deps/__init__.py @@ -109,6 +109,7 @@ class DependencyManager( object ): def find_dep( self, name, version=None, type='package', **kwds ): log.debug('Find dependency %s version %s' % (name, version)) index = kwds.get('index', None) + require_exact = kwds.get('exact', False) for i, resolver in enumerate(self.dependency_resolvers): if index is not None and i != index: continue @@ -116,6 +117,8 @@ class DependencyManager( object ): dependency = resolver.resolve( name, version, type, **kwds ) log.debug('Resolver %s returned %s (isnull? %s)' % (resolver.resolver_type, dependency, dependency == INDETERMINATE_DEPENDENCY)) + if require_exact and not dependency.exact: + dependency = INDETERMINATE_DEPENDENCY if dependency != INDETERMINATE_DEPENDENCY: return dependency return INDETERMINATE_DEPENDENCY diff --git a/lib/galaxy/tools/deps/resolvers/__init__.py b/lib/galaxy/tools/deps/resolvers/__init__.py index f4ca06a90de..256df878439 100644 --- a/lib/galaxy/tools/deps/resolvers/__init__.py +++ b/lib/galaxy/tools/deps/resolvers/__init__.py @@ -1,6 +1,6 @@ from galaxy.util.dictifiable import Dictifiable -from abc import ABCMeta, abstractmethod +from abc import ABCMeta, abstractmethod, abstractproperty class DependencyResolver(Dictifiable, object): @@ -37,7 +37,7 @@ class DependencyResolver(Dictifiable, object): class Dependency(Dictifiable, object): - dict_collection_visible_keys = ['dependency_type'] + dict_collection_visible_keys = ['dependency_type', 'exact'] __metaclass__ = ABCMeta @abstractmethod @@ -46,11 +46,19 @@ class Dependency(Dictifiable, object): Return shell commands to enable this dependency. """ + @abstractproperty + def exact( self ): + """ Return true if version information wasn't discarded to resolve + the dependency. + """ + class NullDependency( Dependency ): dependency_type = None + exact = True def shell_commands( self, requirement ): return None + INDETERMINATE_DEPENDENCY = NullDependency() diff --git a/lib/galaxy/tools/deps/resolvers/conda.py b/lib/galaxy/tools/deps/resolvers/conda.py index 48691a0c12e..a3d4ef5895b 100644 --- a/lib/galaxy/tools/deps/resolvers/conda.py +++ b/lib/galaxy/tools/deps/resolvers/conda.py @@ -85,6 +85,7 @@ class CondaDependencyResolver(DependencyResolver): log.warn("Conda dependency resolver not sent job directory.") return INDETERMINATE_DEPENDENCY + exact = not self.versionless or version is None if self.versionless: version = None @@ -114,7 +115,8 @@ class CondaDependencyResolver(DependencyResolver): if not exit_code: return CondaDepenency( self.conda_context.activate, - conda_environment + conda_environment, + exact, ) else: raise Exception("Conda dependency seemingly installed but failed to build job environment.") @@ -128,9 +130,14 @@ class CondaDepenency(Dependency): dict_collection_visible_keys = Dependency.dict_collection_visible_keys + ['environment_path'] dependency_type = 'conda' - def __init__(self, activate, environment_path): + def __init__(self, activate, environment_path, exact): self.activate = activate self.environment_path = environment_path + self._exact = exact + + @property + def exact(self): + return self._exact def shell_commands(self, requirement): return """[ "$CONDA_DEFAULT_ENV" = "%s" ] || source %s '%s'""" % ( diff --git a/lib/galaxy/tools/deps/resolvers/galaxy_packages.py b/lib/galaxy/tools/deps/resolvers/galaxy_packages.py index 2e9f5527aae..ee5d16113c2 100644 --- a/lib/galaxy/tools/deps/resolvers/galaxy_packages.py +++ b/lib/galaxy/tools/deps/resolvers/galaxy_packages.py @@ -26,31 +26,32 @@ class GalaxyPackageDependencyResolver(DependencyResolver, UsesToolDependencyDirM symbolic link (if found). Returns a triple of: env_script, base_path, real_version """ if version is None or self.versionless: - return self._find_dep_default( name, type=type, **kwds ) + exact = not self.versionless or version is None + return self._find_dep_default( name, type=type, exact=exact, **kwds ) else: return self._find_dep_versioned( name, version, type=type, **kwds ) def _find_dep_versioned( self, name, version, type='package', **kwds ): base_path = self.base_path path = join( base_path, name, version ) - return self._galaxy_package_dep(path, version) + return self._galaxy_package_dep(path, version, True) - def _find_dep_default( self, name, type='package', **kwds ): + def _find_dep_default( self, name, type='package', exact=True, **kwds ): base_path = self.base_path path = join( base_path, name, 'default' ) if islink( path ): real_path = realpath( path ) real_version = basename( real_path ) - return self._galaxy_package_dep(real_path, real_version) + return self._galaxy_package_dep(real_path, real_version, exact) else: return INDETERMINATE_DEPENDENCY - def _galaxy_package_dep( self, path, version ): + def _galaxy_package_dep( self, path, version, exact ): script = join( path, 'env.sh' ) if exists( script ): - return GalaxyPackageDependency(script, path, version) + return GalaxyPackageDependency(script, path, version, exact) elif exists( join( path, 'bin' ) ): - return GalaxyPackageDependency(None, path, version) + return GalaxyPackageDependency(None, path, version, exact) return INDETERMINATE_DEPENDENCY @@ -58,10 +59,15 @@ class GalaxyPackageDependency(Dependency): dict_collection_visible_keys = Dependency.dict_collection_visible_keys + ['script', 'path', 'version'] dependency_type = 'galaxy_package' - def __init__( self, script, path, version ): + def __init__( self, script, path, version, exact=True ): self.script = script self.path = path self.version = version + self._exact = exact + + @property + def exact(self): + return self._exact def shell_commands( self, requirement ): base_path = self.path diff --git a/lib/galaxy/tools/deps/resolvers/modules.py b/lib/galaxy/tools/deps/resolvers/modules.py index a3fa551f35e..7bd22819f3c 100644 --- a/lib/galaxy/tools/deps/resolvers/modules.py +++ b/lib/galaxy/tools/deps/resolvers/modules.py @@ -55,9 +55,9 @@ class ModuleDependencyResolver(DependencyResolver): return INDETERMINATE_DEPENDENCY if self.__has_module(name, version): - return ModuleDependency(self, name, version) + return ModuleDependency(self, name, version, exact=True) elif self.versionless and self.__has_module(name, None): - return ModuleDependency(self, name, None) + return ModuleDependency(self, name, None, exact=False) return INDETERMINATE_DEPENDENCY @@ -155,10 +155,15 @@ class ModuleDependency(Dependency): dict_collection_visible_keys = Dependency.dict_collection_visible_keys + ['module_name', 'module_version'] dependency_type = 'module' - def __init__(self, module_dependency_resolver, module_name, module_version=None): + def __init__(self, module_dependency_resolver, module_name, module_version=None, exact=True): self.module_dependency_resolver = module_dependency_resolver self.module_name = module_name self.module_version = module_version + self._exact = exact + + @property + def exact(self): + return self._exact def shell_commands(self, requirement): module_to_load = self.module_name diff --git a/lib/galaxy/tools/deps/resolvers/resolver_mixins.py b/lib/galaxy/tools/deps/resolvers/resolver_mixins.py index 661d87ecaee..8dfdc422f41 100644 --- a/lib/galaxy/tools/deps/resolvers/resolver_mixins.py +++ b/lib/galaxy/tools/deps/resolvers/resolver_mixins.py @@ -28,7 +28,7 @@ class UsesHomebrewMixin: # Just grab newest installed version - may make sense some day to find # the linked version instead. default_version = sorted(installed_versions, reverse=True)[0] - return self._find_dep_versioned(name, default_version) + return self._find_dep_versioned(name, default_version, exact=version is None) def _installed_versions(self, recipe): recipe_base_path = os.path.join(self.cellar_root, recipe) @@ -62,8 +62,13 @@ class UsesInstalledRepositoriesMixin: class HomebrewDependency(Dependency): - def __init__(self, commands): + def __init__(self, commands, exact=True): self.commands = commands + self._exact = exact + + @property + def exact(self): + return self._exact def shell_commands(self, requirement): raw_commands = self.commands.replace("\n", ";") diff --git a/lib/galaxy/tools/deps/resolvers/tool_shed_packages.py b/lib/galaxy/tools/deps/resolvers/tool_shed_packages.py index d23f9f632bb..c56991fd7dc 100644 --- a/lib/galaxy/tools/deps/resolvers/tool_shed_packages.py +++ b/lib/galaxy/tools/deps/resolvers/tool_shed_packages.py @@ -19,7 +19,7 @@ class ToolShedPackageDependencyResolver(GalaxyPackageDependencyResolver, UsesIns installed_tool_dependency = self._get_installed_dependency( name, type, version=version, **kwds ) if installed_tool_dependency: path = self._get_package_installed_dependency_path( installed_tool_dependency, name, version ) - return self._galaxy_package_dep(path, version) + return self._galaxy_package_dep(path, version, True) else: return INDETERMINATE_DEPENDENCY @@ -32,7 +32,7 @@ class ToolShedPackageDependencyResolver(GalaxyPackageDependencyResolver, UsesIns has_script_dep = is_galaxy_dep and dependency.script and dependency.path if has_script_dep: # Environment settings do not use versions. - return GalaxyPackageDependency(dependency.script, dependency.path, None) + return GalaxyPackageDependency(dependency.script, dependency.path, None, True) return INDETERMINATE_DEPENDENCY def _get_package_installed_dependency_path( self, installed_tool_dependency, name, version ): @@ -58,7 +58,7 @@ class ToolShedPackageDependencyResolver(GalaxyPackageDependencyResolver, UsesIns tool_shed_repository.installed_changeset_revision ) ) if exists( path ): script = join( path, 'env.sh' ) - return GalaxyPackageDependency(script, path, None) + return GalaxyPackageDependency(script, path, None, True) return INDETERMINATE_DEPENDENCY diff --git a/lib/galaxy/tools/deps/resolvers/unlinked_tool_shed_packages.py b/lib/galaxy/tools/deps/resolvers/unlinked_tool_shed_packages.py index 57571ee228b..92f4e7f294b 100644 --- a/lib/galaxy/tools/deps/resolvers/unlinked_tool_shed_packages.py +++ b/lib/galaxy/tools/deps/resolvers/unlinked_tool_shed_packages.py @@ -136,6 +136,11 @@ class UnlinkedToolShedPackageDependencyResolver(GalaxyPackageDependencyResolver) class CandidateDepenency(Dependency): dict_collection_visible_keys = Dependency.dict_collection_visible_keys + ['dependency', 'path', 'owner'] dependency_type = 'unlinked_tool_shed_package' + _exact = True + + @property + def exact(self): + return self._exact def __init__(self, dependency, path, owner=MANUAL): self.dependency = dependency diff --git a/lib/galaxy/webapps/galaxy/api/tool_dependencies.py b/lib/galaxy/webapps/galaxy/api/tool_dependencies.py index 4fe0741832a..adb73232aa0 100644 --- a/lib/galaxy/webapps/galaxy/api/tool_dependencies.py +++ b/lib/galaxy/webapps/galaxy/api/tool_dependencies.py @@ -61,6 +61,9 @@ class ToolDependenciesAPIController( BaseAPIController ): :param name: name of the requirement to find a dependency for (required) :type version: str :param version: version of the requirement to find a dependency for (required) + :type exact: bool + :param version: require an exact match to specify requirement (do not discard + version information to resolve dependency). :rtype: dict :returns: a dictified description of the dependency, with type: null @@ -83,6 +86,9 @@ class ToolDependenciesAPIController( BaseAPIController ): :param name: name of the requirement to find a dependency for (required) :type version: str :param version: version of the requirement to find a dependency for (required) + :type exact: bool + :param version: require an exact match to specify requirement (do not discard + version information to resolve dependency). :rtype: dict :returns: a dictified description of the dependency, with type: null From 71a876840a6c9acf11bd2ac7706ec21b32d2fb88 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Sat, 26 Dec 2015 22:09:08 +0000 Subject: [PATCH 4/8] Implement concept of listable dependency resolvers. DependencyResolver classes which can enumerate the requirements then can resolve "exactly" should mix in ListableDependencyResolver and implement list_dependencies. The galaxy package resolver can easily be enumerated but its subclasses like the tool shed dependency resolver couldn't, so I have separated that class into a base class that there others can inherit and an actual resolver that implements ListableDependencyResolver. Rebase into listable. --- lib/galaxy/tools/deps/resolvers/__init__.py | 19 ++++++++++ lib/galaxy/tools/deps/resolvers/conda.py | 10 ++++- .../tools/deps/resolvers/galaxy_packages.py | 31 +++++++++++++-- .../deps/resolvers/tool_shed_packages.py | 4 +- .../resolvers/unlinked_tool_shed_packages.py | 6 +-- lib/galaxy/tools/deps/views.py | 23 ++++++++++- .../webapps/galaxy/api/tool_dependencies.py | 38 +++++++++++++++++++ lib/galaxy/webapps/galaxy/buildapp.py | 2 + 8 files changed, 123 insertions(+), 10 deletions(-) diff --git a/lib/galaxy/tools/deps/resolvers/__init__.py b/lib/galaxy/tools/deps/resolvers/__init__.py index 256df878439..37d3447f65e 100644 --- a/lib/galaxy/tools/deps/resolvers/__init__.py +++ b/lib/galaxy/tools/deps/resolvers/__init__.py @@ -1,5 +1,7 @@ from galaxy.util.dictifiable import Dictifiable +from ..requirements import ToolRequirement + from abc import ABCMeta, abstractmethod, abstractproperty @@ -36,6 +38,23 @@ class DependencyResolver(Dictifiable, object): return default +class ListableDependencyResolver: + """ Mix this into a ``DependencyResolver`` and implement to indicate + the dependency resolver can iterate over its dependencies and generate + requirements. + """ + __metaclass__ = ABCMeta + + @abstractmethod + def list_dependencies(self): + """ List the "simple" requirements that may be resolved "exact"-ly + by this dependency resolver. + """ + + def _to_requirement(self, name, version=None): + return ToolRequirement(name=name, type="package", version=version) + + class Dependency(Dictifiable, object): dict_collection_visible_keys = ['dependency_type', 'exact'] __metaclass__ = ABCMeta diff --git a/lib/galaxy/tools/deps/resolvers/conda.py b/lib/galaxy/tools/deps/resolvers/conda.py index a3d4ef5895b..1110f3da5dc 100644 --- a/lib/galaxy/tools/deps/resolvers/conda.py +++ b/lib/galaxy/tools/deps/resolvers/conda.py @@ -10,6 +10,7 @@ from ..resolvers import ( DependencyResolver, INDETERMINATE_DEPENDENCY, Dependency, + ListableDependencyResolver, ) from ..conda_util import ( CondaContext, @@ -18,6 +19,7 @@ from ..conda_util import ( is_conda_target_installed, install_conda_target, build_isolated_environment, + installed_conda_targets, ) DEFAULT_BASE_PATH_DIRECTORY = "_conda" @@ -27,7 +29,7 @@ import logging log = logging.getLogger(__name__) -class CondaDependencyResolver(DependencyResolver): +class CondaDependencyResolver(DependencyResolver, ListableDependencyResolver): dict_collection_visible_keys = DependencyResolver.dict_collection_visible_keys + ['conda_prefix', 'versionless', 'ensure_channels', 'auto_install'] resolver_type = "conda" @@ -121,6 +123,12 @@ class CondaDependencyResolver(DependencyResolver): else: raise Exception("Conda dependency seemingly installed but failed to build job environment.") + def list_dependencies(self): + for install_target in installed_conda_targets(self.conda_context): + name = install_target.package + version = install_target.version + yield self._to_requirement(name, version) + @property def prefix(self): return self.conda_context.conda_prefix diff --git a/lib/galaxy/tools/deps/resolvers/galaxy_packages.py b/lib/galaxy/tools/deps/resolvers/galaxy_packages.py index ee5d16113c2..439da23f5f5 100644 --- a/lib/galaxy/tools/deps/resolvers/galaxy_packages.py +++ b/lib/galaxy/tools/deps/resolvers/galaxy_packages.py @@ -1,15 +1,20 @@ +from os import listdir from os.path import join, islink, realpath, basename, exists -from ..resolvers import DependencyResolver, INDETERMINATE_DEPENDENCY, Dependency +from ..resolvers import ( + DependencyResolver, + INDETERMINATE_DEPENDENCY, + Dependency, + ListableDependencyResolver, +) from .resolver_mixins import UsesToolDependencyDirMixin import logging log = logging.getLogger( __name__ ) -class GalaxyPackageDependencyResolver(DependencyResolver, UsesToolDependencyDirMixin): +class BaseGalaxyPackageDependencyResolver(DependencyResolver, UsesToolDependencyDirMixin): dict_collection_visible_keys = DependencyResolver.dict_collection_visible_keys + ['base_path', 'versionless'] - resolver_type = "galaxy_packages" def __init__(self, dependency_manager, **kwds): # Galaxy tool shed requires explicit versions on XML elements, @@ -55,6 +60,26 @@ class GalaxyPackageDependencyResolver(DependencyResolver, UsesToolDependencyDirM return INDETERMINATE_DEPENDENCY +class GalaxyPackageDependencyResolver(BaseGalaxyPackageDependencyResolver, ListableDependencyResolver): + resolver_type = "galaxy_packages" + + def list_dependencies(self): + base_path = self.base_path + for package_name in listdir(base_path): + package_dir = join(base_path, package_name) + for version in listdir(package_dir): + version_dir = join(package_dir, version) + if version == "default": + version = None + valid_dependency = _is_dependency_directory(version_dir) + if valid_dependency: + yield self._to_requirement(package_name, version) + + +def _is_dependency_directory(directory): + return exists(join(directory, 'env.sh')) or exists(join(directory, 'bin')) + + class GalaxyPackageDependency(Dependency): dict_collection_visible_keys = Dependency.dict_collection_visible_keys + ['script', 'path', 'version'] dependency_type = 'galaxy_package' diff --git a/lib/galaxy/tools/deps/resolvers/tool_shed_packages.py b/lib/galaxy/tools/deps/resolvers/tool_shed_packages.py index c56991fd7dc..db62fc56fbc 100644 --- a/lib/galaxy/tools/deps/resolvers/tool_shed_packages.py +++ b/lib/galaxy/tools/deps/resolvers/tool_shed_packages.py @@ -1,11 +1,11 @@ from os.path import abspath, join, exists from .resolver_mixins import UsesInstalledRepositoriesMixin -from .galaxy_packages import GalaxyPackageDependencyResolver, GalaxyPackageDependency +from .galaxy_packages import BaseGalaxyPackageDependencyResolver, GalaxyPackageDependency from ..resolvers import INDETERMINATE_DEPENDENCY -class ToolShedPackageDependencyResolver(GalaxyPackageDependencyResolver, UsesInstalledRepositoriesMixin): +class ToolShedPackageDependencyResolver(BaseGalaxyPackageDependencyResolver, UsesInstalledRepositoriesMixin): resolver_type = "tool_shed_packages" # Resolution of these dependencies depends on more than just the requirement # tag, it depends on the tool installation context - therefore these are diff --git a/lib/galaxy/tools/deps/resolvers/unlinked_tool_shed_packages.py b/lib/galaxy/tools/deps/resolvers/unlinked_tool_shed_packages.py index 92f4e7f294b..96c19ae5fef 100644 --- a/lib/galaxy/tools/deps/resolvers/unlinked_tool_shed_packages.py +++ b/lib/galaxy/tools/deps/resolvers/unlinked_tool_shed_packages.py @@ -22,7 +22,7 @@ See bottom for instructions on how to add this resolver. from os import listdir from os.path import join, exists, getmtime -from .galaxy_packages import GalaxyPackageDependencyResolver +from .galaxy_packages import BaseGalaxyPackageDependencyResolver from ..resolvers import INDETERMINATE_DEPENDENCY, Dependency import logging @@ -32,8 +32,8 @@ MANUAL = "manual" PREFERRED_OWNERS = MANUAL + ",iuc,devteam" -class UnlinkedToolShedPackageDependencyResolver(GalaxyPackageDependencyResolver): - dict_collection_visible_keys = GalaxyPackageDependencyResolver.dict_collection_visible_keys + ['preferred_owners', 'select_by_owner'] +class UnlinkedToolShedPackageDependencyResolver(BaseGalaxyPackageDependencyResolver): + dict_collection_visible_keys = BaseGalaxyPackageDependencyResolver.dict_collection_visible_keys + ['preferred_owners', 'select_by_owner'] resolver_type = "unlinked_tool_shed_packages" def __init__(self, dependency_manager, **kwds): diff --git a/lib/galaxy/tools/deps/views.py b/lib/galaxy/tools/deps/views.py index acc465658f0..3e228a99287 100644 --- a/lib/galaxy/tools/deps/views.py +++ b/lib/galaxy/tools/deps/views.py @@ -1,4 +1,7 @@ -from galaxy.exceptions import RequestParameterMissingException +from galaxy.exceptions import ( + RequestParameterMissingException, + NotImplemented +) class DependencyResolversView(object): @@ -18,6 +21,24 @@ class DependencyResolversView(object): def reload(self): self.toolbox.reload_dependency_manager() + def manager_requirements(self): + requirements = [] + for index, resolver in enumerate(self._dependency_resolvers): + if not hasattr(resolver, "list_dependencies"): + continue + for requirement in resolver.list_dependencies(): + requirements.append({"index": index, "requirement": requirement.to_dict()}) + return requirements + + def resolver_requirements(self, index): + requirements = [] + resolver = self._dependency_resolver(index) + if not hasattr(resolver, "list_dependencies"): + raise NotImplemented() + for requirement in resolver.list_dependencies(): + requirements.append(requirement.to_dict()) + return requirements + def manager_dependency(self, **kwds): return self._dependency(**kwds) diff --git a/lib/galaxy/webapps/galaxy/api/tool_dependencies.py b/lib/galaxy/webapps/galaxy/api/tool_dependencies.py index adb73232aa0..e9e1ee794d4 100644 --- a/lib/galaxy/webapps/galaxy/api/tool_dependencies.py +++ b/lib/galaxy/webapps/galaxy/api/tool_dependencies.py @@ -95,3 +95,41 @@ class ToolDependenciesAPIController( BaseAPIController ): if no match was found. """ return self._view.manager_dependency(**kwds) + + @expose_api + @require_admin + def resolver_requirements(self, trans, id, **kwds): + """ + GET /api/dependencies_resolver/{index}/requirements + + Find all "simple" requirements that could be resolved "exactly" + by this dependency resolver. The dependency resolver must implement + ListDependencyResolver. + + :type index: int + :param index: index of the dependncy resolver + + :rtype: dict + :returns: a dictified description of the requirement that could + be resolved. + """ + return self._view.resolver_requirements(id) + + @expose_api + @require_admin + def manager_requirements(self, trans, **kwds): + """ + GET /api/dependencies_resolver/requirements + + Find all "simple" requirements that could be resolved "exactly" + by all dependency resolvers that support this operation. + + :type index: int + :param index: index of the dependncy resolver + + :rtype: dict + :returns: a dictified description of the requirement that could + be resolved (keyed on 'requirement') and the index of + the corresponding resolver (keyed on 'index'). + """ + return self._view.manager_requirements() diff --git a/lib/galaxy/webapps/galaxy/buildapp.py b/lib/galaxy/webapps/galaxy/buildapp.py index ef4f10b29b1..6752c3102cc 100644 --- a/lib/galaxy/webapps/galaxy/buildapp.py +++ b/lib/galaxy/webapps/galaxy/buildapp.py @@ -252,7 +252,9 @@ def populate_api_routes( webapp, app ): webapp.mapper.resource( 'tool', 'tools', path_prefix='/api' ) webapp.mapper.connect( '/api/dependency_resolvers/dependency', action="manager_dependency", controller="tool_dependencies" ) + webapp.mapper.connect( '/api/dependency_resolvers/requirements', action="manager_requirements", controller="tool_dependencies" ) webapp.mapper.connect( '/api/dependency_resolvers/{id}/dependency', action="resolver_dependency", controller="tool_dependencies" ) + webapp.mapper.connect( '/api/dependency_resolvers/{id}/requirements', action="resolver_requirements", controller="tool_dependencies" ) webapp.mapper.resource( 'dependency_resolver', 'dependency_resolvers', controller="tool_dependencies", path_prefix='api' ) webapp.mapper.resource_with_deleted( 'user', 'users', path_prefix='/api' ) From 29f20cf09c18e24e898517bf685418ec1db4cd11 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Sun, 27 Dec 2015 00:32:50 +0000 Subject: [PATCH 5/8] Implement concept of installable dependency resolvers. DependencyResolver classes which can install "simple" dependences should mix in the new InstallableDependencyResolver and implement install_dependency. Currently this is only the conda dependency resolver, one can easily imagine doing the same for the brew dependency resolver and potentially doing it for some variant of the unlinked dependency resolvers. --- lib/galaxy/tools/deps/resolvers/__init__.py | 13 +++++++++ lib/galaxy/tools/deps/resolvers/conda.py | 16 +++++++++++- lib/galaxy/tools/deps/views.py | 29 +++++++++++++++++---- lib/galaxy/webapps/galaxy/buildapp.py | 3 ++- 4 files changed, 54 insertions(+), 7 deletions(-) diff --git a/lib/galaxy/tools/deps/resolvers/__init__.py b/lib/galaxy/tools/deps/resolvers/__init__.py index 37d3447f65e..86cae5f4a4a 100644 --- a/lib/galaxy/tools/deps/resolvers/__init__.py +++ b/lib/galaxy/tools/deps/resolvers/__init__.py @@ -55,6 +55,19 @@ class ListableDependencyResolver: return ToolRequirement(name=name, type="package", version=version) +class InstallableDependencyResolver: + """ Mix this into a ``DependencyResolver`` and implement to indicate + the dependency resolver can attempt to install new dependencies. + """ + __metaclass__ = ABCMeta + + @abstractmethod + def install_dependency(self, name, version, type, **kwds): + """ Attempt to install this dependency if a recipe to do so + has been registered in some way. + """ + + class Dependency(Dictifiable, object): dict_collection_visible_keys = ['dependency_type', 'exact'] __metaclass__ = ABCMeta diff --git a/lib/galaxy/tools/deps/resolvers/conda.py b/lib/galaxy/tools/deps/resolvers/conda.py index 1110f3da5dc..f7dc5af3b8d 100644 --- a/lib/galaxy/tools/deps/resolvers/conda.py +++ b/lib/galaxy/tools/deps/resolvers/conda.py @@ -6,11 +6,14 @@ incompatible changes coming. import os +from galaxy.exceptions import NotImplemented + from ..resolvers import ( DependencyResolver, INDETERMINATE_DEPENDENCY, Dependency, ListableDependencyResolver, + InstallableDependencyResolver, ) from ..conda_util import ( CondaContext, @@ -29,7 +32,7 @@ import logging log = logging.getLogger(__name__) -class CondaDependencyResolver(DependencyResolver, ListableDependencyResolver): +class CondaDependencyResolver(DependencyResolver, ListableDependencyResolver, InstallableDependencyResolver): dict_collection_visible_keys = DependencyResolver.dict_collection_visible_keys + ['conda_prefix', 'versionless', 'ensure_channels', 'auto_install'] resolver_type = "conda" @@ -129,6 +132,17 @@ class CondaDependencyResolver(DependencyResolver, ListableDependencyResolver): version = install_target.version yield self._to_requirement(name, version) + def install_dependency(self, name, version, type, **kwds): + if type != "package": + raise NotImplemented("Can only install dependencies of type '%s'" % type) + + if self.versionless: + version = None + conda_target = CondaTarget(name, version=version) + + if install_conda_target(conda_target, conda_context=self.conda_context): + raise Exception("Failed to install conda recipe.") + @property def prefix(self): return self.conda_context.conda_prefix diff --git a/lib/galaxy/tools/deps/views.py b/lib/galaxy/tools/deps/views.py index 3e228a99287..f594f87efdb 100644 --- a/lib/galaxy/tools/deps/views.py +++ b/lib/galaxy/tools/deps/views.py @@ -45,24 +45,43 @@ class DependencyResolversView(object): def resolver_dependency(self, index, **kwds): return self._dependency(**kwds) + def install_dependency(self, index, payload): + resolver = self._dependency_resolver(index) + if not hasattr(resolver, "install_dependency"): + raise NotImplemented() + + name, version, type, extra_kwds = self._parse_dependency_info(payload) + return resolver.install_dependency( + name=name, + version=version, + type=type, + **extra_kwds + ) + def _dependency(self, index=None, **kwds): if index is not None: index = int(index) - name = kwds.get("name", None) - if name is None: - raise RequestParameterMissingException("Missing 'name' parameter required for resolution.") - version = kwds.get("version", None) - type = kwds.get("type", "package") + name, version, type, extra_kwds = self._parse_dependency_info(kwds) resolve_kwds = dict( job_directory="/path/to/example/job_directory", index=index, + **extra_kwds ) dependency = self._dependency_manager.find_dep( name, version=version, type=type, **resolve_kwds ) return dependency.to_dict() + def _parse_dependency_info(self, kwds): + extra_kwds = kwds.copy() + name = extra_kwds.pop("name", None) + if name is None: + raise RequestParameterMissingException("Missing 'name' parameter required for resolution.") + version = extra_kwds.pop("version", None) + type = extra_kwds.pop("type", "package") + return name, version, type, extra_kwds + def _dependency_resolver(self, index): index = int(index) return self._dependency_resolvers[index] diff --git a/lib/galaxy/webapps/galaxy/buildapp.py b/lib/galaxy/webapps/galaxy/buildapp.py index 6752c3102cc..350be892581 100644 --- a/lib/galaxy/webapps/galaxy/buildapp.py +++ b/lib/galaxy/webapps/galaxy/buildapp.py @@ -253,7 +253,8 @@ def populate_api_routes( webapp, app ): webapp.mapper.connect( '/api/dependency_resolvers/dependency', action="manager_dependency", controller="tool_dependencies" ) webapp.mapper.connect( '/api/dependency_resolvers/requirements', action="manager_requirements", controller="tool_dependencies" ) - webapp.mapper.connect( '/api/dependency_resolvers/{id}/dependency', action="resolver_dependency", controller="tool_dependencies" ) + webapp.mapper.connect( '/api/dependency_resolvers/{id}/dependency', action="resolver_dependency", controller="tool_dependencies", method=['GET'] ) + webapp.mapper.connect( '/api/dependency_resolvers/{id}/dependency', action="install_dependency", controller="tool_dependencies", method=['POST'] ) webapp.mapper.connect( '/api/dependency_resolvers/{id}/requirements', action="resolver_requirements", controller="tool_dependencies" ) webapp.mapper.resource( 'dependency_resolver', 'dependency_resolvers', controller="tool_dependencies", path_prefix='api' ) From 868a7dfe0e03503adaf6646e10d8184d18b66a34 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Sat, 2 Jan 2016 12:38:20 +0000 Subject: [PATCH 6/8] Modify unmodified conda behavior. - Add configuration option for overriding condarc location used by conda resolve commands. - Default location of condarc to be in tool_dependency_dir. - Delay updating the condarc to ensure channels until this would actually be needed (during package installation). - Add configuration flag to allow Galaxy to *NOT* check the PATH for conda. - Update unit test that verifies conda's installation to never use conda on PATH, since a precondition of this test is that certain packages haven't been installed previously. - Change default behavior to not check the PATH for conda, everything else here I feel pretty good about - this I'm ambivalent about - it will reduce error reports and problems with network filesystems, but doesn't seem to be good Unix best practice. Points 2 and 3 above should address this issue: https://github.com/galaxyproject/ansible-galaxy-extras/pull/33#issuecomment-168384555 This commit also addresses most of #1406. --- lib/galaxy/tools/deps/conda_util.py | 42 ++++++++++++++++++++---- lib/galaxy/tools/deps/resolvers/conda.py | 15 +++++++++ test/unit/tools/test_conda_resolution.py | 1 + 3 files changed, 51 insertions(+), 7 deletions(-) diff --git a/lib/galaxy/tools/deps/conda_util.py b/lib/galaxy/tools/deps/conda_util.py index 2bb33ed5f57..a986038cff8 100644 --- a/lib/galaxy/tools/deps/conda_util.py +++ b/lib/galaxy/tools/deps/conda_util.py @@ -22,6 +22,7 @@ IS_OS_X = _platform == "darwin" CONDA_LICENSE = "http://docs.continuum.io/anaconda/eula" VERSIONED_ENV_DIR_NAME = re.compile(r"__package__(.*)@__version__(.*)") UNVERSIONED_ENV_DIR_NAME = re.compile(r"__package__(.*)@__unversioned__") +USE_PATH_EXEC_DEFAULT = False def conda_link(): @@ -44,8 +45,11 @@ def find_conda_prefix(conda_prefix=None): class CondaContext(object): def __init__(self, conda_prefix=None, conda_exec=None, - shell_exec=None, debug=False, ensure_channels=''): - conda_exec = conda_exec or commands.which("conda") + shell_exec=None, debug=False, ensure_channels='', + condarc_override=None, use_path_exec=USE_PATH_EXEC_DEFAULT): + self.condarc_override = condarc_override + if not conda_exec and use_path_exec: + conda_exec = commands.which("conda") if conda_exec: conda_exec = os.path.normpath(conda_exec) self.conda_exec = conda_exec @@ -65,13 +69,21 @@ class CondaContext(object): if ensure_channels: if not isinstance(ensure_channels, list): ensure_channels = [c for c in ensure_channels.split(",") if c] + else: + ensure_channels = None + self.ensure_channels = ensure_channels + self.ensured_channels = False + + def ensure_channels_configured(self): + if not self.ensured_channels: + self.ensured_channels = True changed = False conda_conf = self.load_condarc() if "channels" not in conda_conf: conda_conf["channels"] = [] channels = conda_conf["channels"] - for channel in ensure_channels: + for channel in self.ensure_channels: if channel not in channels: changed = True channels.append(channel) @@ -97,12 +109,23 @@ class CondaContext(object): def save_condarc(self, conf): condarc = self.condarc - with open(condarc, "w") as f: - return yaml.safe_dump(conf, f) + try: + with open(condarc, "w") as f: + return yaml.safe_dump(conf, f) + except IOError: + template = ("Failed to update write to path [%s] while attempting to update conda configuration, " + "please update the configuration to override the condarc location or " + "grant this application write to the parent directory.") + message = template % condarc + raise Exception(message) @property def condarc(self): - return os.path.join(os.path.expanduser("~"), ".condarc") + if self.condarc_override: + return self.condarc_override + else: + home = os.path.expanduser("~") + return os.path.join(home, ".condarc") def command(self, operation, args): if isinstance(args, list): @@ -114,7 +137,11 @@ class CondaContext(object): def exec_command(self, operation, args): command = self.command(operation, args) - self.shell_exec(command) + env = {} + condarc_override = self.condarc_override + if condarc_override: + env["CONDARC"] = condarc_override + self.shell_exec(command, env=env) def exec_create(self, args): create_base_args = [ @@ -249,6 +276,7 @@ def install_conda_target(conda_target, conda_context=None): """ Install specified target into a its own environment. """ conda_context = _ensure_conda_context(conda_context) + conda_context.ensure_channels_configured() create_args = [ "--name", conda_target.install_environment, # enviornment for package conda_target.package_specifier, diff --git a/lib/galaxy/tools/deps/resolvers/conda.py b/lib/galaxy/tools/deps/resolvers/conda.py index f7dc5af3b8d..2bb304ce439 100644 --- a/lib/galaxy/tools/deps/resolvers/conda.py +++ b/lib/galaxy/tools/deps/resolvers/conda.py @@ -23,9 +23,11 @@ from ..conda_util import ( install_conda_target, build_isolated_environment, installed_conda_targets, + USE_PATH_EXEC_DEFAULT, ) DEFAULT_BASE_PATH_DIRECTORY = "_conda" +DEFAULT_CONDARC_OVERRIDE = "_condarc" DEFAULT_ENSURE_CHANNELS = "r,bioconda" import logging @@ -49,9 +51,20 @@ class CondaDependencyResolver(DependencyResolver, ListableDependencyResolver, In dependency_manager.default_base_path, DEFAULT_BASE_PATH_DIRECTORY ) + condarc_override = get_option("condarc_override") + if condarc_override is None: + condarc_override = os.path.join( + dependency_manager.default_base_path, DEFAULT_CONDARC_OVERRIDE + ) + conda_exec = get_option("exec") debug = _string_as_bool(get_option("debug")) ensure_channels = get_option("ensure_channels") + use_path_exec = get_option("use_path_exec") + if use_path_exec is None: + use_path_exec = USE_PATH_EXEC_DEFAULT + else: + use_path_exec = _string_as_bool(use_path_exec) if ensure_channels is None: ensure_channels = DEFAULT_ENSURE_CHANNELS @@ -60,6 +73,8 @@ class CondaDependencyResolver(DependencyResolver, ListableDependencyResolver, In conda_exec=conda_exec, debug=debug, ensure_channels=ensure_channels, + condarc_override=condarc_override, + use_path_exec=use_path_exec, ) self.ensure_channels = ensure_channels diff --git a/test/unit/tools/test_conda_resolution.py b/test/unit/tools/test_conda_resolution.py index 5e8e7116c76..fa4bab657f2 100644 --- a/test/unit/tools/test_conda_resolution.py +++ b/test/unit/tools/test_conda_resolution.py @@ -30,6 +30,7 @@ def test_conda_resolution(): dependency_manager, auto_init=True, auto_install=True, + use_path_exec=False, # For the test ensure this is always a clean install ) conda_context = resolver.conda_context assert len(list(conda_util.installed_conda_targets(conda_context))) == 0 From ef9e1876a2d4dd02718784f2b9f1045f2f9d0b59 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Sun, 3 Jan 2016 12:29:15 +0000 Subject: [PATCH 7/8] Fixup dependency module and API comments. Thanks to suggestions by @bgruening. --- lib/galaxy/tools/deps/resolvers/__init__.py | 4 +++- .../webapps/galaxy/api/tool_dependencies.py | 16 ++++++++-------- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/lib/galaxy/tools/deps/resolvers/__init__.py b/lib/galaxy/tools/deps/resolvers/__init__.py index 86cae5f4a4a..b4375cf4d07 100644 --- a/lib/galaxy/tools/deps/resolvers/__init__.py +++ b/lib/galaxy/tools/deps/resolvers/__init__.py @@ -10,7 +10,9 @@ class DependencyResolver(Dictifiable, object): dict_collection_visible_keys = ['resolver_type', 'resolves_simple_dependencies'] # A "simple" dependency is one that does not depend on the the tool # resolving the dependency. Classic tool shed dependencies are non-simple - # because the tool shed install. + # because the repository install context is used in dependency resolution + # so the same requirement tags in different tools will have very different + # resolution. resolves_simple_dependencies = True __metaclass__ = ABCMeta diff --git a/lib/galaxy/webapps/galaxy/api/tool_dependencies.py b/lib/galaxy/webapps/galaxy/api/tool_dependencies.py index e9e1ee794d4..7e31996e1f6 100644 --- a/lib/galaxy/webapps/galaxy/api/tool_dependencies.py +++ b/lib/galaxy/webapps/galaxy/api/tool_dependencies.py @@ -54,7 +54,7 @@ class ToolDependenciesAPIController( BaseAPIController ): Resolve described requirement against specified dependency resolver. :type index: int - :param index: index of the dependncy resolver + :param index: index of the dependency resolver :type kwds: dict :param kwds: dictionary structure containing extra parameters :type name: str @@ -62,12 +62,12 @@ class ToolDependenciesAPIController( BaseAPIController ): :type version: str :param version: version of the requirement to find a dependency for (required) :type exact: bool - :param version: require an exact match to specify requirement (do not discard + :param exact: require an exact match to specify requirement (do not discard version information to resolve dependency). :rtype: dict - :returns: a dictified description of the dependency, with type: null - if no match was found. + :returns: a dictified description of the dependency, with attribute + ``dependency_type: null`` if no match was found. """ return self._view.resolver_dependency(id, **kwds) @@ -79,7 +79,7 @@ class ToolDependenciesAPIController( BaseAPIController ): the match with highest priority. :type index: int - :param index: index of the dependncy resolver + :param index: index of the dependency resolver :type kwds: dict :param kwds: dictionary structure containing extra parameters :type name: str @@ -87,7 +87,7 @@ class ToolDependenciesAPIController( BaseAPIController ): :type version: str :param version: version of the requirement to find a dependency for (required) :type exact: bool - :param version: require an exact match to specify requirement (do not discard + :param exact: require an exact match to specify requirement (do not discard version information to resolve dependency). :rtype: dict @@ -107,7 +107,7 @@ class ToolDependenciesAPIController( BaseAPIController ): ListDependencyResolver. :type index: int - :param index: index of the dependncy resolver + :param index: index of the dependency resolver :rtype: dict :returns: a dictified description of the requirement that could @@ -125,7 +125,7 @@ class ToolDependenciesAPIController( BaseAPIController ): by all dependency resolvers that support this operation. :type index: int - :param index: index of the dependncy resolver + :param index: index of the dependency resolver :rtype: dict :returns: a dictified description of the requirement that could From 4650d77bd988b90b6652cc108f83bf6d86cae271 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Sun, 3 Jan 2016 12:32:44 +0000 Subject: [PATCH 8/8] Fix listing conda requirements if conda hasn't been installed. Fix problem discovered by @bgruening. --- lib/galaxy/tools/deps/conda_util.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/galaxy/tools/deps/conda_util.py b/lib/galaxy/tools/deps/conda_util.py index a986038cff8..1ab685ece34 100644 --- a/lib/galaxy/tools/deps/conda_util.py +++ b/lib/galaxy/tools/deps/conda_util.py @@ -188,7 +188,8 @@ class CondaContext(object): def installed_conda_targets(conda_context): envs_path = conda_context.envs_path - for name in os.listdir(envs_path): + dir_contents = os.listdir(envs_path) if os.path.exists(envs_path) else [] + for name in dir_contents: versioned_match = VERSIONED_ENV_DIR_NAME.match(name) if versioned_match: yield CondaTarget(versioned_match.group(1), versioned_match.group(2))