From cd2cf494fe1ede4ae5b43417f0da21b2ca6473a0 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Tue, 17 Jan 2017 20:01:37 -0500 Subject: [PATCH 01/12] Allow finer grain specification of requirements. This adapts CWL-style ``SoftwareRequirement`` ``specs`` to solve #1927. Here I'm trying to implement the CWL specification in a way that helps enable the feasibility of Conda packaging in Galaxy. It is a delicate balancing act aimed to upset as many interested parties as I can. To understand the problem - consider the ``blast+`` requirement found in the Galaxy wrappers. It looks something like this: ``` ``` Great, that works for Galaxy and Tool Shed packages. It doesn't work for bioconda at all. I think this problem is fairly uncommon - most packages have simple names shared across Debian, Brew, Conda, etc... - but it does happen in some cases that there are inconsistencies. Some people have taken to duplicating the requirement - this is bad and should not be done since they are mutually exclusive and Galaxy will attempt to resolve both. This introduces the following syntax for tools with profile >= 16.10: ``` ``` This allows finer grain resolution of the requirement without sacrificing the abstract name at the top. It allows the name and the version to be adapted by resolvers as needed (hopefully rarely so). This syntax is the future facing one, but obviously this tool would not work on older Galaxy versions. To remedy this - an alternative syntax can be used for tools targetting Galaxy verions pre-16.10: ``` blast+ ``` This syntax sucks - but it does give newer Galaxies the ability to resolve the specifications without breaking the more simple functionality for older Galaxies. For more information on the CWL side of this - checkout the discussion on https://github.com/common-workflow-language/cwltool/pull/214. The CWL specification information is defined at http://www.commonwl.org/v1.0/CommandLineTool.html#SoftwarePackage. Conflicts: lib/galaxy/tools/deps/__init__.py test/functional/tools/samples_tool_conf.xml --- lib/galaxy/tools/deps/__init__.py | 13 +++- lib/galaxy/tools/deps/requirements.py | 76 +++++++++++++++++-- lib/galaxy/tools/deps/resolvers/__init__.py | 28 +++++++ lib/galaxy/tools/deps/resolvers/conda.py | 5 +- lib/galaxy/tools/xsd/galaxy.xsd | 39 ++++++---- .../tools/requirement_specification_1.xml | 18 +++++ .../tools/requirement_specification_2.xml | 16 ++++ 7 files changed, 171 insertions(+), 24 deletions(-) create mode 100644 test/functional/tools/requirement_specification_1.xml create mode 100644 test/functional/tools/requirement_specification_2.xml diff --git a/lib/galaxy/tools/deps/__init__.py b/lib/galaxy/tools/deps/__init__.py index 6f254d3965a..cc68b3c6249 100644 --- a/lib/galaxy/tools/deps/__init__.py +++ b/lib/galaxy/tools/deps/__init__.py @@ -141,6 +141,7 @@ class DependencyManager( object ): # Check requirements all at once all_unmet = len(requirement_to_dependency) == 0 if all_unmet and hasattr(resolver, "resolve_all"): + # TODO: Handle specs. dependencies = resolver.resolve_all(resolvable_requirements, **kwds) if dependencies: assert len(dependencies) == len(resolvable_requirements) @@ -155,7 +156,17 @@ class DependencyManager( object ): if requirement in requirement_to_dependency: continue - dependency = resolver.resolve( requirement.name, requirement.version, requirement.type, **kwds ) + name = requirement.name + version = requirement.version + specs = requirement.specs + + if hasattr(resolver, "find_specification"): + spec = resolver.find_specification(specs) + if spec is not None: + name = spec.short_name + version = spec.version or version + + dependency = resolver.resolve( name, version, requirement.type, **kwds ) if require_exact and not dependency.exact: continue diff --git a/lib/galaxy/tools/deps/requirements.py b/lib/galaxy/tools/deps/requirements.py index bb2cfaa612f..2b041fc0348 100644 --- a/lib/galaxy/tools/deps/requirements.py +++ b/lib/galaxy/tools/deps/requirements.py @@ -14,29 +14,66 @@ class ToolRequirement( object ): run (for example, a program, package, or library). Requirements can optionally assert a specific version. """ - def __init__( self, name=None, type=None, version=None ): + def __init__( self, name=None, type=None, version=None, specs=[] ): self.name = name self.type = type self.version = version + self.specs = specs def to_dict( self ): - return dict(name=self.name, type=self.type, version=self.version) + specs = [s.to_dict() for s in self.specs] + return dict(name=self.name, type=self.type, version=self.version, specs=specs) @staticmethod def from_dict( dict ): version = dict.get( "version", None ) name = dict.get("name", None) type = dict.get("type", None) - return ToolRequirement( name=name, type=type, version=version ) + specs = [RequirementSpecification.from_dict(s) for s in dict.get("specs", [])] + return ToolRequirement( name=name, type=type, version=version, specs=specs ) def __eq__(self, other): - return self.name == other.name and self.type == other.type and self.version == other.version + return self.name == other.name and self.type == other.type and self.version == other.version and self.specs == other.specs def __ne__(self, other): return not self.__eq__(other) def __hash__(self): - return hash((self.name, self.type, self.version)) + return hash((self.name, self.type, self.version, frozenset(self.specs))) + + +class RequirementSpecification(object): + """Refine a requirement using a URI.""" + + def __init__(self, uri, version=None): + self.uri = uri + self.version = version + + @property + def specifies_version(self): + return self.version is not None + + @property + def short_name(self): + return self.uri.split("/")[-1] + + def to_dict(self): + return dict(uri=self.uri, version=self.version) + + @staticmethod + def from_dict(dict): + uri = dict.get["uri"] + version = dict.get("version", None) + return RequirementSpecification(uri=uri, version=version) + + def __eq__(self, other): + return self.uri == other.uri and self.version == other.version + + def __ne__(self, other): + return not self.__eq__(other) + + def __hash__(self): + return hash((self.uri, self.version)) class ToolRequirements(object): @@ -173,10 +210,29 @@ def parse_requirements_from_xml( xml_root ): requirements = ToolRequirements() for requirement_elem in requirement_elems: - name = xml_text( requirement_elem ) + if "name" in requirement_elem.attrib: + name = requirement_elem.get( "name" ) + spec_elems = requirement_elem.findall("specification") + specs = map(specification_from_element, spec_elems) + else: + name = xml_text( requirement_elem ) + spec_uris_raw = requirement_elem.attrib.get("specification_uris", "") + specs = [] + for spec_uri in spec_uris_raw.split(","): + if not spec_uri: + continue + version = None + if "@" in spec_uri: + uri, version = spec_uri.split("@", 1) + else: + uri = spec_uri + uri = uri.strip() + if version: + version = version.strip() + specs.append(RequirementSpecification(uri, version)) type = requirement_elem.get( "type", DEFAULT_REQUIREMENT_TYPE ) version = requirement_elem.get( "version", DEFAULT_REQUIREMENT_VERSION ) - requirement = ToolRequirement( name=name, type=type, version=version ) + requirement = ToolRequirement( name=name, type=type, version=version, specs=specs ) requirements.append( requirement ) container_elems = [] @@ -188,6 +244,12 @@ def parse_requirements_from_xml( xml_root ): return requirements, containers +def specification_from_element(specification_elem): + uri = specification_elem.get("uri", None) + version = specification_elem.get("version", None) + return RequirementSpecification(uri, version) + + def container_from_element(container_elem): identifier = xml_text(container_elem) type = container_elem.get("type", DEFAULT_CONTAINER_TYPE) diff --git a/lib/galaxy/tools/deps/resolvers/__init__.py b/lib/galaxy/tools/deps/resolvers/__init__.py index c24c107887a..bebf7539e8a 100644 --- a/lib/galaxy/tools/deps/resolvers/__init__.py +++ b/lib/galaxy/tools/deps/resolvers/__init__.py @@ -67,6 +67,34 @@ class ListableDependencyResolver: return ToolRequirement(name=name, type="package", version=version) +class SpecificationAwareDependencyResolver: + """Mix this into a :class:`DependencyResolver` to implement URI specification matching. + + Allows adapting generic requirements to more specific URIs - to tailor name + or version to specified resolution system. + """ + __metaclass__ = ABCMeta + + @abstractmethod + def find_specification(self, specs): + """Find closest matching specification for discovered resolver.""" + + +class SpecificationPatternDependencyResolver: + """Implement the :class:`SpecificationAwareDependencyResolver` with a regex pattern.""" + + @abstractproperty + def _specification_pattern(self): + """Pattern of URI to match against.""" + + def find_specification(self, specs): + pattern = self._specification_pattern + for spec in specs: + if pattern.match(spec.uri): + return spec + return None + + class InstallableDependencyResolver: """ Mix this into a ``DependencyResolver`` and implement to indicate the dependency resolver can attempt to install new dependencies. diff --git a/lib/galaxy/tools/deps/resolvers/conda.py b/lib/galaxy/tools/deps/resolvers/conda.py index f6fe1d3925f..992fbd879e9 100644 --- a/lib/galaxy/tools/deps/resolvers/conda.py +++ b/lib/galaxy/tools/deps/resolvers/conda.py @@ -5,6 +5,7 @@ incompatible changes coming. import logging import os +import re import galaxy.tools.deps.installable @@ -29,6 +30,7 @@ from ..resolvers import ( InstallableDependencyResolver, ListableDependencyResolver, NullDependency, + SpecificationPatternDependencyResolver, ) @@ -39,9 +41,10 @@ DEFAULT_ENSURE_CHANNELS = "iuc,bioconda,r,defaults,conda-forge" log = logging.getLogger(__name__) -class CondaDependencyResolver(DependencyResolver, ListableDependencyResolver, InstallableDependencyResolver): +class CondaDependencyResolver(DependencyResolver, ListableDependencyResolver, InstallableDependencyResolver, SpecificationPatternDependencyResolver): dict_collection_visible_keys = DependencyResolver.dict_collection_visible_keys + ['conda_prefix', 'versionless', 'ensure_channels', 'auto_install'] resolver_type = "conda" + _specification_pattern = re.compile(r"https\:\/\/anaconda.org\/\w+\/\w+") def __init__(self, dependency_manager, **kwds): self.versionless = _string_as_bool(kwds.get('versionless', 'false')) diff --git a/lib/galaxy/tools/xsd/galaxy.xsd b/lib/galaxy/tools/xsd/galaxy.xsd index 5b589306ac3..6790c324a0c 100644 --- a/lib/galaxy/tools/xsd/galaxy.xsd +++ b/lib/galaxy/tools/xsd/galaxy.xsd @@ -227,7 +227,7 @@ complete descriptions of the runtime of a tool. - + - - - - - This value defines the which type of the 3rd party module required by this tool. - - - - - For package type requirements this value defines a specific version of the tool dependency. - - - - + + + + + + This value defines the which type of the 3rd party module required by this tool. + + + + + For package type requirements this value defines a specific version of the tool dependency. + + + + + Name of requirement (if body of ``requirement`` element contains specification URIs). + + + + + URIs and versions of requirement specification. + + diff --git a/test/functional/tools/requirement_specification_1.xml b/test/functional/tools/requirement_specification_1.xml new file mode 100644 index 00000000000..00744e114f4 --- /dev/null +++ b/test/functional/tools/requirement_specification_1.xml @@ -0,0 +1,18 @@ + + $out_file1 ; + echo "Moo" >> $out_file1 ; + ]]> + + + + + + + + + + + + + diff --git a/test/functional/tools/requirement_specification_2.xml b/test/functional/tools/requirement_specification_2.xml new file mode 100644 index 00000000000..e3190f068cc --- /dev/null +++ b/test/functional/tools/requirement_specification_2.xml @@ -0,0 +1,16 @@ + + $out_file1 ; + echo "Moo" >> $out_file1 ; + ]]> + + + blast+ + + + + + + + + From 330d71d37ce7eaa60b5c8c20ae83aeb36fec4a6c Mon Sep 17 00:00:00 2001 From: John Chilton Date: Wed, 18 Jan 2017 10:17:16 -0500 Subject: [PATCH 02/12] Remove tool XML modifications for requirement specifications. Per discussion on #3117 - some people were in favor of the idea but some were opposed. I think it is still important to implement this for CWL tools and useful from a library perspective. This just removes changes to Galaxy tool XML. --- lib/galaxy/tools/deps/requirements.py | 29 +------------- lib/galaxy/tools/xsd/galaxy.xsd | 39 +++++++------------ .../tools/requirement_specification_1.xml | 18 --------- .../tools/requirement_specification_2.xml | 16 -------- 4 files changed, 17 insertions(+), 85 deletions(-) delete mode 100644 test/functional/tools/requirement_specification_1.xml delete mode 100644 test/functional/tools/requirement_specification_2.xml diff --git a/lib/galaxy/tools/deps/requirements.py b/lib/galaxy/tools/deps/requirements.py index 2b041fc0348..92ba382e74b 100644 --- a/lib/galaxy/tools/deps/requirements.py +++ b/lib/galaxy/tools/deps/requirements.py @@ -210,29 +210,10 @@ def parse_requirements_from_xml( xml_root ): requirements = ToolRequirements() for requirement_elem in requirement_elems: - if "name" in requirement_elem.attrib: - name = requirement_elem.get( "name" ) - spec_elems = requirement_elem.findall("specification") - specs = map(specification_from_element, spec_elems) - else: - name = xml_text( requirement_elem ) - spec_uris_raw = requirement_elem.attrib.get("specification_uris", "") - specs = [] - for spec_uri in spec_uris_raw.split(","): - if not spec_uri: - continue - version = None - if "@" in spec_uri: - uri, version = spec_uri.split("@", 1) - else: - uri = spec_uri - uri = uri.strip() - if version: - version = version.strip() - specs.append(RequirementSpecification(uri, version)) + name = xml_text( requirement_elem ) type = requirement_elem.get( "type", DEFAULT_REQUIREMENT_TYPE ) version = requirement_elem.get( "version", DEFAULT_REQUIREMENT_VERSION ) - requirement = ToolRequirement( name=name, type=type, version=version, specs=specs ) + requirement = ToolRequirement( name=name, type=type, version=version ) requirements.append( requirement ) container_elems = [] @@ -244,12 +225,6 @@ def parse_requirements_from_xml( xml_root ): return requirements, containers -def specification_from_element(specification_elem): - uri = specification_elem.get("uri", None) - version = specification_elem.get("version", None) - return RequirementSpecification(uri, version) - - def container_from_element(container_elem): identifier = xml_text(container_elem) type = container_elem.get("type", DEFAULT_CONTAINER_TYPE) diff --git a/lib/galaxy/tools/xsd/galaxy.xsd b/lib/galaxy/tools/xsd/galaxy.xsd index 6790c324a0c..5b589306ac3 100644 --- a/lib/galaxy/tools/xsd/galaxy.xsd +++ b/lib/galaxy/tools/xsd/galaxy.xsd @@ -227,7 +227,7 @@ complete descriptions of the runtime of a tool. - + - - - - - - This value defines the which type of the 3rd party module required by this tool. - - - - - For package type requirements this value defines a specific version of the tool dependency. - - - - - Name of requirement (if body of ``requirement`` element contains specification URIs). - - - - - URIs and versions of requirement specification. - - + + + + + This value defines the which type of the 3rd party module required by this tool. + + + + + For package type requirements this value defines a specific version of the tool dependency. + + + + diff --git a/test/functional/tools/requirement_specification_1.xml b/test/functional/tools/requirement_specification_1.xml deleted file mode 100644 index 00744e114f4..00000000000 --- a/test/functional/tools/requirement_specification_1.xml +++ /dev/null @@ -1,18 +0,0 @@ - - $out_file1 ; - echo "Moo" >> $out_file1 ; - ]]> - - - - - - - - - - - - - diff --git a/test/functional/tools/requirement_specification_2.xml b/test/functional/tools/requirement_specification_2.xml deleted file mode 100644 index e3190f068cc..00000000000 --- a/test/functional/tools/requirement_specification_2.xml +++ /dev/null @@ -1,16 +0,0 @@ - - $out_file1 ; - echo "Moo" >> $out_file1 ; - ]]> - - - blast+ - - - - - - - - From baf9b82847fc526af56ed8efa7aba4d31d62f400 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Thu, 10 Nov 2016 07:15:41 -0500 Subject: [PATCH 03/12] More modular config option handling for dependency resolvers. The handling of global configuration options for Conda specific options was hacky and a poor design. This is much more modular and allows the elegant cascading option sources for all resolvers. --- lib/galaxy/tools/deps/__init__.py | 55 ++++++++++----------- lib/galaxy/tools/deps/resolvers/__init__.py | 13 +---- lib/galaxy/tools/deps/resolvers/conda.py | 11 ++++- 3 files changed, 38 insertions(+), 41 deletions(-) diff --git a/lib/galaxy/tools/deps/__init__.py b/lib/galaxy/tools/deps/__init__.py index cc68b3c6249..8182db3bba1 100644 --- a/lib/galaxy/tools/deps/__init__.py +++ b/lib/galaxy/tools/deps/__init__.py @@ -19,26 +19,12 @@ from .requirements import ( ToolRequirements ) from .resolvers import NullDependency -from .resolvers.conda import CondaDependencyResolver, DEFAULT_ENSURE_CHANNELS +from .resolvers.conda import CondaDependencyResolver from .resolvers.galaxy_packages import GalaxyPackageDependencyResolver from .resolvers.tool_shed_packages import ToolShedPackageDependencyResolver log = logging.getLogger( __name__ ) -# TODO: Load these from the plugins. Would require a two step initialization of -# DependencyManager - where the plugins are loaded first and then the config -# is parsed and sent through. -EXTRA_CONFIG_KWDS = { - 'conda_prefix': None, - 'conda_exec': None, - 'conda_debug': None, - 'conda_ensure_channels': DEFAULT_ENSURE_CHANNELS, - 'conda_auto_install': False, - 'conda_auto_init': False, - 'conda_copy_dependencies': False, - 'precache_dependencies': True, -} - CONFIG_VAL_NOT_FOUND = object() @@ -47,15 +33,9 @@ def build_dependency_manager( config ): dependency_manager_kwds = { 'default_base_path': config.tool_dependency_dir, 'conf_file': config.dependency_resolvers_config_file, + 'app_config': config, } - for key, default_value in EXTRA_CONFIG_KWDS.items(): - value = getattr(config, key, CONFIG_VAL_NOT_FOUND) - if value is CONFIG_VAL_NOT_FOUND and hasattr(config, "config_dict"): - value = config.config_dict.get(key, CONFIG_VAL_NOT_FOUND) - if value is CONFIG_VAL_NOT_FOUND: - value = default_value - dependency_manager_kwds[key] = value - if config.use_cached_dependency_manager: + if getattr(config, "use_cached_dependency_manager", False): dependency_manager_kwds['tool_dependency_cache_dir'] = config.tool_dependency_cache_dir dependency_manager = CachedDependencyManager(**dependency_manager_kwds) else: @@ -90,7 +70,7 @@ class DependencyManager( object ): and should each contain a file 'env.sh' which can be sourced to make the dependency available in the current shell environment. """ - def __init__( self, default_base_path, conf_file=None, **extra_config ): + def __init__( self, default_base_path, conf_file=None, app_config={} ): """ Create a new dependency manager looking for packages under the paths listed in `base_paths`. The default base path is app.config.tool_dependency_dir. @@ -99,11 +79,30 @@ class DependencyManager( object ): log.warning( "Path '%s' does not exist, ignoring", default_base_path ) if not os.path.isdir( default_base_path ): log.warning( "Path '%s' is not directory, ignoring", default_base_path ) - self.extra_config = extra_config + self.__app_config = app_config self.default_base_path = os.path.abspath( default_base_path ) self.resolver_classes = self.__resolvers_dict() self.dependency_resolvers = self.__build_dependency_resolvers( conf_file ) + def get_resolver_option(self, resolver, key, explicit_resolver_options={}): + """Look in resolver-specific settings for option and then fallback to global settings. + """ + default = resolver.config_options.get(key) + config_prefix = resolver.resolver_type + global_key = "%s_%s" % (config_prefix, key) + value = explicit_resolver_options.get(key, CONFIG_VAL_NOT_FOUND) + if value is CONFIG_VAL_NOT_FOUND: + if isinstance(self.__app_config, dict): + value = self.__app_config.get(global_key, CONFIG_VAL_NOT_FOUND) + else: + value = getattr(self.__app_config, global_key, CONFIG_VAL_NOT_FOUND) + if value is CONFIG_VAL_NOT_FOUND and hasattr(self.__app_config, "config_dict"): + value = self.__app_config.config_dict.get(global_key, CONFIG_VAL_NOT_FOUND) + if value is CONFIG_VAL_NOT_FOUND: + value = default + + return value + def dependency_shell_commands( self, requirements, **kwds ): requirement_to_dependency = self.requirements_to_dependencies(requirements, **kwds) return [dependency.shell_commands(requirement) for requirement, dependency in requirement_to_dependency.items()] @@ -221,8 +220,8 @@ class DependencyManager( object ): class CachedDependencyManager(DependencyManager): - def __init__(self, default_base_path, conf_file=None, **extra_config): - super(CachedDependencyManager, self).__init__(default_base_path=default_base_path, conf_file=conf_file, **extra_config) + def __init__(self, default_base_path, conf_file=None, app_config={}): + super(CachedDependencyManager, self).__init__(default_base_path=default_base_path, conf_file=conf_file, app_config=app_config) def build_cache(self, requirements, **kwds): resolved_dependencies = self.requirements_to_dependencies(requirements, **kwds) @@ -251,7 +250,7 @@ class CachedDependencyManager(DependencyManager): resolved_dependencies = self.requirements_to_dependencies(requirements, **kwds) cacheable_dependencies = [dep for dep in resolved_dependencies.values() if dep.cacheable] hashed_dependencies_dir = self.get_hashed_dependencies_path(cacheable_dependencies) - if not os.path.exists(hashed_dependencies_dir) and self.extra_config['precache_dependencies']: + if not os.path.exists(hashed_dependencies_dir) and self.__app_config.getattr("precache_dependencies", False): # Cache not present, try to create it self.build_cache(requirements, **kwds) if os.path.exists(hashed_dependencies_dir): diff --git a/lib/galaxy/tools/deps/resolvers/__init__.py b/lib/galaxy/tools/deps/resolvers/__init__.py index bebf7539e8a..f4b4939e90d 100644 --- a/lib/galaxy/tools/deps/resolvers/__init__.py +++ b/lib/galaxy/tools/deps/resolvers/__init__.py @@ -22,6 +22,7 @@ class DependencyResolver(Dictifiable, object): # resolution. disabled = False resolves_simple_dependencies = True + config_options = {} __metaclass__ = ABCMeta @abstractmethod @@ -37,18 +38,6 @@ class DependencyResolver(Dictifiable, object): request version is 'default'.) """ - def _get_config_option(self, key, dependency_resolver, default=None, config_prefix=None, **kwds): - """ Look in resolver-specific settings for option and then fallback to - global settings. - """ - global_key = "%s_%s" % (config_prefix, key) - if key in kwds: - return kwds.get(key) - elif global_key in dependency_resolver.extra_config: - return dependency_resolver.extra_config.get(global_key) - else: - return default - class ListableDependencyResolver: """ Mix this into a ``DependencyResolver`` and implement to indicate diff --git a/lib/galaxy/tools/deps/resolvers/conda.py b/lib/galaxy/tools/deps/resolvers/conda.py index 992fbd879e9..bc56f10c2e4 100644 --- a/lib/galaxy/tools/deps/resolvers/conda.py +++ b/lib/galaxy/tools/deps/resolvers/conda.py @@ -44,6 +44,15 @@ log = logging.getLogger(__name__) class CondaDependencyResolver(DependencyResolver, ListableDependencyResolver, InstallableDependencyResolver, SpecificationPatternDependencyResolver): dict_collection_visible_keys = DependencyResolver.dict_collection_visible_keys + ['conda_prefix', 'versionless', 'ensure_channels', 'auto_install'] resolver_type = "conda" + config_options = { + 'prefix': None, + 'exec': None, + 'debug': None, + 'ensure_channels': DEFAULT_ENSURE_CHANNELS, + 'auto_install': False, + 'auto_init': False, + 'copy_dependencies': False, + } _specification_pattern = re.compile(r"https\:\/\/anaconda.org\/\w+\/\w+") def __init__(self, dependency_manager, **kwds): @@ -51,7 +60,7 @@ class CondaDependencyResolver(DependencyResolver, ListableDependencyResolver, In self.dependency_manager = dependency_manager def get_option(name): - return self._get_config_option(name, dependency_manager, config_prefix="conda", **kwds) + return dependency_manager.get_resolver_option(self, name, explicit_resolver_options=kwds) # Conda context options (these define the environment) conda_prefix = get_option("prefix") From 49a11073a49242fbd86e3ec05fb5029194ec23fd Mon Sep 17 00:00:00 2001 From: John Chilton Date: Tue, 17 Jan 2017 20:30:16 -0500 Subject: [PATCH 04/12] Give resolver's with a resolve_all method an interface. This follows the existing pattern within the module. --- lib/galaxy/tools/deps/resolvers/__init__.py | 9 +++++++++ lib/galaxy/tools/deps/resolvers/conda.py | 3 ++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/lib/galaxy/tools/deps/resolvers/__init__.py b/lib/galaxy/tools/deps/resolvers/__init__.py index f4b4939e90d..f2ca0611c6c 100644 --- a/lib/galaxy/tools/deps/resolvers/__init__.py +++ b/lib/galaxy/tools/deps/resolvers/__init__.py @@ -39,6 +39,15 @@ class DependencyResolver(Dictifiable, object): """ +class MultipleDependencyResolver: + """Variant of DependencyResolver that can optionally resolve multiple dependencies together.""" + + @abstractmethod + def resolve_all( self, requirements, **kwds ): + """Given multiple requirements yield Dependency objects if and only if they may all be resolved together. + """ + + class ListableDependencyResolver: """ Mix this into a ``DependencyResolver`` and implement to indicate the dependency resolver can iterate over its dependencies and generate diff --git a/lib/galaxy/tools/deps/resolvers/conda.py b/lib/galaxy/tools/deps/resolvers/conda.py index bc56f10c2e4..d2083b09f23 100644 --- a/lib/galaxy/tools/deps/resolvers/conda.py +++ b/lib/galaxy/tools/deps/resolvers/conda.py @@ -29,6 +29,7 @@ from ..resolvers import ( DependencyResolver, InstallableDependencyResolver, ListableDependencyResolver, + MultipleDependencyResolver, NullDependency, SpecificationPatternDependencyResolver, ) @@ -41,7 +42,7 @@ DEFAULT_ENSURE_CHANNELS = "iuc,bioconda,r,defaults,conda-forge" log = logging.getLogger(__name__) -class CondaDependencyResolver(DependencyResolver, ListableDependencyResolver, InstallableDependencyResolver, SpecificationPatternDependencyResolver): +class CondaDependencyResolver(DependencyResolver, MultipleDependencyResolver, ListableDependencyResolver, InstallableDependencyResolver, SpecificationPatternDependencyResolver): dict_collection_visible_keys = DependencyResolver.dict_collection_visible_keys + ['conda_prefix', 'versionless', 'ensure_channels', 'auto_install'] resolver_type = "conda" config_options = { From 8e20e71556cfc90da548243d390deb52d4e9110d Mon Sep 17 00:00:00 2001 From: John Chilton Date: Tue, 17 Jan 2017 20:43:18 -0500 Subject: [PATCH 05/12] Change interface of depependency resolver's resolve for consistency. If resolve_all takes in requirements, then so should resolve. --- lib/galaxy/tools/deps/__init__.py | 12 +----------- lib/galaxy/tools/deps/requirements.py | 6 ++++++ lib/galaxy/tools/deps/resolvers/__init__.py | 18 +++++++++++++++++- .../resolvers/brewed_tool_shed_packages.py | 3 ++- lib/galaxy/tools/deps/resolvers/conda.py | 7 ++++++- .../tools/deps/resolvers/galaxy_packages.py | 4 +++- lib/galaxy/tools/deps/resolvers/homebrew.py | 4 +++- lib/galaxy/tools/deps/resolvers/modules.py | 4 +++- test/unit/tools/test_tool_deps.py | 6 +++--- 9 files changed, 44 insertions(+), 20 deletions(-) diff --git a/lib/galaxy/tools/deps/__init__.py b/lib/galaxy/tools/deps/__init__.py index 8182db3bba1..b03f33db229 100644 --- a/lib/galaxy/tools/deps/__init__.py +++ b/lib/galaxy/tools/deps/__init__.py @@ -155,17 +155,7 @@ class DependencyManager( object ): if requirement in requirement_to_dependency: continue - name = requirement.name - version = requirement.version - specs = requirement.specs - - if hasattr(resolver, "find_specification"): - spec = resolver.find_specification(specs) - if spec is not None: - name = spec.short_name - version = spec.version or version - - dependency = resolver.resolve( name, version, requirement.type, **kwds ) + dependency = resolver.resolve( requirement, **kwds ) if require_exact and not dependency.exact: continue diff --git a/lib/galaxy/tools/deps/requirements.py b/lib/galaxy/tools/deps/requirements.py index 92ba382e74b..bdb0ae0451a 100644 --- a/lib/galaxy/tools/deps/requirements.py +++ b/lib/galaxy/tools/deps/requirements.py @@ -1,9 +1,12 @@ +import copy + from galaxy.util import ( asbool, xml_text, ) from galaxy.util.oset import OrderedSet + DEFAULT_REQUIREMENT_TYPE = "package" DEFAULT_REQUIREMENT_VERSION = None @@ -24,6 +27,9 @@ class ToolRequirement( object ): specs = [s.to_dict() for s in self.specs] return dict(name=self.name, type=self.type, version=self.version, specs=specs) + def copy( self ): + return copy.deepcopy( self ) + @staticmethod def from_dict( dict ): version = dict.get( "version", None ) diff --git a/lib/galaxy/tools/deps/resolvers/__init__.py b/lib/galaxy/tools/deps/resolvers/__init__.py index f2ca0611c6c..a200878c141 100644 --- a/lib/galaxy/tools/deps/resolvers/__init__.py +++ b/lib/galaxy/tools/deps/resolvers/__init__.py @@ -26,7 +26,7 @@ class DependencyResolver(Dictifiable, object): __metaclass__ = ABCMeta @abstractmethod - def resolve( self, name, version, type, **kwds ): + def resolve( self, requirement, **kwds ): """Given inputs describing dependency in the abstract yield a Dependency object. The Dependency object describes various attributes (script, bin, @@ -92,6 +92,22 @@ class SpecificationPatternDependencyResolver: return spec return None + def resolve_specs(self, requirement): + name = requirement.name + version = requirement.version + specs = requirement.specs + + spec = self.find_specification(specs) + if spec is not None: + name = spec.short_name + version = spec.version or version + + requirement = requirement.copy() + requirement.name = name + requirement.version = version + + return requirement + class InstallableDependencyResolver: """ Mix this into a ``DependencyResolver`` and implement to indicate diff --git a/lib/galaxy/tools/deps/resolvers/brewed_tool_shed_packages.py b/lib/galaxy/tools/deps/resolvers/brewed_tool_shed_packages.py index 5585de56c42..a6b4cde4520 100644 --- a/lib/galaxy/tools/deps/resolvers/brewed_tool_shed_packages.py +++ b/lib/galaxy/tools/deps/resolvers/brewed_tool_shed_packages.py @@ -29,7 +29,8 @@ class HomebrewToolShedDependencyResolver( self._init_homebrew(**kwds) self._init_base_path(dependency_manager, **kwds) - def resolve(self, name, version, type, **kwds): + def resolve(self, requirement, **kwds): + name, version, type = requirement.name, requirement.version, requirement.type if type != "package": return NullDependency(version=version, name=name) if version is None: diff --git a/lib/galaxy/tools/deps/resolvers/conda.py b/lib/galaxy/tools/deps/resolvers/conda.py index d2083b09f23..0495a17b71e 100644 --- a/lib/galaxy/tools/deps/resolvers/conda.py +++ b/lib/galaxy/tools/deps/resolvers/conda.py @@ -146,6 +146,8 @@ class CondaDependencyResolver(DependencyResolver, MultipleDependencyResolver, Li conda_targets = [] for requirement in requirements: + requirement = self.resolve_specs(requirement) + version = requirement.version if self.versionless: version = None @@ -183,7 +185,10 @@ class CondaDependencyResolver(DependencyResolver, MultipleDependencyResolver, Li assert len(conda_targets) == 1 return conda_targets[0].install_environment - def resolve(self, name, version, type, **kwds): + def resolve(self, requirement, **kwds): + requirement = self.resolve_specs(requirement) + name, version, type = requirement.name, requirement.version, requirement.type + # Check for conda just not being there, this way we can enable # conda by default and just do nothing in not configured. if not os.path.isdir(self.conda_context.conda_prefix): diff --git a/lib/galaxy/tools/deps/resolvers/galaxy_packages.py b/lib/galaxy/tools/deps/resolvers/galaxy_packages.py index 858ff0faf98..bbfb3e40168 100644 --- a/lib/galaxy/tools/deps/resolvers/galaxy_packages.py +++ b/lib/galaxy/tools/deps/resolvers/galaxy_packages.py @@ -66,11 +66,13 @@ class BaseGalaxyPackageDependencyResolver(DependencyResolver, UsesToolDependency self.versionless = str(kwds.get('versionless', "false")).lower() == "true" self._init_base_path( dependency_manager, **kwds ) - def resolve( self, name, version, type, **kwds ): + def resolve(self, requirement, **kwds): """ Attempt to find a dependency named `name` at version `version`. If version is None, return the "default" version as determined using a symbolic link (if found). Returns a triple of: env_script, base_path, real_version """ + name, version, type = requirement.name, requirement.version, requirement.type + if version is None or self.versionless: exact = not self.versionless or version is None return self._find_dep_default( name, type=type, exact=exact, **kwds ) diff --git a/lib/galaxy/tools/deps/resolvers/homebrew.py b/lib/galaxy/tools/deps/resolvers/homebrew.py index 395eb1eb518..2a9781d3904 100644 --- a/lib/galaxy/tools/deps/resolvers/homebrew.py +++ b/lib/galaxy/tools/deps/resolvers/homebrew.py @@ -39,7 +39,9 @@ class HomebrewDependencyResolver(DependencyResolver, UsesHomebrewMixin): self._init_homebrew(**kwds) - def resolve(self, name, version, type, **kwds): + def resolve(self, requirement, **kwds): + name, version, type = requirement.name, requirement.version, requirement.type + if type != "package": return NullDependency(version=version, name=name) diff --git a/lib/galaxy/tools/deps/resolvers/modules.py b/lib/galaxy/tools/deps/resolvers/modules.py index 227abb72a9c..cb40ed9f4bc 100644 --- a/lib/galaxy/tools/deps/resolvers/modules.py +++ b/lib/galaxy/tools/deps/resolvers/modules.py @@ -51,7 +51,9 @@ class ModuleDependencyResolver(DependencyResolver): module_path = DEFAULT_MODULE_PATH return module_path - def resolve( self, name, version, type, **kwds ): + def resolve(self, requirement, **kwds): + name, version, type = requirement.name, requirement.version, requirement.type + if type != "package": return NullDependency(version=version, name=name) diff --git a/test/unit/tools/test_tool_deps.py b/test/unit/tools/test_tool_deps.py index 2f4bda30fcb..d328162351c 100644 --- a/test/unit/tools/test_tool_deps.py +++ b/test/unit/tools/test_tool_deps.py @@ -169,15 +169,15 @@ advisor/2013/update2 intel/11.1.080 mkl/10.2.5.035 advisor/2013/update3 intel/12.0 mkl/10.2.7.041 ''') resolver = ModuleDependencyResolver(None, modulecmd=module_script) - module = resolver.resolve( name="R", version=None, type="package" ) + module = resolver.resolve( ToolRequirement( name="R", version=None, type="package" ) ) assert module.module_name == "R" assert module.module_version is None - module = resolver.resolve( name="R", version="3.0.1", type="package" ) + module = resolver.resolve( ToolRequirement( name="R", version="3.0.1", type="package" ) ) assert module.module_name == "R" assert module.module_version == "3.0.1" - module = resolver.resolve( name="R", version="3.0.4", type="package" ) + module = resolver.resolve( ToolRequirement( name="R", version="3.0.4", type="package" ) ) assert isinstance(module, NullDependency) From 35b2b735ff7495576de88b7469c7118f65cb2838 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Wed, 18 Jan 2017 10:44:31 -0500 Subject: [PATCH 06/12] Modify specification interface with better name. --- lib/galaxy/tools/deps/resolvers/__init__.py | 10 +++++----- lib/galaxy/tools/deps/resolvers/conda.py | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/lib/galaxy/tools/deps/resolvers/__init__.py b/lib/galaxy/tools/deps/resolvers/__init__.py index a200878c141..3c277abd8cd 100644 --- a/lib/galaxy/tools/deps/resolvers/__init__.py +++ b/lib/galaxy/tools/deps/resolvers/__init__.py @@ -74,8 +74,8 @@ class SpecificationAwareDependencyResolver: __metaclass__ = ABCMeta @abstractmethod - def find_specification(self, specs): - """Find closest matching specification for discovered resolver.""" + def _expand_specs(self, requirement): + """Find closest matching specification for discovered resolver and return new concrete requirement.""" class SpecificationPatternDependencyResolver: @@ -85,19 +85,19 @@ class SpecificationPatternDependencyResolver: def _specification_pattern(self): """Pattern of URI to match against.""" - def find_specification(self, specs): + def _find_specification(self, specs): pattern = self._specification_pattern for spec in specs: if pattern.match(spec.uri): return spec return None - def resolve_specs(self, requirement): + def _expand_specs(self, requirement): name = requirement.name version = requirement.version specs = requirement.specs - spec = self.find_specification(specs) + spec = self._find_specification(specs) if spec is not None: name = spec.short_name version = spec.version or version diff --git a/lib/galaxy/tools/deps/resolvers/conda.py b/lib/galaxy/tools/deps/resolvers/conda.py index 0495a17b71e..ff78d8a1476 100644 --- a/lib/galaxy/tools/deps/resolvers/conda.py +++ b/lib/galaxy/tools/deps/resolvers/conda.py @@ -146,7 +146,7 @@ class CondaDependencyResolver(DependencyResolver, MultipleDependencyResolver, Li conda_targets = [] for requirement in requirements: - requirement = self.resolve_specs(requirement) + requirement = self._expand_specs(requirement) version = requirement.version if self.versionless: @@ -186,7 +186,7 @@ class CondaDependencyResolver(DependencyResolver, MultipleDependencyResolver, Li return conda_targets[0].install_environment def resolve(self, requirement, **kwds): - requirement = self.resolve_specs(requirement) + requirement = self._expand_specs(requirement) name, version, type = requirement.name, requirement.version, requirement.type # Check for conda just not being there, this way we can enable From 2fad74ddcf63c03fc2d7fd77fa3838de637a1bf4 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Wed, 18 Jan 2017 09:59:11 -0500 Subject: [PATCH 07/12] Implement dependency mapping for environment modules, galaxy packages, and Conda. --- lib/galaxy/tools/deps/resolvers/__init__.py | 69 +++++++++++++++ lib/galaxy/tools/deps/resolvers/conda.py | 11 ++- .../tools/deps/resolvers/galaxy_packages.py | 11 ++- lib/galaxy/tools/deps/resolvers/modules.py | 11 ++- test/unit/tools/test_tool_deps.py | 84 +++++++++++++++++-- 5 files changed, 172 insertions(+), 14 deletions(-) diff --git a/lib/galaxy/tools/deps/resolvers/__init__.py b/lib/galaxy/tools/deps/resolvers/__init__.py index 3c277abd8cd..eb6fecbeaed 100644 --- a/lib/galaxy/tools/deps/resolvers/__init__.py +++ b/lib/galaxy/tools/deps/resolvers/__init__.py @@ -5,6 +5,9 @@ from abc import ( abstractproperty, ) +import yaml + +from galaxy.util import listify from galaxy.util.dictifiable import Dictifiable from ..requirements import ToolRequirement @@ -65,6 +68,72 @@ class ListableDependencyResolver: return ToolRequirement(name=name, type="package", version=version) +class MappableDependencyResolver: + """Mix this into a ``DependencyResolver`` to allow mapping files. + + Mapping files allow adapting generic requirements to specific local implementations. + """ + + def _setup_mapping(self, dependency_manager, **kwds): + mapping_files = dependency_manager.get_resolver_option(self, "mapping_files", explicit_resolver_options=kwds) + mappings = [] + if mapping_files: + mapping_files = listify(mapping_files) + for mapping_file in mapping_files: + mappings.extend(MappableDependencyResolver._mapping_file_to_list(mapping_file)) + self._mappings = mappings + + @staticmethod + def _mapping_file_to_list(mapping_file): + with open(mapping_file, "r") as f: + raw_mapping = yaml.load(f) or [] + return map(RequirementMapping.from_dict, raw_mapping) + + def _expand_mappings(self, requirement): + for mapping in self._mappings: + if requirement.name == mapping.from_name: + if mapping.from_version is not None and mapping.from_version != requirement.version: + continue + + requirement = requirement.copy() + requirement.name = mapping.to_name + if mapping.to_version is not None: + requirement.version = mapping.to_version + + break + + return requirement + + +class RequirementMapping(object): + + def __init__(self, from_name, from_version, to_name, to_version): + self.from_name = from_name + self.from_version = from_version + self.to_name = to_name + self.to_version = to_version + + @staticmethod + def from_dict(raw_mapping): + from_raw = raw_mapping.get("from") + if isinstance(from_raw, dict): + from_name = from_raw.get("name") + from_version = str(from_raw.get("version")) + else: + from_name = from_raw + from_version = None + + to_raw = raw_mapping.get("to") + if isinstance(to_raw, dict): + to_name = to_raw.get("name", from_name) + to_version = str(to_raw.get("version")) + else: + to_name = to_raw + to_version = None + + return RequirementMapping(from_name, from_version, to_name, to_version) + + class SpecificationAwareDependencyResolver: """Mix this into a :class:`DependencyResolver` to implement URI specification matching. diff --git a/lib/galaxy/tools/deps/resolvers/conda.py b/lib/galaxy/tools/deps/resolvers/conda.py index ff78d8a1476..aefb460261e 100644 --- a/lib/galaxy/tools/deps/resolvers/conda.py +++ b/lib/galaxy/tools/deps/resolvers/conda.py @@ -29,6 +29,7 @@ from ..resolvers import ( DependencyResolver, InstallableDependencyResolver, ListableDependencyResolver, + MappableDependencyResolver, MultipleDependencyResolver, NullDependency, SpecificationPatternDependencyResolver, @@ -42,7 +43,7 @@ DEFAULT_ENSURE_CHANNELS = "iuc,bioconda,r,defaults,conda-forge" log = logging.getLogger(__name__) -class CondaDependencyResolver(DependencyResolver, MultipleDependencyResolver, ListableDependencyResolver, InstallableDependencyResolver, SpecificationPatternDependencyResolver): +class CondaDependencyResolver(DependencyResolver, MultipleDependencyResolver, ListableDependencyResolver, InstallableDependencyResolver, SpecificationPatternDependencyResolver, MappableDependencyResolver): dict_collection_visible_keys = DependencyResolver.dict_collection_visible_keys + ['conda_prefix', 'versionless', 'ensure_channels', 'auto_install'] resolver_type = "conda" config_options = { @@ -57,6 +58,7 @@ class CondaDependencyResolver(DependencyResolver, MultipleDependencyResolver, Li _specification_pattern = re.compile(r"https\:\/\/anaconda.org\/\w+\/\w+") def __init__(self, dependency_manager, **kwds): + self._setup_mapping(dependency_manager, **kwds) self.versionless = _string_as_bool(kwds.get('versionless', 'false')) self.dependency_manager = dependency_manager @@ -146,7 +148,7 @@ class CondaDependencyResolver(DependencyResolver, MultipleDependencyResolver, Li conda_targets = [] for requirement in requirements: - requirement = self._expand_specs(requirement) + requirement = self._expand_requirement(requirement) version = requirement.version if self.versionless: @@ -186,7 +188,7 @@ class CondaDependencyResolver(DependencyResolver, MultipleDependencyResolver, Li return conda_targets[0].install_environment def resolve(self, requirement, **kwds): - requirement = self._expand_specs(requirement) + requirement = self._expand_requirement(requirement) name, version, type = requirement.name, requirement.version, requirement.type # Check for conda just not being there, this way we can enable @@ -236,6 +238,9 @@ class CondaDependencyResolver(DependencyResolver, MultipleDependencyResolver, Li preserve_python_environment=preserve_python_environment, ) + def _expand_requirement(self, requirement): + return self._expand_specs(self._expand_mappings(requirement)) + def list_dependencies(self): for install_target in installed_conda_targets(self.conda_context): name = install_target.package diff --git a/lib/galaxy/tools/deps/resolvers/galaxy_packages.py b/lib/galaxy/tools/deps/resolvers/galaxy_packages.py index bbfb3e40168..8e1a1145030 100644 --- a/lib/galaxy/tools/deps/resolvers/galaxy_packages.py +++ b/lib/galaxy/tools/deps/resolvers/galaxy_packages.py @@ -16,6 +16,7 @@ from ..resolvers import ( Dependency, DependencyResolver, ListableDependencyResolver, + MappableDependencyResolver, NullDependency, ) @@ -103,9 +104,17 @@ class BaseGalaxyPackageDependencyResolver(DependencyResolver, UsesToolDependency return NullDependency(version=version, name=name) -class GalaxyPackageDependencyResolver(BaseGalaxyPackageDependencyResolver, ListableDependencyResolver): +class GalaxyPackageDependencyResolver(BaseGalaxyPackageDependencyResolver, ListableDependencyResolver, MappableDependencyResolver): resolver_type = "galaxy_packages" + def __init__(self, dependency_manager, **kwds): + super(GalaxyPackageDependencyResolver, self).__init__(dependency_manager, **kwds) + self._setup_mapping(dependency_manager, **kwds) + + def resolve(self, requirement, **kwds): + requirement = self._expand_mappings(requirement) + return super(GalaxyPackageDependencyResolver, self).resolve(requirement, **kwds) + def list_dependencies(self): base_path = self.base_path for package_name in listdir(base_path): diff --git a/lib/galaxy/tools/deps/resolvers/modules.py b/lib/galaxy/tools/deps/resolvers/modules.py index cb40ed9f4bc..0cbbc5764f8 100644 --- a/lib/galaxy/tools/deps/resolvers/modules.py +++ b/lib/galaxy/tools/deps/resolvers/modules.py @@ -13,7 +13,12 @@ from subprocess import PIPE, Popen from six import StringIO -from ..resolvers import Dependency, DependencyResolver, NullDependency +from ..resolvers import ( + Dependency, + DependencyResolver, + MappableDependencyResolver, + NullDependency, +) log = logging.getLogger( __name__ ) @@ -24,11 +29,12 @@ DEFAULT_MODULE_PREFETCH = "true" UNKNOWN_FIND_BY_MESSAGE = "ModuleDependencyResolver does not know how to find modules by [%s], find_by should be one of %s" -class ModuleDependencyResolver(DependencyResolver): +class ModuleDependencyResolver(DependencyResolver, MappableDependencyResolver): dict_collection_visible_keys = DependencyResolver.dict_collection_visible_keys + ['base_path', 'modulepath'] resolver_type = "modules" def __init__(self, dependency_manager, **kwds): + self._setup_mapping(dependency_manager, **kwds) self.versionless = _string_as_bool(kwds.get('versionless', 'false')) find_by = kwds.get('find_by', 'avail') prefetch = _string_as_bool(kwds.get('prefetch', DEFAULT_MODULE_PREFETCH)) @@ -52,6 +58,7 @@ class ModuleDependencyResolver(DependencyResolver): return module_path def resolve(self, requirement, **kwds): + requirement = self._expand_mappings(requirement) name, version, type = requirement.name, requirement.version, requirement.type if type != "package": diff --git a/test/unit/tools/test_tool_deps.py b/test/unit/tools/test_tool_deps.py index d328162351c..ff60859d71f 100644 --- a/test/unit/tools/test_tool_deps.py +++ b/test/unit/tools/test_tool_deps.py @@ -142,13 +142,7 @@ def test_tool_requirements(): def test_module_dependency_resolver(): with __test_base_path() as temp_directory: - module_script = os.path.join(temp_directory, "modulecmd") - __write_script(module_script, '''#!/bin/sh -cat %s/example_output 1>&2; -''' % temp_directory) - with open(os.path.join(temp_directory, "example_output"), "w") as f: - # Subset of module avail from MSI cluster. - f.write(''' + module_script = _setup_module_command(temp_directory, ''' -------------------------- /soft/modules/modulefiles --------------------------- JAGS/3.2.0-gcc45 JAGS/3.3.0-gcc4.7.2 @@ -168,7 +162,7 @@ advisor/2013/update1 intel/11.1.075 mkl/10.2.1.017 advisor/2013/update2 intel/11.1.080 mkl/10.2.5.035 advisor/2013/update3 intel/12.0 mkl/10.2.7.041 ''') - resolver = ModuleDependencyResolver(None, modulecmd=module_script) + resolver = ModuleDependencyResolver(_SimpleDependencyManager(), modulecmd=module_script) module = resolver.resolve( ToolRequirement( name="R", version=None, type="package" ) ) assert module.module_name == "R" assert module.module_version is None @@ -181,6 +175,74 @@ advisor/2013/update3 intel/12.0 mkl/10.2.7.041 assert isinstance(module, NullDependency) +def test_module_resolver_with_mapping(): + with __test_base_path() as temp_directory: + module_script = _setup_module_command(temp_directory, ''' +-------------------------- /soft/modules/modulefiles --------------------------- +blast/2.24 +''') + mapping_file = os.path.join(temp_directory, "mapping") + with open(mapping_file, "w") as f: + f.write(''' +- from: blast+ + to: blast +''') + + resolver = ModuleDependencyResolver(_SimpleDependencyManager(), modulecmd=module_script, mapping_files=mapping_file) + module = resolver.resolve( ToolRequirement( name="blast+", version="2.24", type="package" ) ) + assert module.module_name == "blast" + assert module.module_version == "2.24", module.module_version + + +def test_module_resolver_with_mapping_versions(): + with __test_base_path() as temp_directory: + module_script = _setup_module_command(temp_directory, ''' +-------------------------- /soft/modules/modulefiles --------------------------- +blast/2.22.0-mpi +blast/2.23 +blast/2.24.0-mpi +''') + mapping_file = os.path.join(temp_directory, "mapping") + with open(mapping_file, "w") as f: + f.write(''' +- from: + name: blast+ + version: 2.24 + to: + name: blast + version: 2.24.0-mpi +- from: + name: blast + version: 2.22 + to: + version: 2.22.0-mpi +''') + + resolver = ModuleDependencyResolver(_SimpleDependencyManager(), modulecmd=module_script, mapping_files=mapping_file) + module = resolver.resolve( ToolRequirement( name="blast+", version="2.24", type="package" ) ) + assert module.module_name == "blast" + assert module.module_version == "2.24.0-mpi", module.module_version + + resolver = ModuleDependencyResolver(_SimpleDependencyManager(), modulecmd=module_script, mapping_files=mapping_file) + module = resolver.resolve( ToolRequirement( name="blast+", version="2.23", type="package" ) ) + assert isinstance(module, NullDependency) + + module = resolver.resolve( ToolRequirement( name="blast", version="2.22", type="package" ) ) + assert module.module_name == "blast" + assert module.module_version == "2.22.0-mpi", module.module_version + + +def _setup_module_command(temp_directory, contents): + module_script = os.path.join(temp_directory, "modulecmd") + __write_script(module_script, '''#!/bin/sh +cat %s/example_output 1>&2; +''' % temp_directory) + with open(os.path.join(temp_directory, "example_output"), "w") as f: + # Subset of module avail from MSI cluster. + f.write(contents) + return module_script + + def test_module_dependency(): with __test_base_path() as temp_directory: # Create mock modulecmd script that just exports a variable @@ -413,3 +475,9 @@ def __dependency_manager(xml_content): f.flush() dm = DependencyManager( default_base_path=base_path, conf_file=f.name ) yield dm + + +class _SimpleDependencyManager(object): + + def get_resolver_option(self, resolver, key, explicit_resolver_options={}): + return explicit_resolver_options.get(key) From 3d9cdeb70c0170b7795c35e458c878ebe19caf1a Mon Sep 17 00:00:00 2001 From: John Chilton Date: Mon, 16 Jan 2017 08:23:13 -0500 Subject: [PATCH 08/12] Outline infrastructure for managing Conda installations outside of Galaxy. - Construct a DependencyManager according to Galaxy's configuration before Galaxy starts to allow auto installation of Conda. - Add script for managing tool dependencies outside of a Galaxy environment. - Refactor to allow this to be done without duplicating logic related to defaults, configuration handling, etc.... Implements #3426. --- lib/galaxy/config.py | 176 +++++++++++++++++----------- lib/galaxy/model/orm/scripts.py | 24 ++-- lib/galaxy/util/properties.py | 24 +++- run.sh | 10 ++ scripts/manage_tool_dependencies.py | 88 ++++++++++++++ 5 files changed, 234 insertions(+), 88 deletions(-) create mode 100644 scripts/manage_tool_dependencies.py diff --git a/lib/galaxy/config.py b/lib/galaxy/config.py index b7c30417c6c..820af066e3c 100644 --- a/lib/galaxy/config.py +++ b/lib/galaxy/config.py @@ -32,6 +32,44 @@ from .version import VERSION_MAJOR log = logging.getLogger( __name__ ) +PATH_DEFAULTS = dict( + auth_config_file=['config/auth_conf.xml', 'config/auth_conf.xml.sample'], + data_manager_config_file=['config/data_manager_conf.xml', 'data_manager_conf.xml', 'config/data_manager_conf.xml.sample'], + datatypes_config_file=['config/datatypes_conf.xml', 'datatypes_conf.xml', 'config/datatypes_conf.xml.sample'], + external_service_type_config_file=['config/external_service_types_conf.xml', 'external_service_types_conf.xml', 'config/external_service_types_conf.xml.sample'], + job_config_file=['config/job_conf.xml', 'job_conf.xml'], + tool_destinations_config_file=['config/tool_destinations.yml', 'config/tool_destinations.yml.sample'], + job_metrics_config_file=['config/job_metrics_conf.xml', 'job_metrics_conf.xml', 'config/job_metrics_conf.xml.sample'], + dependency_resolvers_config_file=['config/dependency_resolvers_conf.xml', 'dependency_resolvers_conf.xml'], + job_resource_params_file=['config/job_resource_params_conf.xml', 'job_resource_params_conf.xml'], + migrated_tools_config=['migrated_tools_conf.xml', 'config/migrated_tools_conf.xml'], + object_store_config_file=['config/object_store_conf.xml', 'object_store_conf.xml'], + openid_config_file=['config/openid_conf.xml', 'openid_conf.xml', 'config/openid_conf.xml.sample'], + shed_data_manager_config_file=['shed_data_manager_conf.xml', 'config/shed_data_manager_conf.xml'], + shed_tool_data_table_config=['shed_tool_data_table_conf.xml', 'config/shed_tool_data_table_conf.xml'], + tool_sheds_config_file=['config/tool_sheds_conf.xml', 'tool_sheds_conf.xml', 'config/tool_sheds_conf.xml.sample'], + workflow_schedulers_config_file=['config/workflow_schedulers_conf.xml', 'config/workflow_schedulers_conf.xml.sample'], +) + +PATH_LIST_DEFAULTS = dict( + tool_data_table_config_path=['config/tool_data_table_conf.xml', 'tool_data_table_conf.xml', 'config/tool_data_table_conf.xml.sample'], + # rationale: + # [0]: user has explicitly created config/tool_conf.xml but did not + # move their existing shed_tool_conf.xml, don't use + # config/shed_tool_conf.xml, which is probably the empty + # version copied from the sample, or else their shed tools + # will disappear + # [1]: user has created config/tool_conf.xml and, having passed + # [0], probably moved their shed_tool_conf.xml as well + # [2]: user has done nothing, use the old files + # [3]: fresh install + tool_config_file=['config/tool_conf.xml,shed_tool_conf.xml', + 'config/tool_conf.xml,config/shed_tool_conf.xml', + 'tool_conf.xml,shed_tool_conf.xml', + 'config/tool_conf.xml.sample,config/shed_tool_conf.xml'] +) + + def resolve_path( path, root ): """If 'path' is relative make absolute by prepending 'root'""" if not os.path.isabs( path ): @@ -39,12 +77,34 @@ def resolve_path( path, root ): return path +def find_path(kwargs, var, root): + """Find a configuration path that may exist at different defaults.""" + defaults = PATH_DEFAULTS[var] + + if kwargs.get(var, None) is not None: + path = kwargs.get(var) + else: + for default in defaults: + if os.path.exists(resolve_path(default, root)): + path = default + break + else: + path = defaults[-1] + + return resolve_path(path, root) + + +def find_root(kwargs): + root = kwargs.get('root_dir', '.') + return root + + class Configuration( object ): deprecated_options = ( 'database_file', ) def __init__( self, **kwargs ): self.config_dict = kwargs - self.root = kwargs.get( 'root_dir', '.' ) + self.root = find_root(kwargs) # Resolve paths of other config files self.__parse_config_file_options( kwargs ) @@ -326,23 +386,13 @@ class Configuration( object ): self.tool_help_boost = kwargs.get( "tool_help_boost", 0.5 ) self.tool_search_limit = kwargs.get( "tool_search_limit", 20 ) # Location for tool dependencies. - # Location for tool dependencies. - tool_dependency_dir = kwargs.get( "tool_dependency_dir", "database/dependencies" ) - if tool_dependency_dir.lower() == "none": - tool_dependency_dir = None - - if tool_dependency_dir is not None: - self.tool_dependency_dir = resolve_path( tool_dependency_dir, self.root ) - # Setting the following flag to true will ultimately cause tool dependencies - # to be located in the shell environment and used by the job that is executing - # the tool. - self.use_tool_dependencies = True - else: - self.tool_dependency_dir = None - self.use_tool_dependencies = os.path.exists(self.dependency_resolvers_config_file) - self.use_cached_dependency_manager = string_as_bool(kwargs.get("use_cached_dependency_manager", 'False')) - self.tool_dependency_cache_dir = kwargs.get( 'tool_dependency_cache_dir', os.path.join(self.tool_dependency_dir, '_cache')) - self.precache_dependencies = string_as_bool(kwargs.get("precache_dependencies", 'True')) + use_tool_dependencies, tool_dependency_dir, use_cached_dependency_manager, tool_dependency_cache_dir, precache_dependencies = \ + parse_dependency_options(kwargs, self.root, self.dependency_resolvers_config_file) + self.use_tool_dependencies = use_tool_dependencies + self.tool_dependency_dir = tool_dependency_dir + self.use_cached_dependency_manager = use_cached_dependency_manager + self.tool_dependency_cache_dir = tool_dependency_cache_dir + self.precache_dependencies = precache_dependencies self.enable_beta_mulled_containers = string_as_bool( kwargs.get( 'enable_beta_mulled_containers', 'False' ) ) containers_resolvers_config_file = kwargs.get( 'containers_resolvers_config_file', None ) @@ -555,56 +605,11 @@ class Configuration( object ): """ Backwards compatibility for config files moved to the config/ dir. """ - defaults = dict( - auth_config_file=[ 'config/auth_conf.xml', 'config/auth_conf.xml.sample' ], - data_manager_config_file=[ 'config/data_manager_conf.xml', 'data_manager_conf.xml', 'config/data_manager_conf.xml.sample' ], - datatypes_config_file=[ 'config/datatypes_conf.xml', 'datatypes_conf.xml', 'config/datatypes_conf.xml.sample' ], - external_service_type_config_file=[ 'config/external_service_types_conf.xml', 'external_service_types_conf.xml', 'config/external_service_types_conf.xml.sample' ], - job_config_file=[ 'config/job_conf.xml', 'job_conf.xml' ], - tool_destinations_config_file=[ 'config/tool_destinations.yml', 'config/tool_destinations.yml.sample' ], - job_metrics_config_file=[ 'config/job_metrics_conf.xml', 'job_metrics_conf.xml', 'config/job_metrics_conf.xml.sample' ], - dependency_resolvers_config_file=[ 'config/dependency_resolvers_conf.xml', 'dependency_resolvers_conf.xml' ], - job_resource_params_file=[ 'config/job_resource_params_conf.xml', 'job_resource_params_conf.xml' ], - migrated_tools_config=[ 'migrated_tools_conf.xml', 'config/migrated_tools_conf.xml' ], - object_store_config_file=[ 'config/object_store_conf.xml', 'object_store_conf.xml' ], - openid_config_file=[ 'config/openid_conf.xml', 'openid_conf.xml', 'config/openid_conf.xml.sample' ], - shed_data_manager_config_file=[ 'shed_data_manager_conf.xml', 'config/shed_data_manager_conf.xml' ], - shed_tool_data_table_config=[ 'shed_tool_data_table_conf.xml', 'config/shed_tool_data_table_conf.xml' ], - tool_sheds_config_file=[ 'config/tool_sheds_conf.xml', 'tool_sheds_conf.xml', 'config/tool_sheds_conf.xml.sample' ], - workflow_schedulers_config_file=['config/workflow_schedulers_conf.xml', 'config/workflow_schedulers_conf.xml.sample'], - ) - listify_defaults = dict( - tool_data_table_config_path=[ 'config/tool_data_table_conf.xml', 'tool_data_table_conf.xml', 'config/tool_data_table_conf.xml.sample' ], - # rationale: - # [0]: user has explicitly created config/tool_conf.xml but did not - # move their existing shed_tool_conf.xml, don't use - # config/shed_tool_conf.xml, which is probably the empty - # version copied from the sample, or else their shed tools - # will disappear - # [1]: user has created config/tool_conf.xml and, having passed - # [0], probably moved their shed_tool_conf.xml as well - # [2]: user has done nothing, use the old files - # [3]: fresh install - tool_config_file=[ 'config/tool_conf.xml,shed_tool_conf.xml', - 'config/tool_conf.xml,config/shed_tool_conf.xml', - 'tool_conf.xml,shed_tool_conf.xml', - 'config/tool_conf.xml.sample,config/shed_tool_conf.xml' ] - ) + for var in PATH_DEFAULTS: + setattr( self, var, find_path( kwargs, var, self.root ) ) - for var, defaults in defaults.items(): - if kwargs.get( var, None ) is not None: - path = kwargs.get( var ) - else: - for default in defaults: - if os.path.exists( resolve_path( default, self.root ) ): - path = default - break - else: - path = defaults[-1] - setattr( self, var, resolve_path( path, self.root ) ) - - for var, defaults in listify_defaults.items(): + for var, defaults in PATH_LIST_DEFAULTS.items(): paths = [] if kwargs.get( var, None ) is not None: paths = listify( kwargs.get( var ) ) @@ -755,6 +760,31 @@ class Configuration( object ): return [ parse( v ) for v in allowed_origin_hostnames if v ] +def parse_dependency_options(kwargs, root, dependency_resolvers_config_file): + # Location for tool dependencies. + tool_dependency_dir = kwargs.get("tool_dependency_dir", "database/dependencies") + if tool_dependency_dir.lower() == "none": + tool_dependency_dir = None + + if tool_dependency_dir is not None: + tool_dependency_dir = resolve_path(tool_dependency_dir, root) + # Setting the following flag to true will ultimately cause tool dependencies + # to be located in the shell environment and used by the job that is executing + # the tool. + use_tool_dependencies = True + tool_dependency_cache_dir = kwargs.get('tool_dependency_cache_dir', os.path.join(tool_dependency_dir, '_cache')) + use_cached_dependency_manager = string_as_bool(kwargs.get("use_cached_dependency_manager", 'False')) + precache_dependencies = string_as_bool(kwargs.get("precache_dependencies", 'True')) + else: + tool_dependency_dir = None + use_tool_dependencies = os.path.exists(dependency_resolvers_config_file) + tool_dependency_cache_dir = None + precache_dependencies = False + use_cached_dependency_manager = False + + return use_tool_dependencies, tool_dependency_dir, use_cached_dependency_manager, tool_dependency_cache_dir, precache_dependencies + + def get_database_engine_options( kwargs, model_prefix='' ): """ Allow options for the SQLAlchemy database engine to be passed by using @@ -784,15 +814,21 @@ def get_database_engine_options( kwargs, model_prefix='' ): def configure_logging( config ): - """ - Allow some basic logging configuration to be read from ini file. + """Allow some basic logging configuration to be read from ini file. + + This should be able to consume either a galaxy.config.Configuration object + or a simple dictionary of configuration variables. """ # Get root logger root = logging.getLogger() # PasteScript will have already configured the logger if the # 'loggers' section was found in the config file, otherwise we do # some simple setup using the 'log_*' values from the config. - paste_configures_logging = config.global_conf_parser.has_section( "loggers" ) + parser = getattr(config, "global_conf_parser", None) + if parser: + paste_configures_logging = config.global_conf_parser.has_section( "loggers" ) + else: + paste_configures_logging = False auto_configure_logging = not paste_configures_logging and string_as_bool( config.get( "auto_configure_logging", "True" ) ) if auto_configure_logging: format = config.get( "log_format", "%(name)s %(levelname)s %(asctime)s %(message)s" ) @@ -823,7 +859,7 @@ def configure_logging( config ): handler.setFormatter( formatter ) root.addHandler( handler ) # If sentry is configured, also log to it - if config.sentry_dsn: + if getattr(config, "sentry_dsn", None): from raven.handlers.logging import SentryHandler sentry_handler = SentryHandler( config.sentry_dsn ) sentry_handler.setLevel( logging.WARN ) diff --git a/lib/galaxy/model/orm/scripts.py b/lib/galaxy/model/orm/scripts.py index d7fce87ecb2..a7a922341cd 100644 --- a/lib/galaxy/model/orm/scripts.py +++ b/lib/galaxy/model/orm/scripts.py @@ -2,9 +2,8 @@ Code to support database helper scripts (create_db.py, manage_db.py, etc...). """ import logging -import os.path -from galaxy.util.properties import load_app_properties +from galaxy.util.properties import find_config_file, load_app_properties log = logging.getLogger( __name__ ) @@ -40,25 +39,21 @@ DATABASE = { } -def read_config_file_arg( argv, default, old_default ): +def read_config_file_arg( argv, default, old_default, cwd=None ): + config_file = None if '-c' in argv: pos = argv.index( '-c' ) argv.pop(pos) config_file = argv.pop( pos ) - else: - if not os.path.exists( default ) and os.path.exists( old_default ): - config_file = old_default - elif os.path.exists( default ): - config_file = default - else: - config_file = default + ".sample" - return config_file + + return find_config_file( default, old_default, config_file, cwd=cwd ) def get_config( argv, cwd=None ): """ Read sys.argv and parse out repository of migrations and database url. + >>> import os >>> from ConfigParser import SafeConfigParser >>> from tempfile import mkdtemp >>> config_dir = mkdtemp() @@ -89,16 +84,11 @@ def get_config( argv, cwd=None ): default = database_defaults.get( 'config_file', DEFAULT_CONFIG_FILE ) old_default = database_defaults.get( 'old_config_file' ) - if cwd is not None: - default = os.path.join( cwd, default ) - old_default = os.path.join( cwd, old_default ) - config_file = read_config_file_arg( argv, default, old_default ) + config_file = read_config_file_arg( argv, default, old_default, cwd=cwd ) repo = database_defaults[ 'repo' ] config_prefix = database_defaults.get( 'config_prefix', DEFAULT_CONFIG_PREFIX ) config_override = database_defaults.get( 'config_override', 'GALAXY_CONFIG_' ) default_sqlite_file = database_defaults[ 'default_sqlite_file' ] - if cwd: - config_file = os.path.join( cwd, config_file ) properties = load_app_properties( ini_file=config_file, config_prefix=config_override ) diff --git a/lib/galaxy/util/properties.py b/lib/galaxy/util/properties.py index f2de11b3ae2..b5743dc0ef2 100644 --- a/lib/galaxy/util/properties.py +++ b/lib/galaxy/util/properties.py @@ -10,6 +10,28 @@ from six import iteritems from six.moves.configparser import ConfigParser +def find_config_file(default, old_default, explicit, cwd=None): + if cwd is not None: + default = os.path.join(cwd, default) + old_default = os.path.join(cwd, old_default) + if explicit is not None: + explicit = os.path.join(cwd, explicit) + + if explicit: + if os.path.exists(explicit): + config_file = explicit + else: + raise Exception("Problem determining Galaxy's configuration - the specified configuration file cannot be found.") + else: + if not os.path.exists( default ) and os.path.exists( old_default ): + config_file = old_default + elif os.path.exists( default ): + config_file = default + else: + config_file = default + ".sample" + return config_file + + def load_app_properties( kwds={}, ini_file=None, @@ -97,4 +119,4 @@ class NicerConfigParser(ConfigParser): raise -__all__ = ('load_app_properties', 'NicerConfigParser') +__all__ = ('find_config_file', 'load_app_properties', 'NicerConfigParser') diff --git a/run.sh b/run.sh index 039047a87b8..9d7dbdfb211 100755 --- a/run.sh +++ b/run.sh @@ -14,6 +14,7 @@ then . $GALAXY_LOCAL_ENV_FILE fi +INITIALIZE_TOOL_DEPENDENCIES=1 # Install Conda if needed. # Pop args meant for common_startup.sh while : do @@ -22,6 +23,10 @@ do common_startup_args="$common_startup_args $1" shift ;; + --skip-tool-dependency-initialization) + INITIALIZE_TOOL_DEPENDENCIES=0 + shift + ;; --skip-venv) skip_venv=1 common_startup_args="$common_startup_args $1" @@ -98,6 +103,11 @@ if [ -z "$GALAXY_CONFIG_FILE" ]; then export GALAXY_CONFIG_FILE fi +if [ $INITIALIZE_TOOL_DEPENDENCIES -eq 1 ]; then + # Install Conda environment if needed. + python ./scripts/manage_tool_dependencies.py -c "$GALAXY_CONFIG_FILE" init_if_needed +fi + if [ -n "$GALAXY_RUN_ALL" ]; then servers=$(sed -n 's/^\[server:\(.*\)\]/\1/ p' "$GALAXY_CONFIG_FILE" | xargs echo) if [ -z "$stop_daemon_arg_set" -a -z "$daemon_or_restart_arg_set" ]; then diff --git a/scripts/manage_tool_dependencies.py b/scripts/manage_tool_dependencies.py new file mode 100644 index 00000000000..71dd63f9118 --- /dev/null +++ b/scripts/manage_tool_dependencies.py @@ -0,0 +1,88 @@ +import argparse +import os.path +import sys + +sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, 'lib'))) + +from galaxy.util.properties import find_config_file, load_app_properties +from galaxy.config import ( + configure_logging, + find_path, + find_root, + parse_dependency_options, +) +from galaxy.tools.deps import CachedDependencyManager, EXTRA_CONFIG_KWDS, DependencyManager, NullDependencyManager + +DESCRIPTION = "Script to manage tool dependencies (with focus on a Conda environments)." + + +def main(argv=None): + """Entry point for conversion process.""" + if argv is None: + argv = sys.argv[1:] + args = _arg_parser().parse_args(argv) + action = args.action + action_func = ACTIONS[action] + action_func(args) + + +def _init_if_needed(args): + kwargs = _app_properties(args) + + # If conda_auto_init is set, simply building the Conda resolver will call handle installation. + _build_dependency_manager_no_config(kwargs) + + +def _app_properties(args): + config_file = find_config_file("config/galaxy.ini", "universe_wsgi.ini", args.config_file) + app_properties = load_app_properties(ini_file=config_file) + return app_properties + + +def _arg_parser(): + parser = argparse.ArgumentParser(description=DESCRIPTION) + parser.add_argument('action', metavar='ACTION', type=str, + choices=ACTIONS.keys(), + help='action to perform') + parser.add_argument("-c", "--config-file", default=None) + return parser + + +def _build_dependency_manager_no_config(kwargs): + """Simplified variant of build_dependency_manager from galaxy.tools.deps. + + The canonical factory method requires a full Galaxy configuration object + which we do not have available in this script (an optimization). + """ + configure_logging(kwargs) + root = find_root(kwargs) + dependency_resolvers_config_file = find_path(kwargs, "dependency_resolvers_config_file", root) + use_dependencies, tool_dependency_dir, use_cached_dependency_manager, tool_dependency_cache_dir, precache_dependencies = \ + parse_dependency_options(kwargs, root, dependency_resolvers_config_file) + + if not use_dependencies: + dependency_manager = NullDependencyManager() + else: + dependency_manager_kwds = { + 'default_base_path': tool_dependency_dir, + 'conf_file': dependency_resolvers_config_file, + } + for key, default_value in EXTRA_CONFIG_KWDS.items(): + dependency_manager_kwds[key] = kwargs.get(key, default_value) + + if use_cached_dependency_manager: + dependency_manager_kwds['tool_dependency_cache_dir'] = tool_dependency_cache_dir + dependency_manager = CachedDependencyManager(**dependency_manager_kwds) + else: + dependency_manager = DependencyManager( **dependency_manager_kwds ) + + return dependency_manager + + +ACTIONS = { + "init_if_needed": _init_if_needed, +} + + +if __name__ == '__main__': + main() From 312f941e46fdbbe6198b6b0e6a3f00867946a6bd Mon Sep 17 00:00:00 2001 From: John Chilton Date: Mon, 16 Jan 2017 08:26:23 -0500 Subject: [PATCH 09/12] Swap default for Conda "auto_init" to True. Everyone gets a Conda when Galaxy starts up. --- config/galaxy.ini.sample | 2 +- lib/galaxy/tools/deps/resolvers/conda.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/config/galaxy.ini.sample b/config/galaxy.ini.sample index d92b517788d..9ea4d1dbc96 100644 --- a/config/galaxy.ini.sample +++ b/config/galaxy.ini.sample @@ -210,7 +210,7 @@ paste.app_factory = galaxy.web.buildapp:app_factory #conda_auto_install = False # Set to True to instruct Galaxy to install Conda from the web automatically # if it cannot find a local copy and conda_exec is not configured. -#conda_auto_init = False +#conda_auto_init = True # You must set this to True if conda_prefix and job_working_directory are not on the same # volume, or some conda dependencies will fail to execute at job runtime. # Conda will copy packages content instead of creating hardlinks or symlinks. diff --git a/lib/galaxy/tools/deps/resolvers/conda.py b/lib/galaxy/tools/deps/resolvers/conda.py index aefb460261e..87a5ec14f53 100644 --- a/lib/galaxy/tools/deps/resolvers/conda.py +++ b/lib/galaxy/tools/deps/resolvers/conda.py @@ -52,7 +52,7 @@ class CondaDependencyResolver(DependencyResolver, MultipleDependencyResolver, Li 'debug': None, 'ensure_channels': DEFAULT_ENSURE_CHANNELS, 'auto_install': False, - 'auto_init': False, + 'auto_init': True, 'copy_dependencies': False, } _specification_pattern = re.compile(r"https\:\/\/anaconda.org\/\w+\/\w+") From 0b7c6761c8dbb2baf7391dc4a08c4dbed2ccc7f2 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Wed, 18 Jan 2017 14:16:07 -0500 Subject: [PATCH 10/12] Rework #3433 for changes in #3444. --- scripts/manage_tool_dependencies.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/scripts/manage_tool_dependencies.py b/scripts/manage_tool_dependencies.py index 71dd63f9118..07b5b8802ca 100644 --- a/scripts/manage_tool_dependencies.py +++ b/scripts/manage_tool_dependencies.py @@ -11,7 +11,7 @@ from galaxy.config import ( find_root, parse_dependency_options, ) -from galaxy.tools.deps import CachedDependencyManager, EXTRA_CONFIG_KWDS, DependencyManager, NullDependencyManager +from galaxy.tools.deps import CachedDependencyManager, DependencyManager, NullDependencyManager DESCRIPTION = "Script to manage tool dependencies (with focus on a Conda environments)." @@ -66,9 +66,8 @@ def _build_dependency_manager_no_config(kwargs): dependency_manager_kwds = { 'default_base_path': tool_dependency_dir, 'conf_file': dependency_resolvers_config_file, + 'app_config': kwargs, } - for key, default_value in EXTRA_CONFIG_KWDS.items(): - dependency_manager_kwds[key] = kwargs.get(key, default_value) if use_cached_dependency_manager: dependency_manager_kwds['tool_dependency_cache_dir'] = tool_dependency_cache_dir From 9baf6d6d8d8d089a8d2c2687ef498a396006b920 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Wed, 18 Jan 2017 14:14:05 -0500 Subject: [PATCH 11/12] Samples and defaults for Conda mapping. - Map R to r-base and blast+ to blast for conda dependencies. - Add sample environment modules file showing off the new mapping feature. - Add integration test case to verify the default conda mapping (R -> r-base in particular). --- config/local_conda_mapping.yml.sample | 2 ++ lib/galaxy/config.py | 11 +++++++++++ .../deps/resolvers/default_conda_mapping.yml | 5 +++++ test/functional/tools/legacy_r.xml | 14 ++++++++++++++ test/functional/tools/samples_tool_conf.xml | 2 ++ test/integration/test_resolvers.py | 17 +++++++++++++++++ 6 files changed, 51 insertions(+) create mode 100644 config/local_conda_mapping.yml.sample create mode 100644 lib/galaxy/tools/deps/resolvers/default_conda_mapping.yml create mode 100644 test/functional/tools/legacy_r.xml diff --git a/config/local_conda_mapping.yml.sample b/config/local_conda_mapping.yml.sample new file mode 100644 index 00000000000..383ce4836c6 --- /dev/null +++ b/config/local_conda_mapping.yml.sample @@ -0,0 +1,2 @@ +# See $GALAXY_ROOT/lib/galaxy/tools/deps/default_conda_mapping.yml for example mapping - +# additional site-specific mappings can be added to config/conda_mapping.yml. diff --git a/lib/galaxy/config.py b/lib/galaxy/config.py index 820af066e3c..9c53aa43d57 100644 --- a/lib/galaxy/config.py +++ b/lib/galaxy/config.py @@ -49,6 +49,8 @@ PATH_DEFAULTS = dict( shed_tool_data_table_config=['shed_tool_data_table_conf.xml', 'config/shed_tool_data_table_conf.xml'], tool_sheds_config_file=['config/tool_sheds_conf.xml', 'tool_sheds_conf.xml', 'config/tool_sheds_conf.xml.sample'], workflow_schedulers_config_file=['config/workflow_schedulers_conf.xml', 'config/workflow_schedulers_conf.xml.sample'], + modules_mapping_files=['config/environment_modules_mapping.yml', 'config/environment_modules_mapping.yml.sample'], + local_conda_mapping_file=['config/local_conda_mapping.yml', 'config/local_conda_mapping.yml.sample'], ) PATH_LIST_DEFAULTS = dict( @@ -393,6 +395,15 @@ class Configuration( object ): self.use_cached_dependency_manager = use_cached_dependency_manager self.tool_dependency_cache_dir = tool_dependency_cache_dir self.precache_dependencies = precache_dependencies + # Deployers may either specify a complete list of mapping files or get the default for free and just + # specify a local mapping file to adapt and extend the default one. + if "conda_mapping_files" in kwargs: + self.conda_mapping_files = kwargs["conda_mapping_files"] + else: + self.conda_mapping_files = [ + self.local_conda_mapping_file, + os.path.join(self.root, "lib", "galaxy", "tools", "deps", "resolvers", "default_conda_mapping.yml"), + ] self.enable_beta_mulled_containers = string_as_bool( kwargs.get( 'enable_beta_mulled_containers', 'False' ) ) containers_resolvers_config_file = kwargs.get( 'containers_resolvers_config_file', None ) diff --git a/lib/galaxy/tools/deps/resolvers/default_conda_mapping.yml b/lib/galaxy/tools/deps/resolvers/default_conda_mapping.yml new file mode 100644 index 00000000000..9b1e59e7a6b --- /dev/null +++ b/lib/galaxy/tools/deps/resolvers/default_conda_mapping.yml @@ -0,0 +1,5 @@ +- from: R + to: r-base +- from: blast+ + to: blast + \ No newline at end of file diff --git a/test/functional/tools/legacy_r.xml b/test/functional/tools/legacy_r.xml new file mode 100644 index 00000000000..5d70feede9f --- /dev/null +++ b/test/functional/tools/legacy_r.xml @@ -0,0 +1,14 @@ + + $out_file1 + ]]> + + R + + + + + + + + diff --git a/test/functional/tools/samples_tool_conf.xml b/test/functional/tools/samples_tool_conf.xml index d678c8be35d..02677a581bd 100644 --- a/test/functional/tools/samples_tool_conf.xml +++ b/test/functional/tools/samples_tool_conf.xml @@ -124,6 +124,8 @@ + + diff --git a/test/integration/test_resolvers.py b/test/integration/test_resolvers.py index 5c36fd30fbe..a676a299ca3 100644 --- a/test/integration/test_resolvers.py +++ b/test/integration/test_resolvers.py @@ -4,6 +4,9 @@ import shutil from tempfile import mkdtemp from base import integration_util +from base.populators import ( + DatasetPopulator, +) GNUPLOT = {u'version': u'4.6', u'type': u'package', u'name': u'gnuplot'} @@ -71,6 +74,20 @@ class CondaResolutionIntegrationTestCase(integration_util.IntegrationTestCase): response = create_response.json() self._assert_dependency_type(response) + def test_legacy_r_mapping( self ): + """ + """ + dataset_populator = DatasetPopulator(self.galaxy_interactor) + history_id = dataset_populator.new_history() + payload = dataset_populator.run_tool_payload( + tool_id="legacy_R", + inputs={}, + history_id=history_id, + ) + create_response = self._post( "tools", data=payload ) + self._assert_status_code_is( create_response, 200 ) + dataset_populator.wait_for_history( history_id, assert_ok=True ) + def test_dependency_status_installed_not_exact( self ): """ GET request to dependency_resolvers/dependency with GNUPLOT dependency. From 3e3d41fb94c00a9c6b068626b6f541e714b2500e Mon Sep 17 00:00:00 2001 From: John Chilton Date: Mon, 23 Jan 2017 09:46:30 -0500 Subject: [PATCH 12/12] Fix unit test test_tool_deps to not auto-install Conda. This became the default behavior with conda_auto_init defaulting to True. --- test/unit/tools/test_tool_deps.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/test/unit/tools/test_tool_deps.py b/test/unit/tools/test_tool_deps.py index ff60859d71f..1c18eedf29e 100644 --- a/test/unit/tools/test_tool_deps.py +++ b/test/unit/tools/test_tool_deps.py @@ -40,7 +40,7 @@ def test_tool_dependencies(): if sub == "env.sh": __touch( os.path.join( p, "env.sh" ) ) - dm = DependencyManager( default_base_path=base_path ) + dm = __dependency_manager_for_base_path( default_base_path=base_path ) dependency = dm.find_dep( "dep1", "1.0" ) assert dependency.script == os.path.join( base_path, 'dep1', '1.0', 'env.sh' ) assert dependency.path == os.path.join( base_path, 'dep1', '1.0' ) @@ -70,7 +70,7 @@ TEST_VERSION = "0.5.9" def test_toolshed_set_enviornment_requiremetns(): with __test_base_path() as base_path: test_repo = __build_test_repo('set_environment') - dm = DependencyManager( default_base_path=base_path ) + dm = __dependency_manager_for_base_path( default_base_path=base_path ) env_settings_dir = os.path.join(base_path, "environment_settings", TEST_REPO_NAME, TEST_REPO_USER, TEST_REPO_NAME, TEST_REPO_CHANGESET) os.makedirs(env_settings_dir) dependency = dm.find_dep( TEST_REPO_NAME, version=None, type='set_environment', installed_tool_dependencies=[test_repo] ) @@ -81,7 +81,7 @@ def test_toolshed_set_enviornment_requiremetns(): def test_toolshed_package_requirements(): with __test_base_path() as base_path: test_repo = __build_test_repo('package', version=TEST_VERSION) - dm = DependencyManager( default_base_path=base_path ) + dm = __dependency_manager_for_base_path( default_base_path=base_path ) package_dir = __build_ts_test_package(base_path) dependency = dm.find_dep( TEST_REPO_NAME, version=TEST_VERSION, type='package', installed_tool_dependencies=[test_repo] ) assert dependency.version == TEST_VERSION @@ -90,7 +90,7 @@ def test_toolshed_package_requirements(): def test_toolshed_tools_fallback_on_manual_dependencies(): with __test_base_path() as base_path: - dm = DependencyManager( default_base_path=base_path ) + dm = __dependency_manager_for_base_path( default_base_path=base_path ) test_repo = __build_test_repo('package', version=TEST_VERSION) env_path = __setup_galaxy_package_dep(base_path, "dep1", "1.0") dependency = dm.find_dep( "dep1", version="1.0", type='package', installed_tool_dependencies=[test_repo] ) @@ -100,7 +100,7 @@ def test_toolshed_tools_fallback_on_manual_dependencies(): def test_toolshed_greater_precendence(): with __test_base_path() as base_path: - dm = DependencyManager( default_base_path=base_path ) + dm = __dependency_manager_for_base_path( default_base_path=base_path ) test_repo = __build_test_repo('package', version=TEST_VERSION) ts_package_dir = __build_ts_test_package(base_path) gx_env_path = __setup_galaxy_package_dep(base_path, TEST_REPO_NAME, TEST_VERSION) @@ -281,7 +281,7 @@ def test_shell_commands_built(): # Test that dependency manager builds valid shell commands for a list of # requirements. with __test_base_path() as base_path: - dm = DependencyManager( default_base_path=base_path ) + dm = __dependency_manager_for_base_path( default_base_path=base_path ) __setup_galaxy_package_dep( base_path, TEST_REPO_NAME, TEST_VERSION, contents="export FOO=\"bar\"" ) mock_requirements = ToolRequirements([{'type': 'package', 'version': TEST_VERSION, 'name': TEST_REPO_NAME}]) commands = dm.dependency_shell_commands( mock_requirements ) @@ -473,10 +473,14 @@ def __dependency_manager(xml_content): f = tempfile.NamedTemporaryFile() f.write(xml_content) f.flush() - dm = DependencyManager( default_base_path=base_path, conf_file=f.name ) + dm = __dependency_manager_for_base_path( default_base_path=base_path, conf_file=f.name ) yield dm +def __dependency_manager_for_base_path(default_base_path, conf_file=None ): + return DependencyManager( default_base_path=default_base_path, conf_file=conf_file, app_config={"conda_auto_init": False} ) + + class _SimpleDependencyManager(object): def get_resolver_option(self, resolver, key, explicit_resolver_options={}):