diff --git a/config/galaxy.ini.sample b/config/galaxy.ini.sample
index 43dbf738559..ef07ff6d049 100644
--- a/config/galaxy.ini.sample
+++ b/config/galaxy.ini.sample
@@ -210,7 +210,7 @@ paste.app_factory = galaxy.web.buildapp:app_factory
#conda_auto_install = False
# Set to True to instruct Galaxy to install Conda from the web automatically
# if it cannot find a local copy and conda_exec is not configured.
-#conda_auto_init = False
+#conda_auto_init = True
# You must set this to True if conda_prefix and job_working_directory are not on the same
# volume, or some conda dependencies will fail to execute at job runtime.
# Conda will copy packages content instead of creating hardlinks or symlinks.
diff --git a/config/local_conda_mapping.yml.sample b/config/local_conda_mapping.yml.sample
new file mode 100644
index 00000000000..383ce4836c6
--- /dev/null
+++ b/config/local_conda_mapping.yml.sample
@@ -0,0 +1,2 @@
+# See $GALAXY_ROOT/lib/galaxy/tools/deps/default_conda_mapping.yml for example mapping -
+# additional site-specific mappings can be added to config/conda_mapping.yml.
diff --git a/lib/galaxy/config.py b/lib/galaxy/config.py
index 8b709f384f0..dea393a0bf8 100644
--- a/lib/galaxy/config.py
+++ b/lib/galaxy/config.py
@@ -32,6 +32,46 @@ from .version import VERSION_MAJOR
log = logging.getLogger( __name__ )
+PATH_DEFAULTS = dict(
+ auth_config_file=['config/auth_conf.xml', 'config/auth_conf.xml.sample'],
+ data_manager_config_file=['config/data_manager_conf.xml', 'data_manager_conf.xml', 'config/data_manager_conf.xml.sample'],
+ datatypes_config_file=['config/datatypes_conf.xml', 'datatypes_conf.xml', 'config/datatypes_conf.xml.sample'],
+ external_service_type_config_file=['config/external_service_types_conf.xml', 'external_service_types_conf.xml', 'config/external_service_types_conf.xml.sample'],
+ job_config_file=['config/job_conf.xml', 'job_conf.xml'],
+ tool_destinations_config_file=['config/tool_destinations.yml', 'config/tool_destinations.yml.sample'],
+ job_metrics_config_file=['config/job_metrics_conf.xml', 'job_metrics_conf.xml', 'config/job_metrics_conf.xml.sample'],
+ dependency_resolvers_config_file=['config/dependency_resolvers_conf.xml', 'dependency_resolvers_conf.xml'],
+ job_resource_params_file=['config/job_resource_params_conf.xml', 'job_resource_params_conf.xml'],
+ migrated_tools_config=['migrated_tools_conf.xml', 'config/migrated_tools_conf.xml'],
+ object_store_config_file=['config/object_store_conf.xml', 'object_store_conf.xml'],
+ openid_config_file=['config/openid_conf.xml', 'openid_conf.xml', 'config/openid_conf.xml.sample'],
+ shed_data_manager_config_file=['shed_data_manager_conf.xml', 'config/shed_data_manager_conf.xml'],
+ shed_tool_data_table_config=['shed_tool_data_table_conf.xml', 'config/shed_tool_data_table_conf.xml'],
+ tool_sheds_config_file=['config/tool_sheds_conf.xml', 'tool_sheds_conf.xml', 'config/tool_sheds_conf.xml.sample'],
+ workflow_schedulers_config_file=['config/workflow_schedulers_conf.xml', 'config/workflow_schedulers_conf.xml.sample'],
+ modules_mapping_files=['config/environment_modules_mapping.yml', 'config/environment_modules_mapping.yml.sample'],
+ local_conda_mapping_file=['config/local_conda_mapping.yml', 'config/local_conda_mapping.yml.sample'],
+)
+
+PATH_LIST_DEFAULTS = dict(
+ tool_data_table_config_path=['config/tool_data_table_conf.xml', 'tool_data_table_conf.xml', 'config/tool_data_table_conf.xml.sample'],
+ # rationale:
+ # [0]: user has explicitly created config/tool_conf.xml but did not
+ # move their existing shed_tool_conf.xml, don't use
+ # config/shed_tool_conf.xml, which is probably the empty
+ # version copied from the sample, or else their shed tools
+ # will disappear
+ # [1]: user has created config/tool_conf.xml and, having passed
+ # [0], probably moved their shed_tool_conf.xml as well
+ # [2]: user has done nothing, use the old files
+ # [3]: fresh install
+ tool_config_file=['config/tool_conf.xml,shed_tool_conf.xml',
+ 'config/tool_conf.xml,config/shed_tool_conf.xml',
+ 'tool_conf.xml,shed_tool_conf.xml',
+ 'config/tool_conf.xml.sample,config/shed_tool_conf.xml']
+)
+
+
def resolve_path( path, root ):
"""If 'path' is relative make absolute by prepending 'root'"""
if not os.path.isabs( path ):
@@ -39,12 +79,34 @@ def resolve_path( path, root ):
return path
+def find_path(kwargs, var, root):
+ """Find a configuration path that may exist at different defaults."""
+ defaults = PATH_DEFAULTS[var]
+
+ if kwargs.get(var, None) is not None:
+ path = kwargs.get(var)
+ else:
+ for default in defaults:
+ if os.path.exists(resolve_path(default, root)):
+ path = default
+ break
+ else:
+ path = defaults[-1]
+
+ return resolve_path(path, root)
+
+
+def find_root(kwargs):
+ root = kwargs.get('root_dir', '.')
+ return root
+
+
class Configuration( object ):
deprecated_options = ( 'database_file', )
def __init__( self, **kwargs ):
self.config_dict = kwargs
- self.root = kwargs.get( 'root_dir', '.' )
+ self.root = find_root(kwargs)
# Resolve paths of other config files
self.__parse_config_file_options( kwargs )
@@ -329,23 +391,22 @@ class Configuration( object ):
self.tool_ngram_minsize = kwargs.get( "tool_ngram_minsize", 3 )
self.tool_ngram_maxsize = kwargs.get( "tool_ngram_maxsize", 4 )
# Location for tool dependencies.
- # Location for tool dependencies.
- tool_dependency_dir = kwargs.get( "tool_dependency_dir", "database/dependencies" )
- if tool_dependency_dir.lower() == "none":
- tool_dependency_dir = None
-
- if tool_dependency_dir is not None:
- self.tool_dependency_dir = resolve_path( tool_dependency_dir, self.root )
- # Setting the following flag to true will ultimately cause tool dependencies
- # to be located in the shell environment and used by the job that is executing
- # the tool.
- self.use_tool_dependencies = True
+ use_tool_dependencies, tool_dependency_dir, use_cached_dependency_manager, tool_dependency_cache_dir, precache_dependencies = \
+ parse_dependency_options(kwargs, self.root, self.dependency_resolvers_config_file)
+ self.use_tool_dependencies = use_tool_dependencies
+ self.tool_dependency_dir = tool_dependency_dir
+ self.use_cached_dependency_manager = use_cached_dependency_manager
+ self.tool_dependency_cache_dir = tool_dependency_cache_dir
+ self.precache_dependencies = precache_dependencies
+ # Deployers may either specify a complete list of mapping files or get the default for free and just
+ # specify a local mapping file to adapt and extend the default one.
+ if "conda_mapping_files" in kwargs:
+ self.conda_mapping_files = kwargs["conda_mapping_files"]
else:
- self.tool_dependency_dir = None
- self.use_tool_dependencies = os.path.exists(self.dependency_resolvers_config_file)
- self.use_cached_dependency_manager = string_as_bool(kwargs.get("use_cached_dependency_manager", 'False'))
- self.tool_dependency_cache_dir = kwargs.get( 'tool_dependency_cache_dir', os.path.join(self.tool_dependency_dir, '_cache'))
- self.precache_dependencies = string_as_bool(kwargs.get("precache_dependencies", 'True'))
+ self.conda_mapping_files = [
+ self.local_conda_mapping_file,
+ os.path.join(self.root, "lib", "galaxy", "tools", "deps", "resolvers", "default_conda_mapping.yml"),
+ ]
self.enable_beta_mulled_containers = string_as_bool( kwargs.get( 'enable_beta_mulled_containers', 'False' ) )
containers_resolvers_config_file = kwargs.get( 'containers_resolvers_config_file', None )
@@ -558,56 +619,11 @@ class Configuration( object ):
"""
Backwards compatibility for config files moved to the config/ dir.
"""
- defaults = dict(
- auth_config_file=[ 'config/auth_conf.xml', 'config/auth_conf.xml.sample' ],
- data_manager_config_file=[ 'config/data_manager_conf.xml', 'data_manager_conf.xml', 'config/data_manager_conf.xml.sample' ],
- datatypes_config_file=[ 'config/datatypes_conf.xml', 'datatypes_conf.xml', 'config/datatypes_conf.xml.sample' ],
- external_service_type_config_file=[ 'config/external_service_types_conf.xml', 'external_service_types_conf.xml', 'config/external_service_types_conf.xml.sample' ],
- job_config_file=[ 'config/job_conf.xml', 'job_conf.xml' ],
- tool_destinations_config_file=[ 'config/tool_destinations.yml', 'config/tool_destinations.yml.sample' ],
- job_metrics_config_file=[ 'config/job_metrics_conf.xml', 'job_metrics_conf.xml', 'config/job_metrics_conf.xml.sample' ],
- dependency_resolvers_config_file=[ 'config/dependency_resolvers_conf.xml', 'dependency_resolvers_conf.xml' ],
- job_resource_params_file=[ 'config/job_resource_params_conf.xml', 'job_resource_params_conf.xml' ],
- migrated_tools_config=[ 'migrated_tools_conf.xml', 'config/migrated_tools_conf.xml' ],
- object_store_config_file=[ 'config/object_store_conf.xml', 'object_store_conf.xml' ],
- openid_config_file=[ 'config/openid_conf.xml', 'openid_conf.xml', 'config/openid_conf.xml.sample' ],
- shed_data_manager_config_file=[ 'shed_data_manager_conf.xml', 'config/shed_data_manager_conf.xml' ],
- shed_tool_data_table_config=[ 'shed_tool_data_table_conf.xml', 'config/shed_tool_data_table_conf.xml' ],
- tool_sheds_config_file=[ 'config/tool_sheds_conf.xml', 'tool_sheds_conf.xml', 'config/tool_sheds_conf.xml.sample' ],
- workflow_schedulers_config_file=['config/workflow_schedulers_conf.xml', 'config/workflow_schedulers_conf.xml.sample'],
- )
- listify_defaults = dict(
- tool_data_table_config_path=[ 'config/tool_data_table_conf.xml', 'tool_data_table_conf.xml', 'config/tool_data_table_conf.xml.sample' ],
- # rationale:
- # [0]: user has explicitly created config/tool_conf.xml but did not
- # move their existing shed_tool_conf.xml, don't use
- # config/shed_tool_conf.xml, which is probably the empty
- # version copied from the sample, or else their shed tools
- # will disappear
- # [1]: user has created config/tool_conf.xml and, having passed
- # [0], probably moved their shed_tool_conf.xml as well
- # [2]: user has done nothing, use the old files
- # [3]: fresh install
- tool_config_file=[ 'config/tool_conf.xml,shed_tool_conf.xml',
- 'config/tool_conf.xml,config/shed_tool_conf.xml',
- 'tool_conf.xml,shed_tool_conf.xml',
- 'config/tool_conf.xml.sample,config/shed_tool_conf.xml' ]
- )
+ for var in PATH_DEFAULTS:
+ setattr( self, var, find_path( kwargs, var, self.root ) )
- for var, defaults in defaults.items():
- if kwargs.get( var, None ) is not None:
- path = kwargs.get( var )
- else:
- for default in defaults:
- if os.path.exists( resolve_path( default, self.root ) ):
- path = default
- break
- else:
- path = defaults[-1]
- setattr( self, var, resolve_path( path, self.root ) )
-
- for var, defaults in listify_defaults.items():
+ for var, defaults in PATH_LIST_DEFAULTS.items():
paths = []
if kwargs.get( var, None ) is not None:
paths = listify( kwargs.get( var ) )
@@ -758,6 +774,31 @@ class Configuration( object ):
return [ parse( v ) for v in allowed_origin_hostnames if v ]
+def parse_dependency_options(kwargs, root, dependency_resolvers_config_file):
+ # Location for tool dependencies.
+ tool_dependency_dir = kwargs.get("tool_dependency_dir", "database/dependencies")
+ if tool_dependency_dir.lower() == "none":
+ tool_dependency_dir = None
+
+ if tool_dependency_dir is not None:
+ tool_dependency_dir = resolve_path(tool_dependency_dir, root)
+ # Setting the following flag to true will ultimately cause tool dependencies
+ # to be located in the shell environment and used by the job that is executing
+ # the tool.
+ use_tool_dependencies = True
+ tool_dependency_cache_dir = kwargs.get('tool_dependency_cache_dir', os.path.join(tool_dependency_dir, '_cache'))
+ use_cached_dependency_manager = string_as_bool(kwargs.get("use_cached_dependency_manager", 'False'))
+ precache_dependencies = string_as_bool(kwargs.get("precache_dependencies", 'True'))
+ else:
+ tool_dependency_dir = None
+ use_tool_dependencies = os.path.exists(dependency_resolvers_config_file)
+ tool_dependency_cache_dir = None
+ precache_dependencies = False
+ use_cached_dependency_manager = False
+
+ return use_tool_dependencies, tool_dependency_dir, use_cached_dependency_manager, tool_dependency_cache_dir, precache_dependencies
+
+
def get_database_engine_options( kwargs, model_prefix='' ):
"""
Allow options for the SQLAlchemy database engine to be passed by using
@@ -787,15 +828,21 @@ def get_database_engine_options( kwargs, model_prefix='' ):
def configure_logging( config ):
- """
- Allow some basic logging configuration to be read from ini file.
+ """Allow some basic logging configuration to be read from ini file.
+
+ This should be able to consume either a galaxy.config.Configuration object
+ or a simple dictionary of configuration variables.
"""
# Get root logger
root = logging.getLogger()
# PasteScript will have already configured the logger if the
# 'loggers' section was found in the config file, otherwise we do
# some simple setup using the 'log_*' values from the config.
- paste_configures_logging = config.global_conf_parser.has_section( "loggers" )
+ parser = getattr(config, "global_conf_parser", None)
+ if parser:
+ paste_configures_logging = config.global_conf_parser.has_section( "loggers" )
+ else:
+ paste_configures_logging = False
auto_configure_logging = not paste_configures_logging and string_as_bool( config.get( "auto_configure_logging", "True" ) )
if auto_configure_logging:
format = config.get( "log_format", "%(name)s %(levelname)s %(asctime)s %(message)s" )
@@ -826,7 +873,7 @@ def configure_logging( config ):
handler.setFormatter( formatter )
root.addHandler( handler )
# If sentry is configured, also log to it
- if config.sentry_dsn:
+ if getattr(config, "sentry_dsn", None):
from raven.handlers.logging import SentryHandler
sentry_handler = SentryHandler( config.sentry_dsn )
sentry_handler.setLevel( logging.WARN )
diff --git a/lib/galaxy/model/orm/scripts.py b/lib/galaxy/model/orm/scripts.py
index d7fce87ecb2..a7a922341cd 100644
--- a/lib/galaxy/model/orm/scripts.py
+++ b/lib/galaxy/model/orm/scripts.py
@@ -2,9 +2,8 @@
Code to support database helper scripts (create_db.py, manage_db.py, etc...).
"""
import logging
-import os.path
-from galaxy.util.properties import load_app_properties
+from galaxy.util.properties import find_config_file, load_app_properties
log = logging.getLogger( __name__ )
@@ -40,25 +39,21 @@ DATABASE = {
}
-def read_config_file_arg( argv, default, old_default ):
+def read_config_file_arg( argv, default, old_default, cwd=None ):
+ config_file = None
if '-c' in argv:
pos = argv.index( '-c' )
argv.pop(pos)
config_file = argv.pop( pos )
- else:
- if not os.path.exists( default ) and os.path.exists( old_default ):
- config_file = old_default
- elif os.path.exists( default ):
- config_file = default
- else:
- config_file = default + ".sample"
- return config_file
+
+ return find_config_file( default, old_default, config_file, cwd=cwd )
def get_config( argv, cwd=None ):
"""
Read sys.argv and parse out repository of migrations and database url.
+ >>> import os
>>> from ConfigParser import SafeConfigParser
>>> from tempfile import mkdtemp
>>> config_dir = mkdtemp()
@@ -89,16 +84,11 @@ def get_config( argv, cwd=None ):
default = database_defaults.get( 'config_file', DEFAULT_CONFIG_FILE )
old_default = database_defaults.get( 'old_config_file' )
- if cwd is not None:
- default = os.path.join( cwd, default )
- old_default = os.path.join( cwd, old_default )
- config_file = read_config_file_arg( argv, default, old_default )
+ config_file = read_config_file_arg( argv, default, old_default, cwd=cwd )
repo = database_defaults[ 'repo' ]
config_prefix = database_defaults.get( 'config_prefix', DEFAULT_CONFIG_PREFIX )
config_override = database_defaults.get( 'config_override', 'GALAXY_CONFIG_' )
default_sqlite_file = database_defaults[ 'default_sqlite_file' ]
- if cwd:
- config_file = os.path.join( cwd, config_file )
properties = load_app_properties( ini_file=config_file, config_prefix=config_override )
diff --git a/lib/galaxy/tools/deps/__init__.py b/lib/galaxy/tools/deps/__init__.py
index 6f254d3965a..b03f33db229 100644
--- a/lib/galaxy/tools/deps/__init__.py
+++ b/lib/galaxy/tools/deps/__init__.py
@@ -19,26 +19,12 @@ from .requirements import (
ToolRequirements
)
from .resolvers import NullDependency
-from .resolvers.conda import CondaDependencyResolver, DEFAULT_ENSURE_CHANNELS
+from .resolvers.conda import CondaDependencyResolver
from .resolvers.galaxy_packages import GalaxyPackageDependencyResolver
from .resolvers.tool_shed_packages import ToolShedPackageDependencyResolver
log = logging.getLogger( __name__ )
-# TODO: Load these from the plugins. Would require a two step initialization of
-# DependencyManager - where the plugins are loaded first and then the config
-# is parsed and sent through.
-EXTRA_CONFIG_KWDS = {
- 'conda_prefix': None,
- 'conda_exec': None,
- 'conda_debug': None,
- 'conda_ensure_channels': DEFAULT_ENSURE_CHANNELS,
- 'conda_auto_install': False,
- 'conda_auto_init': False,
- 'conda_copy_dependencies': False,
- 'precache_dependencies': True,
-}
-
CONFIG_VAL_NOT_FOUND = object()
@@ -47,15 +33,9 @@ def build_dependency_manager( config ):
dependency_manager_kwds = {
'default_base_path': config.tool_dependency_dir,
'conf_file': config.dependency_resolvers_config_file,
+ 'app_config': config,
}
- for key, default_value in EXTRA_CONFIG_KWDS.items():
- value = getattr(config, key, CONFIG_VAL_NOT_FOUND)
- if value is CONFIG_VAL_NOT_FOUND and hasattr(config, "config_dict"):
- value = config.config_dict.get(key, CONFIG_VAL_NOT_FOUND)
- if value is CONFIG_VAL_NOT_FOUND:
- value = default_value
- dependency_manager_kwds[key] = value
- if config.use_cached_dependency_manager:
+ if getattr(config, "use_cached_dependency_manager", False):
dependency_manager_kwds['tool_dependency_cache_dir'] = config.tool_dependency_cache_dir
dependency_manager = CachedDependencyManager(**dependency_manager_kwds)
else:
@@ -90,7 +70,7 @@ class DependencyManager( object ):
and should each contain a file 'env.sh' which can be sourced to make the
dependency available in the current shell environment.
"""
- def __init__( self, default_base_path, conf_file=None, **extra_config ):
+ def __init__( self, default_base_path, conf_file=None, app_config={} ):
"""
Create a new dependency manager looking for packages under the paths listed
in `base_paths`. The default base path is app.config.tool_dependency_dir.
@@ -99,11 +79,30 @@ class DependencyManager( object ):
log.warning( "Path '%s' does not exist, ignoring", default_base_path )
if not os.path.isdir( default_base_path ):
log.warning( "Path '%s' is not directory, ignoring", default_base_path )
- self.extra_config = extra_config
+ self.__app_config = app_config
self.default_base_path = os.path.abspath( default_base_path )
self.resolver_classes = self.__resolvers_dict()
self.dependency_resolvers = self.__build_dependency_resolvers( conf_file )
+ def get_resolver_option(self, resolver, key, explicit_resolver_options={}):
+ """Look in resolver-specific settings for option and then fallback to global settings.
+ """
+ default = resolver.config_options.get(key)
+ config_prefix = resolver.resolver_type
+ global_key = "%s_%s" % (config_prefix, key)
+ value = explicit_resolver_options.get(key, CONFIG_VAL_NOT_FOUND)
+ if value is CONFIG_VAL_NOT_FOUND:
+ if isinstance(self.__app_config, dict):
+ value = self.__app_config.get(global_key, CONFIG_VAL_NOT_FOUND)
+ else:
+ value = getattr(self.__app_config, global_key, CONFIG_VAL_NOT_FOUND)
+ if value is CONFIG_VAL_NOT_FOUND and hasattr(self.__app_config, "config_dict"):
+ value = self.__app_config.config_dict.get(global_key, CONFIG_VAL_NOT_FOUND)
+ if value is CONFIG_VAL_NOT_FOUND:
+ value = default
+
+ return value
+
def dependency_shell_commands( self, requirements, **kwds ):
requirement_to_dependency = self.requirements_to_dependencies(requirements, **kwds)
return [dependency.shell_commands(requirement) for requirement, dependency in requirement_to_dependency.items()]
@@ -141,6 +140,7 @@ class DependencyManager( object ):
# Check requirements all at once
all_unmet = len(requirement_to_dependency) == 0
if all_unmet and hasattr(resolver, "resolve_all"):
+ # TODO: Handle specs.
dependencies = resolver.resolve_all(resolvable_requirements, **kwds)
if dependencies:
assert len(dependencies) == len(resolvable_requirements)
@@ -155,7 +155,7 @@ class DependencyManager( object ):
if requirement in requirement_to_dependency:
continue
- dependency = resolver.resolve( requirement.name, requirement.version, requirement.type, **kwds )
+ dependency = resolver.resolve( requirement, **kwds )
if require_exact and not dependency.exact:
continue
@@ -210,8 +210,8 @@ class DependencyManager( object ):
class CachedDependencyManager(DependencyManager):
- def __init__(self, default_base_path, conf_file=None, **extra_config):
- super(CachedDependencyManager, self).__init__(default_base_path=default_base_path, conf_file=conf_file, **extra_config)
+ def __init__(self, default_base_path, conf_file=None, app_config={}):
+ super(CachedDependencyManager, self).__init__(default_base_path=default_base_path, conf_file=conf_file, app_config=app_config)
def build_cache(self, requirements, **kwds):
resolved_dependencies = self.requirements_to_dependencies(requirements, **kwds)
@@ -240,7 +240,7 @@ class CachedDependencyManager(DependencyManager):
resolved_dependencies = self.requirements_to_dependencies(requirements, **kwds)
cacheable_dependencies = [dep for dep in resolved_dependencies.values() if dep.cacheable]
hashed_dependencies_dir = self.get_hashed_dependencies_path(cacheable_dependencies)
- if not os.path.exists(hashed_dependencies_dir) and self.extra_config['precache_dependencies']:
+ if not os.path.exists(hashed_dependencies_dir) and self.__app_config.getattr("precache_dependencies", False):
# Cache not present, try to create it
self.build_cache(requirements, **kwds)
if os.path.exists(hashed_dependencies_dir):
diff --git a/lib/galaxy/tools/deps/requirements.py b/lib/galaxy/tools/deps/requirements.py
index bb2cfaa612f..bdb0ae0451a 100644
--- a/lib/galaxy/tools/deps/requirements.py
+++ b/lib/galaxy/tools/deps/requirements.py
@@ -1,9 +1,12 @@
+import copy
+
from galaxy.util import (
asbool,
xml_text,
)
from galaxy.util.oset import OrderedSet
+
DEFAULT_REQUIREMENT_TYPE = "package"
DEFAULT_REQUIREMENT_VERSION = None
@@ -14,29 +17,69 @@ class ToolRequirement( object ):
run (for example, a program, package, or library). Requirements can
optionally assert a specific version.
"""
- def __init__( self, name=None, type=None, version=None ):
+ def __init__( self, name=None, type=None, version=None, specs=[] ):
self.name = name
self.type = type
self.version = version
+ self.specs = specs
def to_dict( self ):
- return dict(name=self.name, type=self.type, version=self.version)
+ specs = [s.to_dict() for s in self.specs]
+ return dict(name=self.name, type=self.type, version=self.version, specs=specs)
+
+ def copy( self ):
+ return copy.deepcopy( self )
@staticmethod
def from_dict( dict ):
version = dict.get( "version", None )
name = dict.get("name", None)
type = dict.get("type", None)
- return ToolRequirement( name=name, type=type, version=version )
+ specs = [RequirementSpecification.from_dict(s) for s in dict.get("specs", [])]
+ return ToolRequirement( name=name, type=type, version=version, specs=specs )
def __eq__(self, other):
- return self.name == other.name and self.type == other.type and self.version == other.version
+ return self.name == other.name and self.type == other.type and self.version == other.version and self.specs == other.specs
def __ne__(self, other):
return not self.__eq__(other)
def __hash__(self):
- return hash((self.name, self.type, self.version))
+ return hash((self.name, self.type, self.version, frozenset(self.specs)))
+
+
+class RequirementSpecification(object):
+ """Refine a requirement using a URI."""
+
+ def __init__(self, uri, version=None):
+ self.uri = uri
+ self.version = version
+
+ @property
+ def specifies_version(self):
+ return self.version is not None
+
+ @property
+ def short_name(self):
+ return self.uri.split("/")[-1]
+
+ def to_dict(self):
+ return dict(uri=self.uri, version=self.version)
+
+ @staticmethod
+ def from_dict(dict):
+ uri = dict.get["uri"]
+ version = dict.get("version", None)
+ return RequirementSpecification(uri=uri, version=version)
+
+ def __eq__(self, other):
+ return self.uri == other.uri and self.version == other.version
+
+ def __ne__(self, other):
+ return not self.__eq__(other)
+
+ def __hash__(self):
+ return hash((self.uri, self.version))
class ToolRequirements(object):
diff --git a/lib/galaxy/tools/deps/resolvers/__init__.py b/lib/galaxy/tools/deps/resolvers/__init__.py
index c24c107887a..eb6fecbeaed 100644
--- a/lib/galaxy/tools/deps/resolvers/__init__.py
+++ b/lib/galaxy/tools/deps/resolvers/__init__.py
@@ -5,6 +5,9 @@ from abc import (
abstractproperty,
)
+import yaml
+
+from galaxy.util import listify
from galaxy.util.dictifiable import Dictifiable
from ..requirements import ToolRequirement
@@ -22,10 +25,11 @@ class DependencyResolver(Dictifiable, object):
# resolution.
disabled = False
resolves_simple_dependencies = True
+ config_options = {}
__metaclass__ = ABCMeta
@abstractmethod
- def resolve( self, name, version, type, **kwds ):
+ def resolve( self, requirement, **kwds ):
"""Given inputs describing dependency in the abstract yield a Dependency object.
The Dependency object describes various attributes (script, bin,
@@ -37,17 +41,14 @@ class DependencyResolver(Dictifiable, object):
request version is 'default'.)
"""
- def _get_config_option(self, key, dependency_resolver, default=None, config_prefix=None, **kwds):
- """ Look in resolver-specific settings for option and then fallback to
- global settings.
+
+class MultipleDependencyResolver:
+ """Variant of DependencyResolver that can optionally resolve multiple dependencies together."""
+
+ @abstractmethod
+ def resolve_all( self, requirements, **kwds ):
+ """Given multiple requirements yield Dependency objects if and only if they may all be resolved together.
"""
- global_key = "%s_%s" % (config_prefix, key)
- if key in kwds:
- return kwds.get(key)
- elif global_key in dependency_resolver.extra_config:
- return dependency_resolver.extra_config.get(global_key)
- else:
- return default
class ListableDependencyResolver:
@@ -67,6 +68,116 @@ class ListableDependencyResolver:
return ToolRequirement(name=name, type="package", version=version)
+class MappableDependencyResolver:
+ """Mix this into a ``DependencyResolver`` to allow mapping files.
+
+ Mapping files allow adapting generic requirements to specific local implementations.
+ """
+
+ def _setup_mapping(self, dependency_manager, **kwds):
+ mapping_files = dependency_manager.get_resolver_option(self, "mapping_files", explicit_resolver_options=kwds)
+ mappings = []
+ if mapping_files:
+ mapping_files = listify(mapping_files)
+ for mapping_file in mapping_files:
+ mappings.extend(MappableDependencyResolver._mapping_file_to_list(mapping_file))
+ self._mappings = mappings
+
+ @staticmethod
+ def _mapping_file_to_list(mapping_file):
+ with open(mapping_file, "r") as f:
+ raw_mapping = yaml.load(f) or []
+ return map(RequirementMapping.from_dict, raw_mapping)
+
+ def _expand_mappings(self, requirement):
+ for mapping in self._mappings:
+ if requirement.name == mapping.from_name:
+ if mapping.from_version is not None and mapping.from_version != requirement.version:
+ continue
+
+ requirement = requirement.copy()
+ requirement.name = mapping.to_name
+ if mapping.to_version is not None:
+ requirement.version = mapping.to_version
+
+ break
+
+ return requirement
+
+
+class RequirementMapping(object):
+
+ def __init__(self, from_name, from_version, to_name, to_version):
+ self.from_name = from_name
+ self.from_version = from_version
+ self.to_name = to_name
+ self.to_version = to_version
+
+ @staticmethod
+ def from_dict(raw_mapping):
+ from_raw = raw_mapping.get("from")
+ if isinstance(from_raw, dict):
+ from_name = from_raw.get("name")
+ from_version = str(from_raw.get("version"))
+ else:
+ from_name = from_raw
+ from_version = None
+
+ to_raw = raw_mapping.get("to")
+ if isinstance(to_raw, dict):
+ to_name = to_raw.get("name", from_name)
+ to_version = str(to_raw.get("version"))
+ else:
+ to_name = to_raw
+ to_version = None
+
+ return RequirementMapping(from_name, from_version, to_name, to_version)
+
+
+class SpecificationAwareDependencyResolver:
+ """Mix this into a :class:`DependencyResolver` to implement URI specification matching.
+
+ Allows adapting generic requirements to more specific URIs - to tailor name
+ or version to specified resolution system.
+ """
+ __metaclass__ = ABCMeta
+
+ @abstractmethod
+ def _expand_specs(self, requirement):
+ """Find closest matching specification for discovered resolver and return new concrete requirement."""
+
+
+class SpecificationPatternDependencyResolver:
+ """Implement the :class:`SpecificationAwareDependencyResolver` with a regex pattern."""
+
+ @abstractproperty
+ def _specification_pattern(self):
+ """Pattern of URI to match against."""
+
+ def _find_specification(self, specs):
+ pattern = self._specification_pattern
+ for spec in specs:
+ if pattern.match(spec.uri):
+ return spec
+ return None
+
+ def _expand_specs(self, requirement):
+ name = requirement.name
+ version = requirement.version
+ specs = requirement.specs
+
+ spec = self._find_specification(specs)
+ if spec is not None:
+ name = spec.short_name
+ version = spec.version or version
+
+ requirement = requirement.copy()
+ requirement.name = name
+ requirement.version = version
+
+ return requirement
+
+
class InstallableDependencyResolver:
""" Mix this into a ``DependencyResolver`` and implement to indicate
the dependency resolver can attempt to install new dependencies.
diff --git a/lib/galaxy/tools/deps/resolvers/brewed_tool_shed_packages.py b/lib/galaxy/tools/deps/resolvers/brewed_tool_shed_packages.py
index 5585de56c42..a6b4cde4520 100644
--- a/lib/galaxy/tools/deps/resolvers/brewed_tool_shed_packages.py
+++ b/lib/galaxy/tools/deps/resolvers/brewed_tool_shed_packages.py
@@ -29,7 +29,8 @@ class HomebrewToolShedDependencyResolver(
self._init_homebrew(**kwds)
self._init_base_path(dependency_manager, **kwds)
- def resolve(self, name, version, type, **kwds):
+ def resolve(self, requirement, **kwds):
+ name, version, type = requirement.name, requirement.version, requirement.type
if type != "package":
return NullDependency(version=version, name=name)
if version is None:
diff --git a/lib/galaxy/tools/deps/resolvers/conda.py b/lib/galaxy/tools/deps/resolvers/conda.py
index f6fe1d3925f..87a5ec14f53 100644
--- a/lib/galaxy/tools/deps/resolvers/conda.py
+++ b/lib/galaxy/tools/deps/resolvers/conda.py
@@ -5,6 +5,7 @@ incompatible changes coming.
import logging
import os
+import re
import galaxy.tools.deps.installable
@@ -28,7 +29,10 @@ from ..resolvers import (
DependencyResolver,
InstallableDependencyResolver,
ListableDependencyResolver,
+ MappableDependencyResolver,
+ MultipleDependencyResolver,
NullDependency,
+ SpecificationPatternDependencyResolver,
)
@@ -39,16 +43,27 @@ DEFAULT_ENSURE_CHANNELS = "iuc,bioconda,r,defaults,conda-forge"
log = logging.getLogger(__name__)
-class CondaDependencyResolver(DependencyResolver, ListableDependencyResolver, InstallableDependencyResolver):
+class CondaDependencyResolver(DependencyResolver, MultipleDependencyResolver, ListableDependencyResolver, InstallableDependencyResolver, SpecificationPatternDependencyResolver, MappableDependencyResolver):
dict_collection_visible_keys = DependencyResolver.dict_collection_visible_keys + ['conda_prefix', 'versionless', 'ensure_channels', 'auto_install']
resolver_type = "conda"
+ config_options = {
+ 'prefix': None,
+ 'exec': None,
+ 'debug': None,
+ 'ensure_channels': DEFAULT_ENSURE_CHANNELS,
+ 'auto_install': False,
+ 'auto_init': True,
+ 'copy_dependencies': False,
+ }
+ _specification_pattern = re.compile(r"https\:\/\/anaconda.org\/\w+\/\w+")
def __init__(self, dependency_manager, **kwds):
+ self._setup_mapping(dependency_manager, **kwds)
self.versionless = _string_as_bool(kwds.get('versionless', 'false'))
self.dependency_manager = dependency_manager
def get_option(name):
- return self._get_config_option(name, dependency_manager, config_prefix="conda", **kwds)
+ return dependency_manager.get_resolver_option(self, name, explicit_resolver_options=kwds)
# Conda context options (these define the environment)
conda_prefix = get_option("prefix")
@@ -133,6 +148,8 @@ class CondaDependencyResolver(DependencyResolver, ListableDependencyResolver, In
conda_targets = []
for requirement in requirements:
+ requirement = self._expand_requirement(requirement)
+
version = requirement.version
if self.versionless:
version = None
@@ -170,7 +187,10 @@ class CondaDependencyResolver(DependencyResolver, ListableDependencyResolver, In
assert len(conda_targets) == 1
return conda_targets[0].install_environment
- def resolve(self, name, version, type, **kwds):
+ def resolve(self, requirement, **kwds):
+ requirement = self._expand_requirement(requirement)
+ name, version, type = requirement.name, requirement.version, requirement.type
+
# Check for conda just not being there, this way we can enable
# conda by default and just do nothing in not configured.
if not os.path.isdir(self.conda_context.conda_prefix):
@@ -218,6 +238,9 @@ class CondaDependencyResolver(DependencyResolver, ListableDependencyResolver, In
preserve_python_environment=preserve_python_environment,
)
+ def _expand_requirement(self, requirement):
+ return self._expand_specs(self._expand_mappings(requirement))
+
def list_dependencies(self):
for install_target in installed_conda_targets(self.conda_context):
name = install_target.package
diff --git a/lib/galaxy/tools/deps/resolvers/default_conda_mapping.yml b/lib/galaxy/tools/deps/resolvers/default_conda_mapping.yml
new file mode 100644
index 00000000000..9b1e59e7a6b
--- /dev/null
+++ b/lib/galaxy/tools/deps/resolvers/default_conda_mapping.yml
@@ -0,0 +1,5 @@
+- from: R
+ to: r-base
+- from: blast+
+ to: blast
+
\ No newline at end of file
diff --git a/lib/galaxy/tools/deps/resolvers/galaxy_packages.py b/lib/galaxy/tools/deps/resolvers/galaxy_packages.py
index 858ff0faf98..8e1a1145030 100644
--- a/lib/galaxy/tools/deps/resolvers/galaxy_packages.py
+++ b/lib/galaxy/tools/deps/resolvers/galaxy_packages.py
@@ -16,6 +16,7 @@ from ..resolvers import (
Dependency,
DependencyResolver,
ListableDependencyResolver,
+ MappableDependencyResolver,
NullDependency,
)
@@ -66,11 +67,13 @@ class BaseGalaxyPackageDependencyResolver(DependencyResolver, UsesToolDependency
self.versionless = str(kwds.get('versionless', "false")).lower() == "true"
self._init_base_path( dependency_manager, **kwds )
- def resolve( self, name, version, type, **kwds ):
+ def resolve(self, requirement, **kwds):
"""
Attempt to find a dependency named `name` at version `version`. If version is None, return the "default" version as determined using a
symbolic link (if found). Returns a triple of: env_script, base_path, real_version
"""
+ name, version, type = requirement.name, requirement.version, requirement.type
+
if version is None or self.versionless:
exact = not self.versionless or version is None
return self._find_dep_default( name, type=type, exact=exact, **kwds )
@@ -101,9 +104,17 @@ class BaseGalaxyPackageDependencyResolver(DependencyResolver, UsesToolDependency
return NullDependency(version=version, name=name)
-class GalaxyPackageDependencyResolver(BaseGalaxyPackageDependencyResolver, ListableDependencyResolver):
+class GalaxyPackageDependencyResolver(BaseGalaxyPackageDependencyResolver, ListableDependencyResolver, MappableDependencyResolver):
resolver_type = "galaxy_packages"
+ def __init__(self, dependency_manager, **kwds):
+ super(GalaxyPackageDependencyResolver, self).__init__(dependency_manager, **kwds)
+ self._setup_mapping(dependency_manager, **kwds)
+
+ def resolve(self, requirement, **kwds):
+ requirement = self._expand_mappings(requirement)
+ return super(GalaxyPackageDependencyResolver, self).resolve(requirement, **kwds)
+
def list_dependencies(self):
base_path = self.base_path
for package_name in listdir(base_path):
diff --git a/lib/galaxy/tools/deps/resolvers/homebrew.py b/lib/galaxy/tools/deps/resolvers/homebrew.py
index 395eb1eb518..2a9781d3904 100644
--- a/lib/galaxy/tools/deps/resolvers/homebrew.py
+++ b/lib/galaxy/tools/deps/resolvers/homebrew.py
@@ -39,7 +39,9 @@ class HomebrewDependencyResolver(DependencyResolver, UsesHomebrewMixin):
self._init_homebrew(**kwds)
- def resolve(self, name, version, type, **kwds):
+ def resolve(self, requirement, **kwds):
+ name, version, type = requirement.name, requirement.version, requirement.type
+
if type != "package":
return NullDependency(version=version, name=name)
diff --git a/lib/galaxy/tools/deps/resolvers/modules.py b/lib/galaxy/tools/deps/resolvers/modules.py
index 227abb72a9c..0cbbc5764f8 100644
--- a/lib/galaxy/tools/deps/resolvers/modules.py
+++ b/lib/galaxy/tools/deps/resolvers/modules.py
@@ -13,7 +13,12 @@ from subprocess import PIPE, Popen
from six import StringIO
-from ..resolvers import Dependency, DependencyResolver, NullDependency
+from ..resolvers import (
+ Dependency,
+ DependencyResolver,
+ MappableDependencyResolver,
+ NullDependency,
+)
log = logging.getLogger( __name__ )
@@ -24,11 +29,12 @@ DEFAULT_MODULE_PREFETCH = "true"
UNKNOWN_FIND_BY_MESSAGE = "ModuleDependencyResolver does not know how to find modules by [%s], find_by should be one of %s"
-class ModuleDependencyResolver(DependencyResolver):
+class ModuleDependencyResolver(DependencyResolver, MappableDependencyResolver):
dict_collection_visible_keys = DependencyResolver.dict_collection_visible_keys + ['base_path', 'modulepath']
resolver_type = "modules"
def __init__(self, dependency_manager, **kwds):
+ self._setup_mapping(dependency_manager, **kwds)
self.versionless = _string_as_bool(kwds.get('versionless', 'false'))
find_by = kwds.get('find_by', 'avail')
prefetch = _string_as_bool(kwds.get('prefetch', DEFAULT_MODULE_PREFETCH))
@@ -51,7 +57,10 @@ class ModuleDependencyResolver(DependencyResolver):
module_path = DEFAULT_MODULE_PATH
return module_path
- def resolve( self, name, version, type, **kwds ):
+ def resolve(self, requirement, **kwds):
+ requirement = self._expand_mappings(requirement)
+ name, version, type = requirement.name, requirement.version, requirement.type
+
if type != "package":
return NullDependency(version=version, name=name)
diff --git a/lib/galaxy/util/properties.py b/lib/galaxy/util/properties.py
index f2de11b3ae2..b5743dc0ef2 100644
--- a/lib/galaxy/util/properties.py
+++ b/lib/galaxy/util/properties.py
@@ -10,6 +10,28 @@ from six import iteritems
from six.moves.configparser import ConfigParser
+def find_config_file(default, old_default, explicit, cwd=None):
+ if cwd is not None:
+ default = os.path.join(cwd, default)
+ old_default = os.path.join(cwd, old_default)
+ if explicit is not None:
+ explicit = os.path.join(cwd, explicit)
+
+ if explicit:
+ if os.path.exists(explicit):
+ config_file = explicit
+ else:
+ raise Exception("Problem determining Galaxy's configuration - the specified configuration file cannot be found.")
+ else:
+ if not os.path.exists( default ) and os.path.exists( old_default ):
+ config_file = old_default
+ elif os.path.exists( default ):
+ config_file = default
+ else:
+ config_file = default + ".sample"
+ return config_file
+
+
def load_app_properties(
kwds={},
ini_file=None,
@@ -97,4 +119,4 @@ class NicerConfigParser(ConfigParser):
raise
-__all__ = ('load_app_properties', 'NicerConfigParser')
+__all__ = ('find_config_file', 'load_app_properties', 'NicerConfigParser')
diff --git a/run.sh b/run.sh
index 039047a87b8..9d7dbdfb211 100755
--- a/run.sh
+++ b/run.sh
@@ -14,6 +14,7 @@ then
. $GALAXY_LOCAL_ENV_FILE
fi
+INITIALIZE_TOOL_DEPENDENCIES=1 # Install Conda if needed.
# Pop args meant for common_startup.sh
while :
do
@@ -22,6 +23,10 @@ do
common_startup_args="$common_startup_args $1"
shift
;;
+ --skip-tool-dependency-initialization)
+ INITIALIZE_TOOL_DEPENDENCIES=0
+ shift
+ ;;
--skip-venv)
skip_venv=1
common_startup_args="$common_startup_args $1"
@@ -98,6 +103,11 @@ if [ -z "$GALAXY_CONFIG_FILE" ]; then
export GALAXY_CONFIG_FILE
fi
+if [ $INITIALIZE_TOOL_DEPENDENCIES -eq 1 ]; then
+ # Install Conda environment if needed.
+ python ./scripts/manage_tool_dependencies.py -c "$GALAXY_CONFIG_FILE" init_if_needed
+fi
+
if [ -n "$GALAXY_RUN_ALL" ]; then
servers=$(sed -n 's/^\[server:\(.*\)\]/\1/ p' "$GALAXY_CONFIG_FILE" | xargs echo)
if [ -z "$stop_daemon_arg_set" -a -z "$daemon_or_restart_arg_set" ]; then
diff --git a/scripts/manage_tool_dependencies.py b/scripts/manage_tool_dependencies.py
new file mode 100644
index 00000000000..07b5b8802ca
--- /dev/null
+++ b/scripts/manage_tool_dependencies.py
@@ -0,0 +1,87 @@
+import argparse
+import os.path
+import sys
+
+sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, 'lib')))
+
+from galaxy.util.properties import find_config_file, load_app_properties
+from galaxy.config import (
+ configure_logging,
+ find_path,
+ find_root,
+ parse_dependency_options,
+)
+from galaxy.tools.deps import CachedDependencyManager, DependencyManager, NullDependencyManager
+
+DESCRIPTION = "Script to manage tool dependencies (with focus on a Conda environments)."
+
+
+def main(argv=None):
+ """Entry point for conversion process."""
+ if argv is None:
+ argv = sys.argv[1:]
+ args = _arg_parser().parse_args(argv)
+ action = args.action
+ action_func = ACTIONS[action]
+ action_func(args)
+
+
+def _init_if_needed(args):
+ kwargs = _app_properties(args)
+
+ # If conda_auto_init is set, simply building the Conda resolver will call handle installation.
+ _build_dependency_manager_no_config(kwargs)
+
+
+def _app_properties(args):
+ config_file = find_config_file("config/galaxy.ini", "universe_wsgi.ini", args.config_file)
+ app_properties = load_app_properties(ini_file=config_file)
+ return app_properties
+
+
+def _arg_parser():
+ parser = argparse.ArgumentParser(description=DESCRIPTION)
+ parser.add_argument('action', metavar='ACTION', type=str,
+ choices=ACTIONS.keys(),
+ help='action to perform')
+ parser.add_argument("-c", "--config-file", default=None)
+ return parser
+
+
+def _build_dependency_manager_no_config(kwargs):
+ """Simplified variant of build_dependency_manager from galaxy.tools.deps.
+
+ The canonical factory method requires a full Galaxy configuration object
+ which we do not have available in this script (an optimization).
+ """
+ configure_logging(kwargs)
+ root = find_root(kwargs)
+ dependency_resolvers_config_file = find_path(kwargs, "dependency_resolvers_config_file", root)
+ use_dependencies, tool_dependency_dir, use_cached_dependency_manager, tool_dependency_cache_dir, precache_dependencies = \
+ parse_dependency_options(kwargs, root, dependency_resolvers_config_file)
+
+ if not use_dependencies:
+ dependency_manager = NullDependencyManager()
+ else:
+ dependency_manager_kwds = {
+ 'default_base_path': tool_dependency_dir,
+ 'conf_file': dependency_resolvers_config_file,
+ 'app_config': kwargs,
+ }
+
+ if use_cached_dependency_manager:
+ dependency_manager_kwds['tool_dependency_cache_dir'] = tool_dependency_cache_dir
+ dependency_manager = CachedDependencyManager(**dependency_manager_kwds)
+ else:
+ dependency_manager = DependencyManager( **dependency_manager_kwds )
+
+ return dependency_manager
+
+
+ACTIONS = {
+ "init_if_needed": _init_if_needed,
+}
+
+
+if __name__ == '__main__':
+ main()
diff --git a/test/functional/tools/legacy_r.xml b/test/functional/tools/legacy_r.xml
new file mode 100644
index 00000000000..5d70feede9f
--- /dev/null
+++ b/test/functional/tools/legacy_r.xml
@@ -0,0 +1,14 @@
+
+ $out_file1
+ ]]>
+
+ R
+
+
+
+
+
+
+
+
diff --git a/test/functional/tools/samples_tool_conf.xml b/test/functional/tools/samples_tool_conf.xml
index d678c8be35d..02677a581bd 100644
--- a/test/functional/tools/samples_tool_conf.xml
+++ b/test/functional/tools/samples_tool_conf.xml
@@ -124,6 +124,8 @@
+
+
diff --git a/test/integration/test_resolvers.py b/test/integration/test_resolvers.py
index 5c36fd30fbe..a676a299ca3 100644
--- a/test/integration/test_resolvers.py
+++ b/test/integration/test_resolvers.py
@@ -4,6 +4,9 @@ import shutil
from tempfile import mkdtemp
from base import integration_util
+from base.populators import (
+ DatasetPopulator,
+)
GNUPLOT = {u'version': u'4.6', u'type': u'package', u'name': u'gnuplot'}
@@ -71,6 +74,20 @@ class CondaResolutionIntegrationTestCase(integration_util.IntegrationTestCase):
response = create_response.json()
self._assert_dependency_type(response)
+ def test_legacy_r_mapping( self ):
+ """
+ """
+ dataset_populator = DatasetPopulator(self.galaxy_interactor)
+ history_id = dataset_populator.new_history()
+ payload = dataset_populator.run_tool_payload(
+ tool_id="legacy_R",
+ inputs={},
+ history_id=history_id,
+ )
+ create_response = self._post( "tools", data=payload )
+ self._assert_status_code_is( create_response, 200 )
+ dataset_populator.wait_for_history( history_id, assert_ok=True )
+
def test_dependency_status_installed_not_exact( self ):
"""
GET request to dependency_resolvers/dependency with GNUPLOT dependency.
diff --git a/test/unit/tools/test_tool_deps.py b/test/unit/tools/test_tool_deps.py
index 2f4bda30fcb..1c18eedf29e 100644
--- a/test/unit/tools/test_tool_deps.py
+++ b/test/unit/tools/test_tool_deps.py
@@ -40,7 +40,7 @@ def test_tool_dependencies():
if sub == "env.sh":
__touch( os.path.join( p, "env.sh" ) )
- dm = DependencyManager( default_base_path=base_path )
+ dm = __dependency_manager_for_base_path( default_base_path=base_path )
dependency = dm.find_dep( "dep1", "1.0" )
assert dependency.script == os.path.join( base_path, 'dep1', '1.0', 'env.sh' )
assert dependency.path == os.path.join( base_path, 'dep1', '1.0' )
@@ -70,7 +70,7 @@ TEST_VERSION = "0.5.9"
def test_toolshed_set_enviornment_requiremetns():
with __test_base_path() as base_path:
test_repo = __build_test_repo('set_environment')
- dm = DependencyManager( default_base_path=base_path )
+ dm = __dependency_manager_for_base_path( default_base_path=base_path )
env_settings_dir = os.path.join(base_path, "environment_settings", TEST_REPO_NAME, TEST_REPO_USER, TEST_REPO_NAME, TEST_REPO_CHANGESET)
os.makedirs(env_settings_dir)
dependency = dm.find_dep( TEST_REPO_NAME, version=None, type='set_environment', installed_tool_dependencies=[test_repo] )
@@ -81,7 +81,7 @@ def test_toolshed_set_enviornment_requiremetns():
def test_toolshed_package_requirements():
with __test_base_path() as base_path:
test_repo = __build_test_repo('package', version=TEST_VERSION)
- dm = DependencyManager( default_base_path=base_path )
+ dm = __dependency_manager_for_base_path( default_base_path=base_path )
package_dir = __build_ts_test_package(base_path)
dependency = dm.find_dep( TEST_REPO_NAME, version=TEST_VERSION, type='package', installed_tool_dependencies=[test_repo] )
assert dependency.version == TEST_VERSION
@@ -90,7 +90,7 @@ def test_toolshed_package_requirements():
def test_toolshed_tools_fallback_on_manual_dependencies():
with __test_base_path() as base_path:
- dm = DependencyManager( default_base_path=base_path )
+ dm = __dependency_manager_for_base_path( default_base_path=base_path )
test_repo = __build_test_repo('package', version=TEST_VERSION)
env_path = __setup_galaxy_package_dep(base_path, "dep1", "1.0")
dependency = dm.find_dep( "dep1", version="1.0", type='package', installed_tool_dependencies=[test_repo] )
@@ -100,7 +100,7 @@ def test_toolshed_tools_fallback_on_manual_dependencies():
def test_toolshed_greater_precendence():
with __test_base_path() as base_path:
- dm = DependencyManager( default_base_path=base_path )
+ dm = __dependency_manager_for_base_path( default_base_path=base_path )
test_repo = __build_test_repo('package', version=TEST_VERSION)
ts_package_dir = __build_ts_test_package(base_path)
gx_env_path = __setup_galaxy_package_dep(base_path, TEST_REPO_NAME, TEST_VERSION)
@@ -142,13 +142,7 @@ def test_tool_requirements():
def test_module_dependency_resolver():
with __test_base_path() as temp_directory:
- module_script = os.path.join(temp_directory, "modulecmd")
- __write_script(module_script, '''#!/bin/sh
-cat %s/example_output 1>&2;
-''' % temp_directory)
- with open(os.path.join(temp_directory, "example_output"), "w") as f:
- # Subset of module avail from MSI cluster.
- f.write('''
+ module_script = _setup_module_command(temp_directory, '''
-------------------------- /soft/modules/modulefiles ---------------------------
JAGS/3.2.0-gcc45
JAGS/3.3.0-gcc4.7.2
@@ -168,19 +162,87 @@ advisor/2013/update1 intel/11.1.075 mkl/10.2.1.017
advisor/2013/update2 intel/11.1.080 mkl/10.2.5.035
advisor/2013/update3 intel/12.0 mkl/10.2.7.041
''')
- resolver = ModuleDependencyResolver(None, modulecmd=module_script)
- module = resolver.resolve( name="R", version=None, type="package" )
+ resolver = ModuleDependencyResolver(_SimpleDependencyManager(), modulecmd=module_script)
+ module = resolver.resolve( ToolRequirement( name="R", version=None, type="package" ) )
assert module.module_name == "R"
assert module.module_version is None
- module = resolver.resolve( name="R", version="3.0.1", type="package" )
+ module = resolver.resolve( ToolRequirement( name="R", version="3.0.1", type="package" ) )
assert module.module_name == "R"
assert module.module_version == "3.0.1"
- module = resolver.resolve( name="R", version="3.0.4", type="package" )
+ module = resolver.resolve( ToolRequirement( name="R", version="3.0.4", type="package" ) )
assert isinstance(module, NullDependency)
+def test_module_resolver_with_mapping():
+ with __test_base_path() as temp_directory:
+ module_script = _setup_module_command(temp_directory, '''
+-------------------------- /soft/modules/modulefiles ---------------------------
+blast/2.24
+''')
+ mapping_file = os.path.join(temp_directory, "mapping")
+ with open(mapping_file, "w") as f:
+ f.write('''
+- from: blast+
+ to: blast
+''')
+
+ resolver = ModuleDependencyResolver(_SimpleDependencyManager(), modulecmd=module_script, mapping_files=mapping_file)
+ module = resolver.resolve( ToolRequirement( name="blast+", version="2.24", type="package" ) )
+ assert module.module_name == "blast"
+ assert module.module_version == "2.24", module.module_version
+
+
+def test_module_resolver_with_mapping_versions():
+ with __test_base_path() as temp_directory:
+ module_script = _setup_module_command(temp_directory, '''
+-------------------------- /soft/modules/modulefiles ---------------------------
+blast/2.22.0-mpi
+blast/2.23
+blast/2.24.0-mpi
+''')
+ mapping_file = os.path.join(temp_directory, "mapping")
+ with open(mapping_file, "w") as f:
+ f.write('''
+- from:
+ name: blast+
+ version: 2.24
+ to:
+ name: blast
+ version: 2.24.0-mpi
+- from:
+ name: blast
+ version: 2.22
+ to:
+ version: 2.22.0-mpi
+''')
+
+ resolver = ModuleDependencyResolver(_SimpleDependencyManager(), modulecmd=module_script, mapping_files=mapping_file)
+ module = resolver.resolve( ToolRequirement( name="blast+", version="2.24", type="package" ) )
+ assert module.module_name == "blast"
+ assert module.module_version == "2.24.0-mpi", module.module_version
+
+ resolver = ModuleDependencyResolver(_SimpleDependencyManager(), modulecmd=module_script, mapping_files=mapping_file)
+ module = resolver.resolve( ToolRequirement( name="blast+", version="2.23", type="package" ) )
+ assert isinstance(module, NullDependency)
+
+ module = resolver.resolve( ToolRequirement( name="blast", version="2.22", type="package" ) )
+ assert module.module_name == "blast"
+ assert module.module_version == "2.22.0-mpi", module.module_version
+
+
+def _setup_module_command(temp_directory, contents):
+ module_script = os.path.join(temp_directory, "modulecmd")
+ __write_script(module_script, '''#!/bin/sh
+cat %s/example_output 1>&2;
+''' % temp_directory)
+ with open(os.path.join(temp_directory, "example_output"), "w") as f:
+ # Subset of module avail from MSI cluster.
+ f.write(contents)
+ return module_script
+
+
def test_module_dependency():
with __test_base_path() as temp_directory:
# Create mock modulecmd script that just exports a variable
@@ -219,7 +281,7 @@ def test_shell_commands_built():
# Test that dependency manager builds valid shell commands for a list of
# requirements.
with __test_base_path() as base_path:
- dm = DependencyManager( default_base_path=base_path )
+ dm = __dependency_manager_for_base_path( default_base_path=base_path )
__setup_galaxy_package_dep( base_path, TEST_REPO_NAME, TEST_VERSION, contents="export FOO=\"bar\"" )
mock_requirements = ToolRequirements([{'type': 'package', 'version': TEST_VERSION, 'name': TEST_REPO_NAME}])
commands = dm.dependency_shell_commands( mock_requirements )
@@ -411,5 +473,15 @@ def __dependency_manager(xml_content):
f = tempfile.NamedTemporaryFile()
f.write(xml_content)
f.flush()
- dm = DependencyManager( default_base_path=base_path, conf_file=f.name )
+ dm = __dependency_manager_for_base_path( default_base_path=base_path, conf_file=f.name )
yield dm
+
+
+def __dependency_manager_for_base_path(default_base_path, conf_file=None ):
+ return DependencyManager( default_base_path=default_base_path, conf_file=conf_file, app_config={"conda_auto_init": False} )
+
+
+class _SimpleDependencyManager(object):
+
+ def get_resolver_option(self, resolver, key, explicit_resolver_options={}):
+ return explicit_resolver_options.get(key)