diff --git a/config/galaxy.ini.sample b/config/galaxy.ini.sample index 43dbf738559..ef07ff6d049 100644 --- a/config/galaxy.ini.sample +++ b/config/galaxy.ini.sample @@ -210,7 +210,7 @@ paste.app_factory = galaxy.web.buildapp:app_factory #conda_auto_install = False # Set to True to instruct Galaxy to install Conda from the web automatically # if it cannot find a local copy and conda_exec is not configured. -#conda_auto_init = False +#conda_auto_init = True # You must set this to True if conda_prefix and job_working_directory are not on the same # volume, or some conda dependencies will fail to execute at job runtime. # Conda will copy packages content instead of creating hardlinks or symlinks. diff --git a/config/local_conda_mapping.yml.sample b/config/local_conda_mapping.yml.sample new file mode 100644 index 00000000000..383ce4836c6 --- /dev/null +++ b/config/local_conda_mapping.yml.sample @@ -0,0 +1,2 @@ +# See $GALAXY_ROOT/lib/galaxy/tools/deps/default_conda_mapping.yml for example mapping - +# additional site-specific mappings can be added to config/conda_mapping.yml. diff --git a/lib/galaxy/config.py b/lib/galaxy/config.py index 8b709f384f0..dea393a0bf8 100644 --- a/lib/galaxy/config.py +++ b/lib/galaxy/config.py @@ -32,6 +32,46 @@ from .version import VERSION_MAJOR log = logging.getLogger( __name__ ) +PATH_DEFAULTS = dict( + auth_config_file=['config/auth_conf.xml', 'config/auth_conf.xml.sample'], + data_manager_config_file=['config/data_manager_conf.xml', 'data_manager_conf.xml', 'config/data_manager_conf.xml.sample'], + datatypes_config_file=['config/datatypes_conf.xml', 'datatypes_conf.xml', 'config/datatypes_conf.xml.sample'], + external_service_type_config_file=['config/external_service_types_conf.xml', 'external_service_types_conf.xml', 'config/external_service_types_conf.xml.sample'], + job_config_file=['config/job_conf.xml', 'job_conf.xml'], + tool_destinations_config_file=['config/tool_destinations.yml', 'config/tool_destinations.yml.sample'], + job_metrics_config_file=['config/job_metrics_conf.xml', 'job_metrics_conf.xml', 'config/job_metrics_conf.xml.sample'], + dependency_resolvers_config_file=['config/dependency_resolvers_conf.xml', 'dependency_resolvers_conf.xml'], + job_resource_params_file=['config/job_resource_params_conf.xml', 'job_resource_params_conf.xml'], + migrated_tools_config=['migrated_tools_conf.xml', 'config/migrated_tools_conf.xml'], + object_store_config_file=['config/object_store_conf.xml', 'object_store_conf.xml'], + openid_config_file=['config/openid_conf.xml', 'openid_conf.xml', 'config/openid_conf.xml.sample'], + shed_data_manager_config_file=['shed_data_manager_conf.xml', 'config/shed_data_manager_conf.xml'], + shed_tool_data_table_config=['shed_tool_data_table_conf.xml', 'config/shed_tool_data_table_conf.xml'], + tool_sheds_config_file=['config/tool_sheds_conf.xml', 'tool_sheds_conf.xml', 'config/tool_sheds_conf.xml.sample'], + workflow_schedulers_config_file=['config/workflow_schedulers_conf.xml', 'config/workflow_schedulers_conf.xml.sample'], + modules_mapping_files=['config/environment_modules_mapping.yml', 'config/environment_modules_mapping.yml.sample'], + local_conda_mapping_file=['config/local_conda_mapping.yml', 'config/local_conda_mapping.yml.sample'], +) + +PATH_LIST_DEFAULTS = dict( + tool_data_table_config_path=['config/tool_data_table_conf.xml', 'tool_data_table_conf.xml', 'config/tool_data_table_conf.xml.sample'], + # rationale: + # [0]: user has explicitly created config/tool_conf.xml but did not + # move their existing shed_tool_conf.xml, don't use + # config/shed_tool_conf.xml, which is probably the empty + # version copied from the sample, or else their shed tools + # will disappear + # [1]: user has created config/tool_conf.xml and, having passed + # [0], probably moved their shed_tool_conf.xml as well + # [2]: user has done nothing, use the old files + # [3]: fresh install + tool_config_file=['config/tool_conf.xml,shed_tool_conf.xml', + 'config/tool_conf.xml,config/shed_tool_conf.xml', + 'tool_conf.xml,shed_tool_conf.xml', + 'config/tool_conf.xml.sample,config/shed_tool_conf.xml'] +) + + def resolve_path( path, root ): """If 'path' is relative make absolute by prepending 'root'""" if not os.path.isabs( path ): @@ -39,12 +79,34 @@ def resolve_path( path, root ): return path +def find_path(kwargs, var, root): + """Find a configuration path that may exist at different defaults.""" + defaults = PATH_DEFAULTS[var] + + if kwargs.get(var, None) is not None: + path = kwargs.get(var) + else: + for default in defaults: + if os.path.exists(resolve_path(default, root)): + path = default + break + else: + path = defaults[-1] + + return resolve_path(path, root) + + +def find_root(kwargs): + root = kwargs.get('root_dir', '.') + return root + + class Configuration( object ): deprecated_options = ( 'database_file', ) def __init__( self, **kwargs ): self.config_dict = kwargs - self.root = kwargs.get( 'root_dir', '.' ) + self.root = find_root(kwargs) # Resolve paths of other config files self.__parse_config_file_options( kwargs ) @@ -329,23 +391,22 @@ class Configuration( object ): self.tool_ngram_minsize = kwargs.get( "tool_ngram_minsize", 3 ) self.tool_ngram_maxsize = kwargs.get( "tool_ngram_maxsize", 4 ) # Location for tool dependencies. - # Location for tool dependencies. - tool_dependency_dir = kwargs.get( "tool_dependency_dir", "database/dependencies" ) - if tool_dependency_dir.lower() == "none": - tool_dependency_dir = None - - if tool_dependency_dir is not None: - self.tool_dependency_dir = resolve_path( tool_dependency_dir, self.root ) - # Setting the following flag to true will ultimately cause tool dependencies - # to be located in the shell environment and used by the job that is executing - # the tool. - self.use_tool_dependencies = True + use_tool_dependencies, tool_dependency_dir, use_cached_dependency_manager, tool_dependency_cache_dir, precache_dependencies = \ + parse_dependency_options(kwargs, self.root, self.dependency_resolvers_config_file) + self.use_tool_dependencies = use_tool_dependencies + self.tool_dependency_dir = tool_dependency_dir + self.use_cached_dependency_manager = use_cached_dependency_manager + self.tool_dependency_cache_dir = tool_dependency_cache_dir + self.precache_dependencies = precache_dependencies + # Deployers may either specify a complete list of mapping files or get the default for free and just + # specify a local mapping file to adapt and extend the default one. + if "conda_mapping_files" in kwargs: + self.conda_mapping_files = kwargs["conda_mapping_files"] else: - self.tool_dependency_dir = None - self.use_tool_dependencies = os.path.exists(self.dependency_resolvers_config_file) - self.use_cached_dependency_manager = string_as_bool(kwargs.get("use_cached_dependency_manager", 'False')) - self.tool_dependency_cache_dir = kwargs.get( 'tool_dependency_cache_dir', os.path.join(self.tool_dependency_dir, '_cache')) - self.precache_dependencies = string_as_bool(kwargs.get("precache_dependencies", 'True')) + self.conda_mapping_files = [ + self.local_conda_mapping_file, + os.path.join(self.root, "lib", "galaxy", "tools", "deps", "resolvers", "default_conda_mapping.yml"), + ] self.enable_beta_mulled_containers = string_as_bool( kwargs.get( 'enable_beta_mulled_containers', 'False' ) ) containers_resolvers_config_file = kwargs.get( 'containers_resolvers_config_file', None ) @@ -558,56 +619,11 @@ class Configuration( object ): """ Backwards compatibility for config files moved to the config/ dir. """ - defaults = dict( - auth_config_file=[ 'config/auth_conf.xml', 'config/auth_conf.xml.sample' ], - data_manager_config_file=[ 'config/data_manager_conf.xml', 'data_manager_conf.xml', 'config/data_manager_conf.xml.sample' ], - datatypes_config_file=[ 'config/datatypes_conf.xml', 'datatypes_conf.xml', 'config/datatypes_conf.xml.sample' ], - external_service_type_config_file=[ 'config/external_service_types_conf.xml', 'external_service_types_conf.xml', 'config/external_service_types_conf.xml.sample' ], - job_config_file=[ 'config/job_conf.xml', 'job_conf.xml' ], - tool_destinations_config_file=[ 'config/tool_destinations.yml', 'config/tool_destinations.yml.sample' ], - job_metrics_config_file=[ 'config/job_metrics_conf.xml', 'job_metrics_conf.xml', 'config/job_metrics_conf.xml.sample' ], - dependency_resolvers_config_file=[ 'config/dependency_resolvers_conf.xml', 'dependency_resolvers_conf.xml' ], - job_resource_params_file=[ 'config/job_resource_params_conf.xml', 'job_resource_params_conf.xml' ], - migrated_tools_config=[ 'migrated_tools_conf.xml', 'config/migrated_tools_conf.xml' ], - object_store_config_file=[ 'config/object_store_conf.xml', 'object_store_conf.xml' ], - openid_config_file=[ 'config/openid_conf.xml', 'openid_conf.xml', 'config/openid_conf.xml.sample' ], - shed_data_manager_config_file=[ 'shed_data_manager_conf.xml', 'config/shed_data_manager_conf.xml' ], - shed_tool_data_table_config=[ 'shed_tool_data_table_conf.xml', 'config/shed_tool_data_table_conf.xml' ], - tool_sheds_config_file=[ 'config/tool_sheds_conf.xml', 'tool_sheds_conf.xml', 'config/tool_sheds_conf.xml.sample' ], - workflow_schedulers_config_file=['config/workflow_schedulers_conf.xml', 'config/workflow_schedulers_conf.xml.sample'], - ) - listify_defaults = dict( - tool_data_table_config_path=[ 'config/tool_data_table_conf.xml', 'tool_data_table_conf.xml', 'config/tool_data_table_conf.xml.sample' ], - # rationale: - # [0]: user has explicitly created config/tool_conf.xml but did not - # move their existing shed_tool_conf.xml, don't use - # config/shed_tool_conf.xml, which is probably the empty - # version copied from the sample, or else their shed tools - # will disappear - # [1]: user has created config/tool_conf.xml and, having passed - # [0], probably moved their shed_tool_conf.xml as well - # [2]: user has done nothing, use the old files - # [3]: fresh install - tool_config_file=[ 'config/tool_conf.xml,shed_tool_conf.xml', - 'config/tool_conf.xml,config/shed_tool_conf.xml', - 'tool_conf.xml,shed_tool_conf.xml', - 'config/tool_conf.xml.sample,config/shed_tool_conf.xml' ] - ) + for var in PATH_DEFAULTS: + setattr( self, var, find_path( kwargs, var, self.root ) ) - for var, defaults in defaults.items(): - if kwargs.get( var, None ) is not None: - path = kwargs.get( var ) - else: - for default in defaults: - if os.path.exists( resolve_path( default, self.root ) ): - path = default - break - else: - path = defaults[-1] - setattr( self, var, resolve_path( path, self.root ) ) - - for var, defaults in listify_defaults.items(): + for var, defaults in PATH_LIST_DEFAULTS.items(): paths = [] if kwargs.get( var, None ) is not None: paths = listify( kwargs.get( var ) ) @@ -758,6 +774,31 @@ class Configuration( object ): return [ parse( v ) for v in allowed_origin_hostnames if v ] +def parse_dependency_options(kwargs, root, dependency_resolvers_config_file): + # Location for tool dependencies. + tool_dependency_dir = kwargs.get("tool_dependency_dir", "database/dependencies") + if tool_dependency_dir.lower() == "none": + tool_dependency_dir = None + + if tool_dependency_dir is not None: + tool_dependency_dir = resolve_path(tool_dependency_dir, root) + # Setting the following flag to true will ultimately cause tool dependencies + # to be located in the shell environment and used by the job that is executing + # the tool. + use_tool_dependencies = True + tool_dependency_cache_dir = kwargs.get('tool_dependency_cache_dir', os.path.join(tool_dependency_dir, '_cache')) + use_cached_dependency_manager = string_as_bool(kwargs.get("use_cached_dependency_manager", 'False')) + precache_dependencies = string_as_bool(kwargs.get("precache_dependencies", 'True')) + else: + tool_dependency_dir = None + use_tool_dependencies = os.path.exists(dependency_resolvers_config_file) + tool_dependency_cache_dir = None + precache_dependencies = False + use_cached_dependency_manager = False + + return use_tool_dependencies, tool_dependency_dir, use_cached_dependency_manager, tool_dependency_cache_dir, precache_dependencies + + def get_database_engine_options( kwargs, model_prefix='' ): """ Allow options for the SQLAlchemy database engine to be passed by using @@ -787,15 +828,21 @@ def get_database_engine_options( kwargs, model_prefix='' ): def configure_logging( config ): - """ - Allow some basic logging configuration to be read from ini file. + """Allow some basic logging configuration to be read from ini file. + + This should be able to consume either a galaxy.config.Configuration object + or a simple dictionary of configuration variables. """ # Get root logger root = logging.getLogger() # PasteScript will have already configured the logger if the # 'loggers' section was found in the config file, otherwise we do # some simple setup using the 'log_*' values from the config. - paste_configures_logging = config.global_conf_parser.has_section( "loggers" ) + parser = getattr(config, "global_conf_parser", None) + if parser: + paste_configures_logging = config.global_conf_parser.has_section( "loggers" ) + else: + paste_configures_logging = False auto_configure_logging = not paste_configures_logging and string_as_bool( config.get( "auto_configure_logging", "True" ) ) if auto_configure_logging: format = config.get( "log_format", "%(name)s %(levelname)s %(asctime)s %(message)s" ) @@ -826,7 +873,7 @@ def configure_logging( config ): handler.setFormatter( formatter ) root.addHandler( handler ) # If sentry is configured, also log to it - if config.sentry_dsn: + if getattr(config, "sentry_dsn", None): from raven.handlers.logging import SentryHandler sentry_handler = SentryHandler( config.sentry_dsn ) sentry_handler.setLevel( logging.WARN ) diff --git a/lib/galaxy/model/orm/scripts.py b/lib/galaxy/model/orm/scripts.py index d7fce87ecb2..a7a922341cd 100644 --- a/lib/galaxy/model/orm/scripts.py +++ b/lib/galaxy/model/orm/scripts.py @@ -2,9 +2,8 @@ Code to support database helper scripts (create_db.py, manage_db.py, etc...). """ import logging -import os.path -from galaxy.util.properties import load_app_properties +from galaxy.util.properties import find_config_file, load_app_properties log = logging.getLogger( __name__ ) @@ -40,25 +39,21 @@ DATABASE = { } -def read_config_file_arg( argv, default, old_default ): +def read_config_file_arg( argv, default, old_default, cwd=None ): + config_file = None if '-c' in argv: pos = argv.index( '-c' ) argv.pop(pos) config_file = argv.pop( pos ) - else: - if not os.path.exists( default ) and os.path.exists( old_default ): - config_file = old_default - elif os.path.exists( default ): - config_file = default - else: - config_file = default + ".sample" - return config_file + + return find_config_file( default, old_default, config_file, cwd=cwd ) def get_config( argv, cwd=None ): """ Read sys.argv and parse out repository of migrations and database url. + >>> import os >>> from ConfigParser import SafeConfigParser >>> from tempfile import mkdtemp >>> config_dir = mkdtemp() @@ -89,16 +84,11 @@ def get_config( argv, cwd=None ): default = database_defaults.get( 'config_file', DEFAULT_CONFIG_FILE ) old_default = database_defaults.get( 'old_config_file' ) - if cwd is not None: - default = os.path.join( cwd, default ) - old_default = os.path.join( cwd, old_default ) - config_file = read_config_file_arg( argv, default, old_default ) + config_file = read_config_file_arg( argv, default, old_default, cwd=cwd ) repo = database_defaults[ 'repo' ] config_prefix = database_defaults.get( 'config_prefix', DEFAULT_CONFIG_PREFIX ) config_override = database_defaults.get( 'config_override', 'GALAXY_CONFIG_' ) default_sqlite_file = database_defaults[ 'default_sqlite_file' ] - if cwd: - config_file = os.path.join( cwd, config_file ) properties = load_app_properties( ini_file=config_file, config_prefix=config_override ) diff --git a/lib/galaxy/tools/deps/__init__.py b/lib/galaxy/tools/deps/__init__.py index 6f254d3965a..b03f33db229 100644 --- a/lib/galaxy/tools/deps/__init__.py +++ b/lib/galaxy/tools/deps/__init__.py @@ -19,26 +19,12 @@ from .requirements import ( ToolRequirements ) from .resolvers import NullDependency -from .resolvers.conda import CondaDependencyResolver, DEFAULT_ENSURE_CHANNELS +from .resolvers.conda import CondaDependencyResolver from .resolvers.galaxy_packages import GalaxyPackageDependencyResolver from .resolvers.tool_shed_packages import ToolShedPackageDependencyResolver log = logging.getLogger( __name__ ) -# TODO: Load these from the plugins. Would require a two step initialization of -# DependencyManager - where the plugins are loaded first and then the config -# is parsed and sent through. -EXTRA_CONFIG_KWDS = { - 'conda_prefix': None, - 'conda_exec': None, - 'conda_debug': None, - 'conda_ensure_channels': DEFAULT_ENSURE_CHANNELS, - 'conda_auto_install': False, - 'conda_auto_init': False, - 'conda_copy_dependencies': False, - 'precache_dependencies': True, -} - CONFIG_VAL_NOT_FOUND = object() @@ -47,15 +33,9 @@ def build_dependency_manager( config ): dependency_manager_kwds = { 'default_base_path': config.tool_dependency_dir, 'conf_file': config.dependency_resolvers_config_file, + 'app_config': config, } - for key, default_value in EXTRA_CONFIG_KWDS.items(): - value = getattr(config, key, CONFIG_VAL_NOT_FOUND) - if value is CONFIG_VAL_NOT_FOUND and hasattr(config, "config_dict"): - value = config.config_dict.get(key, CONFIG_VAL_NOT_FOUND) - if value is CONFIG_VAL_NOT_FOUND: - value = default_value - dependency_manager_kwds[key] = value - if config.use_cached_dependency_manager: + if getattr(config, "use_cached_dependency_manager", False): dependency_manager_kwds['tool_dependency_cache_dir'] = config.tool_dependency_cache_dir dependency_manager = CachedDependencyManager(**dependency_manager_kwds) else: @@ -90,7 +70,7 @@ class DependencyManager( object ): and should each contain a file 'env.sh' which can be sourced to make the dependency available in the current shell environment. """ - def __init__( self, default_base_path, conf_file=None, **extra_config ): + def __init__( self, default_base_path, conf_file=None, app_config={} ): """ Create a new dependency manager looking for packages under the paths listed in `base_paths`. The default base path is app.config.tool_dependency_dir. @@ -99,11 +79,30 @@ class DependencyManager( object ): log.warning( "Path '%s' does not exist, ignoring", default_base_path ) if not os.path.isdir( default_base_path ): log.warning( "Path '%s' is not directory, ignoring", default_base_path ) - self.extra_config = extra_config + self.__app_config = app_config self.default_base_path = os.path.abspath( default_base_path ) self.resolver_classes = self.__resolvers_dict() self.dependency_resolvers = self.__build_dependency_resolvers( conf_file ) + def get_resolver_option(self, resolver, key, explicit_resolver_options={}): + """Look in resolver-specific settings for option and then fallback to global settings. + """ + default = resolver.config_options.get(key) + config_prefix = resolver.resolver_type + global_key = "%s_%s" % (config_prefix, key) + value = explicit_resolver_options.get(key, CONFIG_VAL_NOT_FOUND) + if value is CONFIG_VAL_NOT_FOUND: + if isinstance(self.__app_config, dict): + value = self.__app_config.get(global_key, CONFIG_VAL_NOT_FOUND) + else: + value = getattr(self.__app_config, global_key, CONFIG_VAL_NOT_FOUND) + if value is CONFIG_VAL_NOT_FOUND and hasattr(self.__app_config, "config_dict"): + value = self.__app_config.config_dict.get(global_key, CONFIG_VAL_NOT_FOUND) + if value is CONFIG_VAL_NOT_FOUND: + value = default + + return value + def dependency_shell_commands( self, requirements, **kwds ): requirement_to_dependency = self.requirements_to_dependencies(requirements, **kwds) return [dependency.shell_commands(requirement) for requirement, dependency in requirement_to_dependency.items()] @@ -141,6 +140,7 @@ class DependencyManager( object ): # Check requirements all at once all_unmet = len(requirement_to_dependency) == 0 if all_unmet and hasattr(resolver, "resolve_all"): + # TODO: Handle specs. dependencies = resolver.resolve_all(resolvable_requirements, **kwds) if dependencies: assert len(dependencies) == len(resolvable_requirements) @@ -155,7 +155,7 @@ class DependencyManager( object ): if requirement in requirement_to_dependency: continue - dependency = resolver.resolve( requirement.name, requirement.version, requirement.type, **kwds ) + dependency = resolver.resolve( requirement, **kwds ) if require_exact and not dependency.exact: continue @@ -210,8 +210,8 @@ class DependencyManager( object ): class CachedDependencyManager(DependencyManager): - def __init__(self, default_base_path, conf_file=None, **extra_config): - super(CachedDependencyManager, self).__init__(default_base_path=default_base_path, conf_file=conf_file, **extra_config) + def __init__(self, default_base_path, conf_file=None, app_config={}): + super(CachedDependencyManager, self).__init__(default_base_path=default_base_path, conf_file=conf_file, app_config=app_config) def build_cache(self, requirements, **kwds): resolved_dependencies = self.requirements_to_dependencies(requirements, **kwds) @@ -240,7 +240,7 @@ class CachedDependencyManager(DependencyManager): resolved_dependencies = self.requirements_to_dependencies(requirements, **kwds) cacheable_dependencies = [dep for dep in resolved_dependencies.values() if dep.cacheable] hashed_dependencies_dir = self.get_hashed_dependencies_path(cacheable_dependencies) - if not os.path.exists(hashed_dependencies_dir) and self.extra_config['precache_dependencies']: + if not os.path.exists(hashed_dependencies_dir) and self.__app_config.getattr("precache_dependencies", False): # Cache not present, try to create it self.build_cache(requirements, **kwds) if os.path.exists(hashed_dependencies_dir): diff --git a/lib/galaxy/tools/deps/requirements.py b/lib/galaxy/tools/deps/requirements.py index bb2cfaa612f..bdb0ae0451a 100644 --- a/lib/galaxy/tools/deps/requirements.py +++ b/lib/galaxy/tools/deps/requirements.py @@ -1,9 +1,12 @@ +import copy + from galaxy.util import ( asbool, xml_text, ) from galaxy.util.oset import OrderedSet + DEFAULT_REQUIREMENT_TYPE = "package" DEFAULT_REQUIREMENT_VERSION = None @@ -14,29 +17,69 @@ class ToolRequirement( object ): run (for example, a program, package, or library). Requirements can optionally assert a specific version. """ - def __init__( self, name=None, type=None, version=None ): + def __init__( self, name=None, type=None, version=None, specs=[] ): self.name = name self.type = type self.version = version + self.specs = specs def to_dict( self ): - return dict(name=self.name, type=self.type, version=self.version) + specs = [s.to_dict() for s in self.specs] + return dict(name=self.name, type=self.type, version=self.version, specs=specs) + + def copy( self ): + return copy.deepcopy( self ) @staticmethod def from_dict( dict ): version = dict.get( "version", None ) name = dict.get("name", None) type = dict.get("type", None) - return ToolRequirement( name=name, type=type, version=version ) + specs = [RequirementSpecification.from_dict(s) for s in dict.get("specs", [])] + return ToolRequirement( name=name, type=type, version=version, specs=specs ) def __eq__(self, other): - return self.name == other.name and self.type == other.type and self.version == other.version + return self.name == other.name and self.type == other.type and self.version == other.version and self.specs == other.specs def __ne__(self, other): return not self.__eq__(other) def __hash__(self): - return hash((self.name, self.type, self.version)) + return hash((self.name, self.type, self.version, frozenset(self.specs))) + + +class RequirementSpecification(object): + """Refine a requirement using a URI.""" + + def __init__(self, uri, version=None): + self.uri = uri + self.version = version + + @property + def specifies_version(self): + return self.version is not None + + @property + def short_name(self): + return self.uri.split("/")[-1] + + def to_dict(self): + return dict(uri=self.uri, version=self.version) + + @staticmethod + def from_dict(dict): + uri = dict.get["uri"] + version = dict.get("version", None) + return RequirementSpecification(uri=uri, version=version) + + def __eq__(self, other): + return self.uri == other.uri and self.version == other.version + + def __ne__(self, other): + return not self.__eq__(other) + + def __hash__(self): + return hash((self.uri, self.version)) class ToolRequirements(object): diff --git a/lib/galaxy/tools/deps/resolvers/__init__.py b/lib/galaxy/tools/deps/resolvers/__init__.py index c24c107887a..eb6fecbeaed 100644 --- a/lib/galaxy/tools/deps/resolvers/__init__.py +++ b/lib/galaxy/tools/deps/resolvers/__init__.py @@ -5,6 +5,9 @@ from abc import ( abstractproperty, ) +import yaml + +from galaxy.util import listify from galaxy.util.dictifiable import Dictifiable from ..requirements import ToolRequirement @@ -22,10 +25,11 @@ class DependencyResolver(Dictifiable, object): # resolution. disabled = False resolves_simple_dependencies = True + config_options = {} __metaclass__ = ABCMeta @abstractmethod - def resolve( self, name, version, type, **kwds ): + def resolve( self, requirement, **kwds ): """Given inputs describing dependency in the abstract yield a Dependency object. The Dependency object describes various attributes (script, bin, @@ -37,17 +41,14 @@ class DependencyResolver(Dictifiable, object): request version is 'default'.) """ - def _get_config_option(self, key, dependency_resolver, default=None, config_prefix=None, **kwds): - """ Look in resolver-specific settings for option and then fallback to - global settings. + +class MultipleDependencyResolver: + """Variant of DependencyResolver that can optionally resolve multiple dependencies together.""" + + @abstractmethod + def resolve_all( self, requirements, **kwds ): + """Given multiple requirements yield Dependency objects if and only if they may all be resolved together. """ - global_key = "%s_%s" % (config_prefix, key) - if key in kwds: - return kwds.get(key) - elif global_key in dependency_resolver.extra_config: - return dependency_resolver.extra_config.get(global_key) - else: - return default class ListableDependencyResolver: @@ -67,6 +68,116 @@ class ListableDependencyResolver: return ToolRequirement(name=name, type="package", version=version) +class MappableDependencyResolver: + """Mix this into a ``DependencyResolver`` to allow mapping files. + + Mapping files allow adapting generic requirements to specific local implementations. + """ + + def _setup_mapping(self, dependency_manager, **kwds): + mapping_files = dependency_manager.get_resolver_option(self, "mapping_files", explicit_resolver_options=kwds) + mappings = [] + if mapping_files: + mapping_files = listify(mapping_files) + for mapping_file in mapping_files: + mappings.extend(MappableDependencyResolver._mapping_file_to_list(mapping_file)) + self._mappings = mappings + + @staticmethod + def _mapping_file_to_list(mapping_file): + with open(mapping_file, "r") as f: + raw_mapping = yaml.load(f) or [] + return map(RequirementMapping.from_dict, raw_mapping) + + def _expand_mappings(self, requirement): + for mapping in self._mappings: + if requirement.name == mapping.from_name: + if mapping.from_version is not None and mapping.from_version != requirement.version: + continue + + requirement = requirement.copy() + requirement.name = mapping.to_name + if mapping.to_version is not None: + requirement.version = mapping.to_version + + break + + return requirement + + +class RequirementMapping(object): + + def __init__(self, from_name, from_version, to_name, to_version): + self.from_name = from_name + self.from_version = from_version + self.to_name = to_name + self.to_version = to_version + + @staticmethod + def from_dict(raw_mapping): + from_raw = raw_mapping.get("from") + if isinstance(from_raw, dict): + from_name = from_raw.get("name") + from_version = str(from_raw.get("version")) + else: + from_name = from_raw + from_version = None + + to_raw = raw_mapping.get("to") + if isinstance(to_raw, dict): + to_name = to_raw.get("name", from_name) + to_version = str(to_raw.get("version")) + else: + to_name = to_raw + to_version = None + + return RequirementMapping(from_name, from_version, to_name, to_version) + + +class SpecificationAwareDependencyResolver: + """Mix this into a :class:`DependencyResolver` to implement URI specification matching. + + Allows adapting generic requirements to more specific URIs - to tailor name + or version to specified resolution system. + """ + __metaclass__ = ABCMeta + + @abstractmethod + def _expand_specs(self, requirement): + """Find closest matching specification for discovered resolver and return new concrete requirement.""" + + +class SpecificationPatternDependencyResolver: + """Implement the :class:`SpecificationAwareDependencyResolver` with a regex pattern.""" + + @abstractproperty + def _specification_pattern(self): + """Pattern of URI to match against.""" + + def _find_specification(self, specs): + pattern = self._specification_pattern + for spec in specs: + if pattern.match(spec.uri): + return spec + return None + + def _expand_specs(self, requirement): + name = requirement.name + version = requirement.version + specs = requirement.specs + + spec = self._find_specification(specs) + if spec is not None: + name = spec.short_name + version = spec.version or version + + requirement = requirement.copy() + requirement.name = name + requirement.version = version + + return requirement + + class InstallableDependencyResolver: """ Mix this into a ``DependencyResolver`` and implement to indicate the dependency resolver can attempt to install new dependencies. diff --git a/lib/galaxy/tools/deps/resolvers/brewed_tool_shed_packages.py b/lib/galaxy/tools/deps/resolvers/brewed_tool_shed_packages.py index 5585de56c42..a6b4cde4520 100644 --- a/lib/galaxy/tools/deps/resolvers/brewed_tool_shed_packages.py +++ b/lib/galaxy/tools/deps/resolvers/brewed_tool_shed_packages.py @@ -29,7 +29,8 @@ class HomebrewToolShedDependencyResolver( self._init_homebrew(**kwds) self._init_base_path(dependency_manager, **kwds) - def resolve(self, name, version, type, **kwds): + def resolve(self, requirement, **kwds): + name, version, type = requirement.name, requirement.version, requirement.type if type != "package": return NullDependency(version=version, name=name) if version is None: diff --git a/lib/galaxy/tools/deps/resolvers/conda.py b/lib/galaxy/tools/deps/resolvers/conda.py index f6fe1d3925f..87a5ec14f53 100644 --- a/lib/galaxy/tools/deps/resolvers/conda.py +++ b/lib/galaxy/tools/deps/resolvers/conda.py @@ -5,6 +5,7 @@ incompatible changes coming. import logging import os +import re import galaxy.tools.deps.installable @@ -28,7 +29,10 @@ from ..resolvers import ( DependencyResolver, InstallableDependencyResolver, ListableDependencyResolver, + MappableDependencyResolver, + MultipleDependencyResolver, NullDependency, + SpecificationPatternDependencyResolver, ) @@ -39,16 +43,27 @@ DEFAULT_ENSURE_CHANNELS = "iuc,bioconda,r,defaults,conda-forge" log = logging.getLogger(__name__) -class CondaDependencyResolver(DependencyResolver, ListableDependencyResolver, InstallableDependencyResolver): +class CondaDependencyResolver(DependencyResolver, MultipleDependencyResolver, ListableDependencyResolver, InstallableDependencyResolver, SpecificationPatternDependencyResolver, MappableDependencyResolver): dict_collection_visible_keys = DependencyResolver.dict_collection_visible_keys + ['conda_prefix', 'versionless', 'ensure_channels', 'auto_install'] resolver_type = "conda" + config_options = { + 'prefix': None, + 'exec': None, + 'debug': None, + 'ensure_channels': DEFAULT_ENSURE_CHANNELS, + 'auto_install': False, + 'auto_init': True, + 'copy_dependencies': False, + } + _specification_pattern = re.compile(r"https\:\/\/anaconda.org\/\w+\/\w+") def __init__(self, dependency_manager, **kwds): + self._setup_mapping(dependency_manager, **kwds) self.versionless = _string_as_bool(kwds.get('versionless', 'false')) self.dependency_manager = dependency_manager def get_option(name): - return self._get_config_option(name, dependency_manager, config_prefix="conda", **kwds) + return dependency_manager.get_resolver_option(self, name, explicit_resolver_options=kwds) # Conda context options (these define the environment) conda_prefix = get_option("prefix") @@ -133,6 +148,8 @@ class CondaDependencyResolver(DependencyResolver, ListableDependencyResolver, In conda_targets = [] for requirement in requirements: + requirement = self._expand_requirement(requirement) + version = requirement.version if self.versionless: version = None @@ -170,7 +187,10 @@ class CondaDependencyResolver(DependencyResolver, ListableDependencyResolver, In assert len(conda_targets) == 1 return conda_targets[0].install_environment - def resolve(self, name, version, type, **kwds): + def resolve(self, requirement, **kwds): + requirement = self._expand_requirement(requirement) + name, version, type = requirement.name, requirement.version, requirement.type + # Check for conda just not being there, this way we can enable # conda by default and just do nothing in not configured. if not os.path.isdir(self.conda_context.conda_prefix): @@ -218,6 +238,9 @@ class CondaDependencyResolver(DependencyResolver, ListableDependencyResolver, In preserve_python_environment=preserve_python_environment, ) + def _expand_requirement(self, requirement): + return self._expand_specs(self._expand_mappings(requirement)) + def list_dependencies(self): for install_target in installed_conda_targets(self.conda_context): name = install_target.package diff --git a/lib/galaxy/tools/deps/resolvers/default_conda_mapping.yml b/lib/galaxy/tools/deps/resolvers/default_conda_mapping.yml new file mode 100644 index 00000000000..9b1e59e7a6b --- /dev/null +++ b/lib/galaxy/tools/deps/resolvers/default_conda_mapping.yml @@ -0,0 +1,5 @@ +- from: R + to: r-base +- from: blast+ + to: blast + \ No newline at end of file diff --git a/lib/galaxy/tools/deps/resolvers/galaxy_packages.py b/lib/galaxy/tools/deps/resolvers/galaxy_packages.py index 858ff0faf98..8e1a1145030 100644 --- a/lib/galaxy/tools/deps/resolvers/galaxy_packages.py +++ b/lib/galaxy/tools/deps/resolvers/galaxy_packages.py @@ -16,6 +16,7 @@ from ..resolvers import ( Dependency, DependencyResolver, ListableDependencyResolver, + MappableDependencyResolver, NullDependency, ) @@ -66,11 +67,13 @@ class BaseGalaxyPackageDependencyResolver(DependencyResolver, UsesToolDependency self.versionless = str(kwds.get('versionless', "false")).lower() == "true" self._init_base_path( dependency_manager, **kwds ) - def resolve( self, name, version, type, **kwds ): + def resolve(self, requirement, **kwds): """ Attempt to find a dependency named `name` at version `version`. If version is None, return the "default" version as determined using a symbolic link (if found). Returns a triple of: env_script, base_path, real_version """ + name, version, type = requirement.name, requirement.version, requirement.type + if version is None or self.versionless: exact = not self.versionless or version is None return self._find_dep_default( name, type=type, exact=exact, **kwds ) @@ -101,9 +104,17 @@ class BaseGalaxyPackageDependencyResolver(DependencyResolver, UsesToolDependency return NullDependency(version=version, name=name) -class GalaxyPackageDependencyResolver(BaseGalaxyPackageDependencyResolver, ListableDependencyResolver): +class GalaxyPackageDependencyResolver(BaseGalaxyPackageDependencyResolver, ListableDependencyResolver, MappableDependencyResolver): resolver_type = "galaxy_packages" + def __init__(self, dependency_manager, **kwds): + super(GalaxyPackageDependencyResolver, self).__init__(dependency_manager, **kwds) + self._setup_mapping(dependency_manager, **kwds) + + def resolve(self, requirement, **kwds): + requirement = self._expand_mappings(requirement) + return super(GalaxyPackageDependencyResolver, self).resolve(requirement, **kwds) + def list_dependencies(self): base_path = self.base_path for package_name in listdir(base_path): diff --git a/lib/galaxy/tools/deps/resolvers/homebrew.py b/lib/galaxy/tools/deps/resolvers/homebrew.py index 395eb1eb518..2a9781d3904 100644 --- a/lib/galaxy/tools/deps/resolvers/homebrew.py +++ b/lib/galaxy/tools/deps/resolvers/homebrew.py @@ -39,7 +39,9 @@ class HomebrewDependencyResolver(DependencyResolver, UsesHomebrewMixin): self._init_homebrew(**kwds) - def resolve(self, name, version, type, **kwds): + def resolve(self, requirement, **kwds): + name, version, type = requirement.name, requirement.version, requirement.type + if type != "package": return NullDependency(version=version, name=name) diff --git a/lib/galaxy/tools/deps/resolvers/modules.py b/lib/galaxy/tools/deps/resolvers/modules.py index 227abb72a9c..0cbbc5764f8 100644 --- a/lib/galaxy/tools/deps/resolvers/modules.py +++ b/lib/galaxy/tools/deps/resolvers/modules.py @@ -13,7 +13,12 @@ from subprocess import PIPE, Popen from six import StringIO -from ..resolvers import Dependency, DependencyResolver, NullDependency +from ..resolvers import ( + Dependency, + DependencyResolver, + MappableDependencyResolver, + NullDependency, +) log = logging.getLogger( __name__ ) @@ -24,11 +29,12 @@ DEFAULT_MODULE_PREFETCH = "true" UNKNOWN_FIND_BY_MESSAGE = "ModuleDependencyResolver does not know how to find modules by [%s], find_by should be one of %s" -class ModuleDependencyResolver(DependencyResolver): +class ModuleDependencyResolver(DependencyResolver, MappableDependencyResolver): dict_collection_visible_keys = DependencyResolver.dict_collection_visible_keys + ['base_path', 'modulepath'] resolver_type = "modules" def __init__(self, dependency_manager, **kwds): + self._setup_mapping(dependency_manager, **kwds) self.versionless = _string_as_bool(kwds.get('versionless', 'false')) find_by = kwds.get('find_by', 'avail') prefetch = _string_as_bool(kwds.get('prefetch', DEFAULT_MODULE_PREFETCH)) @@ -51,7 +57,10 @@ class ModuleDependencyResolver(DependencyResolver): module_path = DEFAULT_MODULE_PATH return module_path - def resolve( self, name, version, type, **kwds ): + def resolve(self, requirement, **kwds): + requirement = self._expand_mappings(requirement) + name, version, type = requirement.name, requirement.version, requirement.type + if type != "package": return NullDependency(version=version, name=name) diff --git a/lib/galaxy/util/properties.py b/lib/galaxy/util/properties.py index f2de11b3ae2..b5743dc0ef2 100644 --- a/lib/galaxy/util/properties.py +++ b/lib/galaxy/util/properties.py @@ -10,6 +10,28 @@ from six import iteritems from six.moves.configparser import ConfigParser +def find_config_file(default, old_default, explicit, cwd=None): + if cwd is not None: + default = os.path.join(cwd, default) + old_default = os.path.join(cwd, old_default) + if explicit is not None: + explicit = os.path.join(cwd, explicit) + + if explicit: + if os.path.exists(explicit): + config_file = explicit + else: + raise Exception("Problem determining Galaxy's configuration - the specified configuration file cannot be found.") + else: + if not os.path.exists( default ) and os.path.exists( old_default ): + config_file = old_default + elif os.path.exists( default ): + config_file = default + else: + config_file = default + ".sample" + return config_file + + def load_app_properties( kwds={}, ini_file=None, @@ -97,4 +119,4 @@ class NicerConfigParser(ConfigParser): raise -__all__ = ('load_app_properties', 'NicerConfigParser') +__all__ = ('find_config_file', 'load_app_properties', 'NicerConfigParser') diff --git a/run.sh b/run.sh index 039047a87b8..9d7dbdfb211 100755 --- a/run.sh +++ b/run.sh @@ -14,6 +14,7 @@ then . $GALAXY_LOCAL_ENV_FILE fi +INITIALIZE_TOOL_DEPENDENCIES=1 # Install Conda if needed. # Pop args meant for common_startup.sh while : do @@ -22,6 +23,10 @@ do common_startup_args="$common_startup_args $1" shift ;; + --skip-tool-dependency-initialization) + INITIALIZE_TOOL_DEPENDENCIES=0 + shift + ;; --skip-venv) skip_venv=1 common_startup_args="$common_startup_args $1" @@ -98,6 +103,11 @@ if [ -z "$GALAXY_CONFIG_FILE" ]; then export GALAXY_CONFIG_FILE fi +if [ $INITIALIZE_TOOL_DEPENDENCIES -eq 1 ]; then + # Install Conda environment if needed. + python ./scripts/manage_tool_dependencies.py -c "$GALAXY_CONFIG_FILE" init_if_needed +fi + if [ -n "$GALAXY_RUN_ALL" ]; then servers=$(sed -n 's/^\[server:\(.*\)\]/\1/ p' "$GALAXY_CONFIG_FILE" | xargs echo) if [ -z "$stop_daemon_arg_set" -a -z "$daemon_or_restart_arg_set" ]; then diff --git a/scripts/manage_tool_dependencies.py b/scripts/manage_tool_dependencies.py new file mode 100644 index 00000000000..07b5b8802ca --- /dev/null +++ b/scripts/manage_tool_dependencies.py @@ -0,0 +1,87 @@ +import argparse +import os.path +import sys + +sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, 'lib'))) + +from galaxy.util.properties import find_config_file, load_app_properties +from galaxy.config import ( + configure_logging, + find_path, + find_root, + parse_dependency_options, +) +from galaxy.tools.deps import CachedDependencyManager, DependencyManager, NullDependencyManager + +DESCRIPTION = "Script to manage tool dependencies (with focus on a Conda environments)." + + +def main(argv=None): + """Entry point for conversion process.""" + if argv is None: + argv = sys.argv[1:] + args = _arg_parser().parse_args(argv) + action = args.action + action_func = ACTIONS[action] + action_func(args) + + +def _init_if_needed(args): + kwargs = _app_properties(args) + + # If conda_auto_init is set, simply building the Conda resolver will call handle installation. + _build_dependency_manager_no_config(kwargs) + + +def _app_properties(args): + config_file = find_config_file("config/galaxy.ini", "universe_wsgi.ini", args.config_file) + app_properties = load_app_properties(ini_file=config_file) + return app_properties + + +def _arg_parser(): + parser = argparse.ArgumentParser(description=DESCRIPTION) + parser.add_argument('action', metavar='ACTION', type=str, + choices=ACTIONS.keys(), + help='action to perform') + parser.add_argument("-c", "--config-file", default=None) + return parser + + +def _build_dependency_manager_no_config(kwargs): + """Simplified variant of build_dependency_manager from galaxy.tools.deps. + + The canonical factory method requires a full Galaxy configuration object + which we do not have available in this script (an optimization). + """ + configure_logging(kwargs) + root = find_root(kwargs) + dependency_resolvers_config_file = find_path(kwargs, "dependency_resolvers_config_file", root) + use_dependencies, tool_dependency_dir, use_cached_dependency_manager, tool_dependency_cache_dir, precache_dependencies = \ + parse_dependency_options(kwargs, root, dependency_resolvers_config_file) + + if not use_dependencies: + dependency_manager = NullDependencyManager() + else: + dependency_manager_kwds = { + 'default_base_path': tool_dependency_dir, + 'conf_file': dependency_resolvers_config_file, + 'app_config': kwargs, + } + + if use_cached_dependency_manager: + dependency_manager_kwds['tool_dependency_cache_dir'] = tool_dependency_cache_dir + dependency_manager = CachedDependencyManager(**dependency_manager_kwds) + else: + dependency_manager = DependencyManager( **dependency_manager_kwds ) + + return dependency_manager + + +ACTIONS = { + "init_if_needed": _init_if_needed, +} + + +if __name__ == '__main__': + main() diff --git a/test/functional/tools/legacy_r.xml b/test/functional/tools/legacy_r.xml new file mode 100644 index 00000000000..5d70feede9f --- /dev/null +++ b/test/functional/tools/legacy_r.xml @@ -0,0 +1,14 @@ + + $out_file1 + ]]> + + R + + + + + + + + diff --git a/test/functional/tools/samples_tool_conf.xml b/test/functional/tools/samples_tool_conf.xml index d678c8be35d..02677a581bd 100644 --- a/test/functional/tools/samples_tool_conf.xml +++ b/test/functional/tools/samples_tool_conf.xml @@ -124,6 +124,8 @@ + + diff --git a/test/integration/test_resolvers.py b/test/integration/test_resolvers.py index 5c36fd30fbe..a676a299ca3 100644 --- a/test/integration/test_resolvers.py +++ b/test/integration/test_resolvers.py @@ -4,6 +4,9 @@ import shutil from tempfile import mkdtemp from base import integration_util +from base.populators import ( + DatasetPopulator, +) GNUPLOT = {u'version': u'4.6', u'type': u'package', u'name': u'gnuplot'} @@ -71,6 +74,20 @@ class CondaResolutionIntegrationTestCase(integration_util.IntegrationTestCase): response = create_response.json() self._assert_dependency_type(response) + def test_legacy_r_mapping( self ): + """ + """ + dataset_populator = DatasetPopulator(self.galaxy_interactor) + history_id = dataset_populator.new_history() + payload = dataset_populator.run_tool_payload( + tool_id="legacy_R", + inputs={}, + history_id=history_id, + ) + create_response = self._post( "tools", data=payload ) + self._assert_status_code_is( create_response, 200 ) + dataset_populator.wait_for_history( history_id, assert_ok=True ) + def test_dependency_status_installed_not_exact( self ): """ GET request to dependency_resolvers/dependency with GNUPLOT dependency. diff --git a/test/unit/tools/test_tool_deps.py b/test/unit/tools/test_tool_deps.py index 2f4bda30fcb..1c18eedf29e 100644 --- a/test/unit/tools/test_tool_deps.py +++ b/test/unit/tools/test_tool_deps.py @@ -40,7 +40,7 @@ def test_tool_dependencies(): if sub == "env.sh": __touch( os.path.join( p, "env.sh" ) ) - dm = DependencyManager( default_base_path=base_path ) + dm = __dependency_manager_for_base_path( default_base_path=base_path ) dependency = dm.find_dep( "dep1", "1.0" ) assert dependency.script == os.path.join( base_path, 'dep1', '1.0', 'env.sh' ) assert dependency.path == os.path.join( base_path, 'dep1', '1.0' ) @@ -70,7 +70,7 @@ TEST_VERSION = "0.5.9" def test_toolshed_set_enviornment_requiremetns(): with __test_base_path() as base_path: test_repo = __build_test_repo('set_environment') - dm = DependencyManager( default_base_path=base_path ) + dm = __dependency_manager_for_base_path( default_base_path=base_path ) env_settings_dir = os.path.join(base_path, "environment_settings", TEST_REPO_NAME, TEST_REPO_USER, TEST_REPO_NAME, TEST_REPO_CHANGESET) os.makedirs(env_settings_dir) dependency = dm.find_dep( TEST_REPO_NAME, version=None, type='set_environment', installed_tool_dependencies=[test_repo] ) @@ -81,7 +81,7 @@ def test_toolshed_set_enviornment_requiremetns(): def test_toolshed_package_requirements(): with __test_base_path() as base_path: test_repo = __build_test_repo('package', version=TEST_VERSION) - dm = DependencyManager( default_base_path=base_path ) + dm = __dependency_manager_for_base_path( default_base_path=base_path ) package_dir = __build_ts_test_package(base_path) dependency = dm.find_dep( TEST_REPO_NAME, version=TEST_VERSION, type='package', installed_tool_dependencies=[test_repo] ) assert dependency.version == TEST_VERSION @@ -90,7 +90,7 @@ def test_toolshed_package_requirements(): def test_toolshed_tools_fallback_on_manual_dependencies(): with __test_base_path() as base_path: - dm = DependencyManager( default_base_path=base_path ) + dm = __dependency_manager_for_base_path( default_base_path=base_path ) test_repo = __build_test_repo('package', version=TEST_VERSION) env_path = __setup_galaxy_package_dep(base_path, "dep1", "1.0") dependency = dm.find_dep( "dep1", version="1.0", type='package', installed_tool_dependencies=[test_repo] ) @@ -100,7 +100,7 @@ def test_toolshed_tools_fallback_on_manual_dependencies(): def test_toolshed_greater_precendence(): with __test_base_path() as base_path: - dm = DependencyManager( default_base_path=base_path ) + dm = __dependency_manager_for_base_path( default_base_path=base_path ) test_repo = __build_test_repo('package', version=TEST_VERSION) ts_package_dir = __build_ts_test_package(base_path) gx_env_path = __setup_galaxy_package_dep(base_path, TEST_REPO_NAME, TEST_VERSION) @@ -142,13 +142,7 @@ def test_tool_requirements(): def test_module_dependency_resolver(): with __test_base_path() as temp_directory: - module_script = os.path.join(temp_directory, "modulecmd") - __write_script(module_script, '''#!/bin/sh -cat %s/example_output 1>&2; -''' % temp_directory) - with open(os.path.join(temp_directory, "example_output"), "w") as f: - # Subset of module avail from MSI cluster. - f.write(''' + module_script = _setup_module_command(temp_directory, ''' -------------------------- /soft/modules/modulefiles --------------------------- JAGS/3.2.0-gcc45 JAGS/3.3.0-gcc4.7.2 @@ -168,19 +162,87 @@ advisor/2013/update1 intel/11.1.075 mkl/10.2.1.017 advisor/2013/update2 intel/11.1.080 mkl/10.2.5.035 advisor/2013/update3 intel/12.0 mkl/10.2.7.041 ''') - resolver = ModuleDependencyResolver(None, modulecmd=module_script) - module = resolver.resolve( name="R", version=None, type="package" ) + resolver = ModuleDependencyResolver(_SimpleDependencyManager(), modulecmd=module_script) + module = resolver.resolve( ToolRequirement( name="R", version=None, type="package" ) ) assert module.module_name == "R" assert module.module_version is None - module = resolver.resolve( name="R", version="3.0.1", type="package" ) + module = resolver.resolve( ToolRequirement( name="R", version="3.0.1", type="package" ) ) assert module.module_name == "R" assert module.module_version == "3.0.1" - module = resolver.resolve( name="R", version="3.0.4", type="package" ) + module = resolver.resolve( ToolRequirement( name="R", version="3.0.4", type="package" ) ) assert isinstance(module, NullDependency) +def test_module_resolver_with_mapping(): + with __test_base_path() as temp_directory: + module_script = _setup_module_command(temp_directory, ''' +-------------------------- /soft/modules/modulefiles --------------------------- +blast/2.24 +''') + mapping_file = os.path.join(temp_directory, "mapping") + with open(mapping_file, "w") as f: + f.write(''' +- from: blast+ + to: blast +''') + + resolver = ModuleDependencyResolver(_SimpleDependencyManager(), modulecmd=module_script, mapping_files=mapping_file) + module = resolver.resolve( ToolRequirement( name="blast+", version="2.24", type="package" ) ) + assert module.module_name == "blast" + assert module.module_version == "2.24", module.module_version + + +def test_module_resolver_with_mapping_versions(): + with __test_base_path() as temp_directory: + module_script = _setup_module_command(temp_directory, ''' +-------------------------- /soft/modules/modulefiles --------------------------- +blast/2.22.0-mpi +blast/2.23 +blast/2.24.0-mpi +''') + mapping_file = os.path.join(temp_directory, "mapping") + with open(mapping_file, "w") as f: + f.write(''' +- from: + name: blast+ + version: 2.24 + to: + name: blast + version: 2.24.0-mpi +- from: + name: blast + version: 2.22 + to: + version: 2.22.0-mpi +''') + + resolver = ModuleDependencyResolver(_SimpleDependencyManager(), modulecmd=module_script, mapping_files=mapping_file) + module = resolver.resolve( ToolRequirement( name="blast+", version="2.24", type="package" ) ) + assert module.module_name == "blast" + assert module.module_version == "2.24.0-mpi", module.module_version + + resolver = ModuleDependencyResolver(_SimpleDependencyManager(), modulecmd=module_script, mapping_files=mapping_file) + module = resolver.resolve( ToolRequirement( name="blast+", version="2.23", type="package" ) ) + assert isinstance(module, NullDependency) + + module = resolver.resolve( ToolRequirement( name="blast", version="2.22", type="package" ) ) + assert module.module_name == "blast" + assert module.module_version == "2.22.0-mpi", module.module_version + + +def _setup_module_command(temp_directory, contents): + module_script = os.path.join(temp_directory, "modulecmd") + __write_script(module_script, '''#!/bin/sh +cat %s/example_output 1>&2; +''' % temp_directory) + with open(os.path.join(temp_directory, "example_output"), "w") as f: + # Subset of module avail from MSI cluster. + f.write(contents) + return module_script + + def test_module_dependency(): with __test_base_path() as temp_directory: # Create mock modulecmd script that just exports a variable @@ -219,7 +281,7 @@ def test_shell_commands_built(): # Test that dependency manager builds valid shell commands for a list of # requirements. with __test_base_path() as base_path: - dm = DependencyManager( default_base_path=base_path ) + dm = __dependency_manager_for_base_path( default_base_path=base_path ) __setup_galaxy_package_dep( base_path, TEST_REPO_NAME, TEST_VERSION, contents="export FOO=\"bar\"" ) mock_requirements = ToolRequirements([{'type': 'package', 'version': TEST_VERSION, 'name': TEST_REPO_NAME}]) commands = dm.dependency_shell_commands( mock_requirements ) @@ -411,5 +473,15 @@ def __dependency_manager(xml_content): f = tempfile.NamedTemporaryFile() f.write(xml_content) f.flush() - dm = DependencyManager( default_base_path=base_path, conf_file=f.name ) + dm = __dependency_manager_for_base_path( default_base_path=base_path, conf_file=f.name ) yield dm + + +def __dependency_manager_for_base_path(default_base_path, conf_file=None ): + return DependencyManager( default_base_path=default_base_path, conf_file=conf_file, app_config={"conda_auto_init": False} ) + + +class _SimpleDependencyManager(object): + + def get_resolver_option(self, resolver, key, explicit_resolver_options={}): + return explicit_resolver_options.get(key)