diff --git a/lib/galaxy/config.py b/lib/galaxy/config.py index f81a2293e95..c5db70f5fbc 100644 --- a/lib/galaxy/config.py +++ b/lib/galaxy/config.py @@ -501,24 +501,16 @@ class Configuration(object): self.tool_ngram_maxsize = kwargs.get("tool_ngram_maxsize", 4) default_tool_test_data_directories = os.environ.get("GALAXY_TEST_FILE_DIR", resolve_path("test-data", self.root)) self.tool_test_data_directories = kwargs.get("tool_test_data_directories", default_tool_test_data_directories) - # Location for tool dependencies. - use_tool_dependencies, tool_dependency_dir, use_cached_dependency_manager, tool_dependency_cache_dir, precache_dependencies = \ - parse_dependency_options(kwargs, self.root, self.dependency_resolvers_config_file) - self.use_tool_dependencies = use_tool_dependencies - self.tool_dependency_dir = tool_dependency_dir - self.use_cached_dependency_manager = use_cached_dependency_manager - self.tool_dependency_cache_dir = tool_dependency_cache_dir - self.precache_dependencies = precache_dependencies - self.dependency_resolvers = kwargs.get("dependency_resolvers") # Deployers may either specify a complete list of mapping files or get the default for free and just # specify a local mapping file to adapt and extend the default one. - if "conda_mapping_files" in kwargs: - self.conda_mapping_files = kwargs["conda_mapping_files"] - else: - self.conda_mapping_files = [ + if "conda_mapping_files" not in kwargs: + conda_mapping_files = [ self.local_conda_mapping_file, os.path.join(self.root, "lib", "galaxy", "tool_util", "deps", "resolvers", "default_conda_mapping.yml"), ] + # dependency resolution options are consumed via config_dict - so don't populate + # self, populate config_dict + self.config_dict["conda_mapping_files"] = conda_mapping_files self.enable_beta_mulled_containers = string_as_bool(kwargs.get('enable_beta_mulled_containers', 'False')) containers_resolvers_config_file = kwargs.get('containers_resolvers_config_file', None) @@ -528,7 +520,10 @@ class Configuration(object): involucro_path = kwargs.get('involucro_path', None) if involucro_path is None: - involucro_path = os.path.join(tool_dependency_dir or "database", "involucro") + target_dir = kwargs.get("tool_dependency_dir", "database/dependencies") + if target_dir == "none": + target_dir = "database" + involucro_path = os.path.join(target_dir, "involucro") self.involucro_path = resolve_path(involucro_path, self.root) self.involucro_auto_init = string_as_bool(kwargs.get('involucro_auto_init', True)) mulled_channels = kwargs.get('mulled_channels') @@ -921,34 +916,6 @@ class Configuration(object): return [parse(v) for v in allowed_origin_hostnames if v] -def parse_dependency_options(kwargs, root, dependency_resolvers_config_file): - # Location for tool dependencies. - tool_dependency_dir = kwargs.get("tool_dependency_dir", "database/dependencies") - if tool_dependency_dir.lower() == "none": - tool_dependency_dir = None - - if tool_dependency_dir is not None: - tool_dependency_dir = resolve_path(tool_dependency_dir, root) - # Setting the following flag to true will ultimately cause tool dependencies - # to be located in the shell environment and used by the job that is executing - # the tool. - use_tool_dependencies = True - tool_dependency_cache_dir = kwargs.get('tool_dependency_cache_dir', os.path.join(tool_dependency_dir, '_cache')) - use_cached_dependency_manager = string_as_bool(kwargs.get("use_cached_dependency_manager", 'False')) - precache_dependencies = string_as_bool(kwargs.get("precache_dependencies", 'True')) - else: - tool_dependency_dir = None - if dependency_resolvers_config_file is None: - use_tool_dependencies = bool(kwargs.get("dependency_resolvers", None)) - else: - use_tool_dependencies = os.path.exists(dependency_resolvers_config_file) - tool_dependency_cache_dir = None - precache_dependencies = False - use_cached_dependency_manager = False - - return use_tool_dependencies, tool_dependency_dir, use_cached_dependency_manager, tool_dependency_cache_dir, precache_dependencies - - def get_database_engine_options(kwargs, model_prefix=''): """ Allow options for the SQLAlchemy database engine to be passed by using @@ -1225,3 +1192,7 @@ class ConfiguresGalaxyMixin(object): except Exception: log.info("Waiting for database: attempt %d of %d" % (i, attempts)) time.sleep(pause) + + @property + def tool_dependency_dir(self): + return self.toolbox.dependency_manager.default_base_path diff --git a/lib/galaxy/model/tool_shed_install/__init__.py b/lib/galaxy/model/tool_shed_install/__init__.py index 4aef91b43b7..430642f60dc 100644 --- a/lib/galaxy/model/tool_shed_install/__init__.py +++ b/lib/galaxy/model/tool_shed_install/__init__.py @@ -555,14 +555,14 @@ class ToolDependency(object): def installation_directory(self, app): if self.type == 'package': - return os.path.join(app.config.tool_dependency_dir, + return os.path.join(app.tool_dependency_dir, self.name, self.version, self.tool_shed_repository.owner, self.tool_shed_repository.name, self.tool_shed_repository.installed_changeset_revision) if self.type == 'set_environment': - return os.path.join(app.config.tool_dependency_dir, + return os.path.join(app.tool_dependency_dir, 'environment_settings', self.name, self.tool_shed_repository.owner, diff --git a/lib/galaxy/tool_util/deps/__init__.py b/lib/galaxy/tool_util/deps/__init__.py index 397a9a12faf..61e7f46a1c7 100644 --- a/lib/galaxy/tool_util/deps/__init__.py +++ b/lib/galaxy/tool_util/deps/__init__.py @@ -10,7 +10,8 @@ from collections import OrderedDict from galaxy.util import ( hash_util, - plugin_config + plugin_config, + string_as_bool, ) from galaxy.util.oset import OrderedSet from .container_resolvers import ContainerResolver @@ -31,27 +32,59 @@ log = logging.getLogger(__name__) CONFIG_VAL_NOT_FOUND = object() -def build_dependency_manager(config): - config_dict = { - "use_tool_dependencies": getattr(config, "use_tool_dependencies", False), - "default_base_path": getattr(config, "tool_dependency_dir", None), - "conf_file": getattr(config, "dependency_resolvers_config_file", None), - "cache": getattr(config, "use_cached_dependency_manager", False), - "app_config": config, - "dependency_resolvers": getattr(config, "dependency_resolvers", None), - } - return build_dependency_manager_from_dict(config_dict) +def build_dependency_manager(app_config_dict=None, resolution_config_dict=None, conf_file=None, default_tool_dependency_dir=None): + """Build a DependencyManager object from app and/or resolution config. + If app_config_dict is specified, it should be application configuration information + and configuration options are generally named to identify the context is dependency + management (e.g. conda_prefix not prefix or use_cached_dependency_manager not cache). + resolution_config_dict if specified is assumed to be the to_dict() version of a + DependencyManager and should only contain dependency configuration options. + """ -def build_dependency_manager_from_dict(config_dict): - if config_dict.get("use_tool_dependencies", False): - dependency_manager_kwds = { - 'default_base_path': config_dict.get("default_base_path"), - 'conf_file': config_dict.get("conf_file"), - 'app_config': config_dict.get("app_config"), - "dependency_resolver_dicts": config_dict.get("dependency_resolvers"), + if app_config_dict is None: + app_config_dict = {} + else: + app_config_dict = app_config_dict.copy() + + tool_dependency_dir = app_config_dict.get("tool_dependency_dir", default_tool_dependency_dir) + if tool_dependency_dir and tool_dependency_dir.lower() == "none": + app_config_dict["tool_dependency_dir"] = None + + if resolution_config_dict is None and "dependency_resolution" in app_config_dict: + resolution_config_dict = app_config_dict["dependency_resolution"] + + if resolution_config_dict: + # Convert local to_dict options into global ones. + + # to_dict() has "cache", "cache_dir", "use", "default_base_path", "resolvers", "precache" + app_config_props_from_resolution_config = { + "use_tool_dependencies": resolution_config_dict.get("use", None), + "tool_dependency_dir": resolution_config_dict.get("default_base_path", None), + "dependency_resolvers": resolution_config_dict.get("resolvers", None), + "tool_dependency_cache_dir": resolution_config_dict.get("cache_dir", None), + "precache_dependencies": resolution_config_dict.get("precache", None), + "use_cached_dependency_manager": resolution_config_dict.get("cache", None), } - if config_dict.get("cache"): + + for key, value in app_config_props_from_resolution_config.items(): + if value is not None: + app_config_dict[key] = value + + use_tool_dependencies = app_config_dict.get("use_tool_dependencies", None) + # if we haven't set an explicit True or False, try to infer from config... + if use_tool_dependencies is None: + use_tool_dependencies = app_config_dict.get("tool_dependency_dir", default_tool_dependency_dir) is not None or \ + app_config_dict.get("dependency_resolvers") or \ + (conf_file and os.path.exists(conf_file)) + + if use_tool_dependencies: + dependency_manager_kwds = { + "default_base_path": app_config_dict.get("tool_dependency_dir", default_tool_dependency_dir), + "conf_file": conf_file, + "app_config": app_config_dict, + } + if string_as_bool(app_config_dict.get("use_cached_dependency_manager")): dependency_manager = CachedDependencyManager(**dependency_manager_kwds) else: dependency_manager = DependencyManager(**dependency_manager_kwds) @@ -74,7 +107,7 @@ class DependencyManager(object): """ cached = False - def __init__(self, default_base_path, conf_file=None, app_config={}, dependency_resolver_dicts=None): + def __init__(self, default_base_path, conf_file=None, app_config={}): """ Create a new dependency manager looking for packages under the paths listed in `base_paths`. The default base path is app.config.tool_dependency_dir. @@ -88,6 +121,7 @@ class DependencyManager(object): self.resolver_classes = self.__resolvers_dict() plugin_source = None + dependency_resolver_dicts = app_config.get("dependency_resolvers") if dependency_resolver_dicts is not None: plugin_source = ('dict', dependency_resolver_dicts) else: @@ -139,6 +173,10 @@ class DependencyManager(object): value = default return value + @property + def precache(self): + return string_as_bool(self.get_app_option("precache_dependencies", True)) + def dependency_shell_commands(self, requirements, **kwds): requirements_to_dependencies = self.requirements_to_dependencies(requirements, **kwds) ordered_dependencies = OrderedSet(requirements_to_dependencies.values()) @@ -281,11 +319,12 @@ class DependencyManager(object): def to_dict(self): return { + "use": True, "cache": self.cached, - "use_tool_dependencies": True, + "precache": self.precache, + "cache_dir": getattr(self, "tool_dependency_cache_dir", None), "default_base_path": self.default_base_path, - "dependency_resolvers": [m.to_dict() for m in self.dependency_resolvers], - "tool_dependency_cache_dir": getattr(self, "tool_dependency_cache_dir", None), + "resolvers": [m.to_dict() for m in self.dependency_resolvers], } @@ -294,7 +333,7 @@ class CachedDependencyManager(DependencyManager): def __init__(self, default_base_path, **kwd): super(CachedDependencyManager, self).__init__(default_base_path=default_base_path, **kwd) - self.tool_dependency_cache_dir = self.get_app_option("tool_dependency_cache_dir") + self.tool_dependency_cache_dir = self.get_app_option("tool_dependency_cache_dir") or os.path.join(default_base_path, "_cache") def build_cache(self, requirements, **kwds): resolved_dependencies = self.requirements_to_dependencies(requirements, **kwds) @@ -323,7 +362,7 @@ class CachedDependencyManager(DependencyManager): resolved_dependencies = self.requirements_to_dependencies(requirements, **kwds) cacheable_dependencies = [dep for dep in resolved_dependencies.values() if dep.cacheable] hashed_dependencies_dir = self.get_hashed_dependencies_path(cacheable_dependencies) - if not os.path.exists(hashed_dependencies_dir) and self.get_app_option("precache_dependencies", False): + if not os.path.exists(hashed_dependencies_dir) and self.precache: # Cache not present, try to create it self.build_cache(requirements, **kwds) if os.path.exists(hashed_dependencies_dir): @@ -352,6 +391,7 @@ class CachedDependencyManager(DependencyManager): class NullDependencyManager(DependencyManager): + cached = False def __init__(self, default_base_path=None, conf_file=None, app_config={}): self.__app_config = app_config @@ -359,6 +399,7 @@ class NullDependencyManager(DependencyManager): self.dependency_resolvers = [] self._enabled_container_types = [] self._destination_for_container_type = {} + self.default_base_path = None def uses_tool_shed_dependencies(self): return False @@ -370,4 +411,4 @@ class NullDependencyManager(DependencyManager): return NullDependency(version=version, name=name) def to_dict(self): - return {"use_tool_dependencies": False} + return {"use": False} diff --git a/lib/galaxy/tool_util/deps/resolvers/conda.py b/lib/galaxy/tool_util/deps/resolvers/conda.py index e7f4d3a25f0..745f843b723 100644 --- a/lib/galaxy/tool_util/deps/resolvers/conda.py +++ b/lib/galaxy/tool_util/deps/resolvers/conda.py @@ -65,7 +65,7 @@ log = logging.getLogger(__name__) class CondaDependencyResolver(DependencyResolver, MultipleDependencyResolver, ListableDependencyResolver, InstallableDependencyResolver, SpecificationPatternDependencyResolver, MappableDependencyResolver): - dict_collection_visible_keys = DependencyResolver.dict_collection_visible_keys + ['conda_prefix', 'versionless', 'ensure_channels', 'auto_install', 'auto_init'] + dict_collection_visible_keys = DependencyResolver.dict_collection_visible_keys + ['conda_prefix', 'versionless', 'ensure_channels', 'auto_install', 'auto_init', 'use_local'] resolver_type = "conda" config_options = { 'prefix': None, @@ -127,6 +127,7 @@ class CondaDependencyResolver(DependencyResolver, MultipleDependencyResolver, Li copy_dependencies=copy_dependencies, use_local=use_local, ) + self.use_local = use_local self.ensure_channels = ensure_channels # Conda operations options (these define how resolution will occur) diff --git a/lib/galaxy/tools/toolbox/base.py b/lib/galaxy/tools/toolbox/base.py index a85979cbf76..959332466a2 100644 --- a/lib/galaxy/tools/toolbox/base.py +++ b/lib/galaxy/tools/toolbox/base.py @@ -1139,7 +1139,9 @@ class BaseGalaxyToolBox(AbstractToolBox): return looks_like_a_tool(path, enable_beta_formats=getattr(self.app.config, "enable_beta_tool_formats", False)) def _init_dependency_manager(self): - self.dependency_manager = build_dependency_manager(self.app.config) + app_config_dict = self.app.config.config_dict + conf_file = app_config_dict.get("dependency_resolvers_config_file") + self.dependency_manager = build_dependency_manager(app_config_dict=app_config_dict, conf_file=conf_file, default_tool_dependency_dir="database/dependencies") def reload_dependency_manager(self): self._init_dependency_manager() diff --git a/lib/galaxy/webapps/galaxy/config_schema.yml b/lib/galaxy/webapps/galaxy/config_schema.yml index 91875eff914..2c2ebbc91cb 100644 --- a/lib/galaxy/webapps/galaxy/config_schema.yml +++ b/lib/galaxy/webapps/galaxy/config_schema.yml @@ -219,13 +219,13 @@ mapping: default: database/dependencies required: false desc: | - Path to the directory in which tool dependencies are placed. This is used by - the Tool Shed to install dependencies and can also be used by administrators - to manually install or link to dependencies. For details, see: - https://galaxyproject.org/admin/config/tool-dependencies + Various dependency resolver configuration parameters will have defaults set relative + to this path, such as the default conda prefix, default Galaxy packages path, legacy + tool shed dependencies path, and the dependency cache directory. + Set the string to None to explicitly disable tool dependency handling. If this option is set to none or an invalid path, installing tools with dependencies - from the Tool Shed will fail. + from the Tool Shed or in Conda will fail. dependency_resolvers_config_file: type: str @@ -316,6 +316,10 @@ mapping: used if you experience long waiting times before a job is actually submitted to your cluster. + This only affects tools where some requirements can be resolved but not others, + most modern best practice tools can use prebuilt environments in the Conda + directory. + tool_dependency_cache_dir: type: str default: /_cache @@ -2371,6 +2375,31 @@ mapping: resolvers to enable can be embedded into Galaxy's config with this option. This has no effect if a dependency_resolvers_config_file is used. + dependency_resolution: + type: map + desc: | + Alternative representation of various dependency resolution parameters. Takes the + dictified version of a DependencyManager object - so this is ideal for automating the + configuration of dependency resolution from one application that uses a DependencyManager + to another. + mapping: + cache: + type: bool + default: false + desc: See description of 'use_cached_dependency_manager'. + precache: + type: bool + default: true + desc: See description of 'precache_dependencies'. + cache_dir: + type: str + default: '/_cache' + desc: See description of 'tool_dependency_cache_dir'. + default_base_path: + default: database/dependencies + desc: See description of 'tool_dependency_dir'. + resolvers: + desc: See description of 'dependency_resolvers'. default_job_resubmission_condition: type: str required: false diff --git a/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py b/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py index 001deeb2794..e2678aad8a5 100644 --- a/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py +++ b/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py @@ -524,11 +524,11 @@ class AdminToolshed(AdminGalaxy): (escape(str(repository.name)), updating_to_changeset_revision) self.initiate_tool_dependency_installation(trans, tool_dependencies, message=message, status=status) # Handle tool dependencies check box. - if trans.app.config.tool_dependency_dir is None: + if not trans.app.toolbox.dependency_manager.uses_tool_shed_dependencies: if tool_dependencies_dict: message = ("Tool dependencies defined in this repository can be automatically installed if you set " "the value of your tool_dependency_dir setting in your Galaxy config file " - "(galaxy.ini) and restart your Galaxy server.") + "(galaxy.yml) and restart your Galaxy server.") status = "warning" install_tool_dependencies_check_box_checked = False else: @@ -696,7 +696,7 @@ class AdminToolshed(AdminGalaxy): message = 'No selected tool dependencies can be uninstalled, you may need to use the Repair repository feature.' status = 'error' elif operation == "install": - if trans.app.config.tool_dependency_dir: + if trans.app.toolbox.dependency_manager.uses_tool_shed_dependencies: tool_dependencies_for_installation = [] for tool_dependency_id in tool_dependency_ids: tool_dependency = tool_dependency_util.get_tool_dependency(trans.app, tool_dependency_id) @@ -756,7 +756,7 @@ class AdminToolshed(AdminGalaxy): action='browse_tool_dependency', **kwd)) elif operation == "install": - if trans.app.config.tool_dependency_dir: + if trans.app.toolbox.dependency_manager.uses_tool_shed_dependencies: tool_dependencies_for_installation = [] for tool_dependency_id in tool_dependency_ids: tool_dependency = tool_dependency_util.get_tool_dependency(trans.app, tool_dependency_id) @@ -1048,11 +1048,11 @@ class AdminToolshed(AdminGalaxy): # Merge all containers into a single container. containers_dict = dd.merge_containers_dicts_for_new_install(containers_dicts) # Handle tool dependencies check box. - if trans.app.config.tool_dependency_dir is None: + if not trans.app.toolbox.dependency_manager.uses_tool_shed_dependencies: if includes_tool_dependencies: message = "Tool dependencies defined in this repository can be automatically installed if you set " message += "the value of your tool_dependency_dir setting in your Galaxy config file " - message += "(galaxy.ini) and restart your Galaxy server before installing the repository." + message += "(galaxy.yml) and restart your Galaxy server before installing the repository." status = "warning" install_tool_dependencies_check_box_checked = False else: @@ -1429,10 +1429,10 @@ class AdminToolshed(AdminGalaxy): # Handle repository dependencies check box. install_repository_dependencies_check_box = CheckboxField('install_repository_dependencies', value=True) # Handle tool dependencies check box. - if trans.app.config.tool_dependency_dir is None: + if not trans.app.toolbox.dependency_manager.uses_tool_shed_dependencies: if includes_tool_dependencies: message += "Tool dependencies defined in this repository can be automatically installed if you set the value of your tool_dependency_dir " - message += "setting in your Galaxy config file (galaxy.ini) and restart your Galaxy server before installing the repository. " + message += "setting in your Galaxy config file (galaxy.yml) and restart your Galaxy server before installing the repository. " status = "warning" install_tool_dependencies_check_box_checked = False else: diff --git a/lib/tool_shed/galaxy_install/dependency_display.py b/lib/tool_shed/galaxy_install/dependency_display.py index 2ab5f97b706..44779571903 100644 --- a/lib/tool_shed/galaxy_install/dependency_display.py +++ b/lib/tool_shed/galaxy_install/dependency_display.py @@ -37,8 +37,8 @@ class DependencyDisplayer(object): changeset_revision = requirements_dict.get('changeset_revision', 'unknown') dependency_name = requirements_dict['name'] version = requirements_dict['version'] - if self.app.config.tool_dependency_dir: - root_dir = self.app.config.tool_dependency_dir + if self.app.tool_dependency_dir: + root_dir = self.app.tool_dependency_dir else: root_dir = '' install_dir = os.path.join(root_dir, diff --git a/lib/tool_shed/galaxy_install/install_manager.py b/lib/tool_shed/galaxy_install/install_manager.py index e75d0e0930d..20605e8fce1 100644 --- a/lib/tool_shed/galaxy_install/install_manager.py +++ b/lib/tool_shed/galaxy_install/install_manager.py @@ -899,8 +899,9 @@ class InstallRepositoryManager(object): new_tools = [self.app.toolbox._tools_by_id.get(tool_d['guid'], None) for tool_d in metadata['tools']] new_requirements = set([tool.requirements.packages for tool in new_tools if tool]) [self._view.install_dependencies(r) for r in new_requirements] - if self.app.config.use_cached_dependency_manager: - [self.app.toolbox.dependency_manager.build_cache(r) for r in new_requirements] + dependency_manager = self.app.toolbox.dependency_manager + if dependency_manager.cached: + [dependency_manager.build_cache(r) for r in new_requirements] if install_tool_dependencies and tool_shed_repository.tool_dependencies and 'tool_dependencies' in metadata: work_dir = tempfile.mkdtemp(prefix="tmp-toolshed-itsr") @@ -992,7 +993,7 @@ class InstallRepositoryManager(object): self.install_model.context.flush() def __assert_can_install_dependencies(self): - if self.app.config.tool_dependency_dir is None: + if self.app.tool_dependency_dir is None: no_tool_dependency_dir_message = "Tool dependencies can be automatically installed only if you set " no_tool_dependency_dir_message += "the value of your 'tool_dependency_dir' setting in your Galaxy " no_tool_dependency_dir_message += "configuration file (galaxy.ini) and restart your Galaxy server. " diff --git a/lib/tool_shed/galaxy_install/tool_dependencies/recipe/step_handler.py b/lib/tool_shed/galaxy_install/tool_dependencies/recipe/step_handler.py index b83281196c9..6b7176dee3a 100644 --- a/lib/tool_shed/galaxy_install/tool_dependencies/recipe/step_handler.py +++ b/lib/tool_shed/galaxy_install/tool_dependencies/recipe/step_handler.py @@ -1576,7 +1576,7 @@ class SetupVirtualEnv(Download, RecipeStep): # This class is not currently used during stage 1 of the installation process, so filter_actions # are not affected, and dir is not set. Enhancements can easily be made to this function if this # class is needed in stage 1. - venv_src_directory = os.path.abspath(os.path.join(self.app.config.tool_dependency_dir, '__virtualenv_src')) + venv_src_directory = os.path.abspath(os.path.join(self.app.tool_dependency_dir, '__virtualenv_src')) if not self.install_virtualenv(install_environment, venv_src_directory): log.debug('Unable to install virtualenv') return tool_dependency, None, None diff --git a/lib/tool_shed/galaxy_install/tool_migration_manager.py b/lib/tool_shed/galaxy_install/tool_migration_manager.py index e8de5dd15cb..945b433f111 100644 --- a/lib/tool_shed/galaxy_install/tool_migration_manager.py +++ b/lib/tool_shed/galaxy_install/tool_migration_manager.py @@ -47,7 +47,7 @@ class ToolMigrationManager(object): self.tpm = tool_panel_manager.ToolPanelManager(self.app) # If install_dependencies is True but tool_dependency_dir is not set, do not attempt # to install but print informative error message. - if install_dependencies and app.config.tool_dependency_dir is None: + if install_dependencies and app.tool_dependency_dir is None: message = 'You are attempting to install tool dependencies but do not have a value ' message += 'for "tool_dependency_dir" set in your galaxy.ini file. Set this ' message += 'location value to the path where you want tool dependencies installed and ' diff --git a/lib/tool_shed/util/tool_dependency_util.py b/lib/tool_shed/util/tool_dependency_util.py index 3b83716a515..fcddf11f867 100644 --- a/lib/tool_shed/util/tool_dependency_util.py +++ b/lib/tool_shed/util/tool_dependency_util.py @@ -207,14 +207,14 @@ def get_tool_dependency_ids(as_string=False, **kwd): def get_tool_dependency_install_dir(app, repository_name, repository_owner, repository_changeset_revision, tool_dependency_type, tool_dependency_name, tool_dependency_version): if tool_dependency_type == 'package': - return os.path.abspath(os.path.join(app.config.tool_dependency_dir, + return os.path.abspath(os.path.join(app.tool_dependency_dir, tool_dependency_name, tool_dependency_version, repository_owner, repository_name, repository_changeset_revision)) if tool_dependency_type == 'set_environment': - return os.path.abspath(os.path.join(app.config.tool_dependency_dir, + return os.path.abspath(os.path.join(app.tool_dependency_dir, 'environment_settings', tool_dependency_name, repository_owner, diff --git a/scripts/manage_tool_dependencies.py b/scripts/manage_tool_dependencies.py index e1de8bcf562..57175f4837f 100644 --- a/scripts/manage_tool_dependencies.py +++ b/scripts/manage_tool_dependencies.py @@ -7,9 +7,8 @@ from galaxy.config import ( configure_logging, find_path, find_root, - parse_dependency_options, ) -from galaxy.tool_util.deps import CachedDependencyManager, DependencyManager, NullDependencyManager +from galaxy.tool_util.deps import build_dependency_manager from galaxy.util.script import main_factory DESCRIPTION = "Script to manage tool dependencies (with focus on a Conda environments)." @@ -29,24 +28,7 @@ def _build_dependency_manager_no_config(kwargs): configure_logging(kwargs) root = find_root(kwargs) dependency_resolvers_config_file = find_path(kwargs, "dependency_resolvers_config_file", root) - use_dependencies, tool_dependency_dir, use_cached_dependency_manager, tool_dependency_cache_dir, precache_dependencies = \ - parse_dependency_options(kwargs, root, dependency_resolvers_config_file) - - if not use_dependencies: - dependency_manager = NullDependencyManager() - else: - dependency_manager_kwds = { - 'default_base_path': tool_dependency_dir, - 'conf_file': dependency_resolvers_config_file, - 'app_config': kwargs, - } - - if use_cached_dependency_manager: - dependency_manager_kwds['tool_dependency_cache_dir'] = tool_dependency_cache_dir - dependency_manager = CachedDependencyManager(**dependency_manager_kwds) - else: - dependency_manager = DependencyManager(**dependency_manager_kwds) - + dependency_manager = build_dependency_manager(app_config_dict=kwargs, conf_file=dependency_resolvers_config_file, default_tool_dependency_dir="database/dependencies") return dependency_manager diff --git a/templates/admin/tool_shed_repository/common.mako b/templates/admin/tool_shed_repository/common.mako index 427bb5a9a6f..16f9d793f5b 100644 --- a/templates/admin/tool_shed_repository/common.mako +++ b/templates/admin/tool_shed_repository/common.mako @@ -189,7 +189,7 @@ %if install_tool_dependencies_check_box is not None:
- <% disabled = trans.app.config.tool_dependency_dir is None %> + <% disabled = trans.app.tool_dependency_dir is None %> ${render_checkbox(install_tool_dependencies_check_box, disabled=disabled)}
%if disabled: diff --git a/templates/admin/tool_shed_repository/install_tool_dependencies_with_update.mako b/templates/admin/tool_shed_repository/install_tool_dependencies_with_update.mako index 9b360d10758..366eeb3bd3c 100644 --- a/templates/admin/tool_shed_repository/install_tool_dependencies_with_update.mako +++ b/templates/admin/tool_shed_repository/install_tool_dependencies_with_update.mako @@ -31,7 +31,7 @@ ${render_galaxy_repository_actions( repository )} %if install_tool_dependencies_check_box is not None:
- <% disabled = trans.app.config.tool_dependency_dir is None %> + <% disabled = trans.app.tool_dependency_dir is None %> ${render_checkbox(install_tool_dependencies_check_box, disabled=disabled)}
%if disabled: @@ -63,7 +63,7 @@ ${render_galaxy_repository_actions( repository )} key_name = key_items[ 0 ] key_version = key_items[ 1 ] readme_text = requirements_dict.get( 'readme', None ) - install_dir = os.path.join( trans.app.config.tool_dependency_dir, + install_dir = os.path.join( trans.app.tool_dependency_dir, key_name, key_version, repository.owner, diff --git a/templates/admin/tool_shed_repository/uninstall_tool_dependencies.mako b/templates/admin/tool_shed_repository/uninstall_tool_dependencies.mako index d85700fb28d..321d5f708d2 100644 --- a/templates/admin/tool_shed_repository/uninstall_tool_dependencies.mako +++ b/templates/admin/tool_shed_repository/uninstall_tool_dependencies.mako @@ -26,14 +26,14 @@ ${render_galaxy_repository_actions( repository )} <% if tool_dependency.type == 'package': - install_dir = os.path.join( trans.app.config.tool_dependency_dir, + install_dir = os.path.join( trans.app.tool_dependency_dir, tool_dependency.name, tool_dependency.version, tool_dependency.tool_shed_repository.owner, tool_dependency.tool_shed_repository.name, tool_dependency.tool_shed_repository.installed_changeset_revision ) elif tool_dependency.type == 'set_environment': - install_dir = os.path.join( trans.app.config.tool_dependency_dir, + install_dir = os.path.join( trans.app.tool_dependency_dir, 'environment_settings', tool_dependency.name, tool_dependency.tool_shed_repository.owner, diff --git a/test/unit/tools/test_tool_deps.py b/test/unit/tools/test_tool_deps.py index 0f609201a77..c28818bd526 100644 --- a/test/unit/tools/test_tool_deps.py +++ b/test/unit/tools/test_tool_deps.py @@ -12,7 +12,7 @@ from shutil import rmtree from stat import S_IXUSR from subprocess import PIPE, Popen -from galaxy.tool_util.deps import build_dependency_manager_from_dict, DependencyManager +from galaxy.tool_util.deps import build_dependency_manager, DependencyManager from galaxy.tool_util.deps.requirements import ( ToolRequirement, ToolRequirements @@ -610,6 +610,114 @@ def test_config_module_directory_searcher(): assert module_resolver.module_checker.directories == ["/opt/Modules/modulefiles"] +def test_dependency_manager_config_options_global(): + app_config = { + "tool_dependency_dir": "/tmp", + "tool_dependency_cache_dir": "/tmp", + "conda_auto_init": False, + } + dm = __dependency_manager_for_config(app_config.copy()) + assert not dm.to_dict()["cache"] + + app_config["use_cached_dependency_manager"] = True + dm = __dependency_manager_for_config(app_config.copy()) + assert dm.to_dict()["cache"] + assert dm.to_dict()["precache"] + + app_config["precache_dependencies"] = False + dm = __dependency_manager_for_config(app_config.copy()) + assert not dm.to_dict()["precache"] + + conda_opts = _first_conda_resolver_options(dm) + assert conda_opts["use_local"] is False + + app_config["conda_use_local"] = True + dm = __dependency_manager_for_config(app_config.copy()) + conda_opts = _first_conda_resolver_options(dm) + assert conda_opts["use_local"] is True + + +def test_dependency_manager_config_options_embedded_config(): + app_config = { + "dependency_resolution": { + "default_base_path": "/tmp", + "cache_dir": "/tmp", + }, + "conda_auto_init": False, + } + dm = __dependency_manager_for_config(app_config.copy()) + assert not dm.to_dict()["cache"] + + app_config["dependency_resolution"]["cache"] = True + dm = __dependency_manager_for_config(app_config.copy()) + assert dm.to_dict()["cache"] + assert dm.to_dict()["precache"] + + app_config["dependency_resolution"]["precache"] = False + dm = __dependency_manager_for_config(app_config.copy()) + assert not dm.to_dict()["precache"] + + conda_opts = _first_conda_resolver_options(dm) + assert conda_opts["use_local"] is False + + app_config["conda_use_local"] = True + dm = __dependency_manager_for_config(app_config.copy()) + conda_opts = _first_conda_resolver_options(dm) + assert conda_opts["use_local"] is True + + +def test_dependency_manager_config_options_resolution_config(): + app_config = { + "conda_auto_init": False, + } + resolution_config = { + "default_base_path": "/tmp", + "cache_dir": "/tmp", + } + dm = __dependency_manager_for_config(app_config.copy(), resolution_config=resolution_config.copy()) + assert not dm.to_dict()["cache"] + + resolution_config["cache"] = True + dm = __dependency_manager_for_config(app_config.copy(), resolution_config=resolution_config.copy()) + assert dm.to_dict()["cache"] + assert dm.to_dict()["precache"] + + resolution_config["precache"] = False + dm = __dependency_manager_for_config(app_config.copy(), resolution_config=resolution_config.copy()) + assert not dm.to_dict()["precache"] + + conda_opts = _first_conda_resolver_options(dm) + assert conda_opts["use_local"] is False + + app_config["conda_use_local"] = True + dm = __dependency_manager_for_config(app_config.copy(), resolution_config=resolution_config.copy()) + conda_opts = _first_conda_resolver_options(dm) + assert conda_opts["use_local"] is True + + +def test_dependency_manager_none(): + # by default tool_dependency_dir will be use to create some default resolvers... + app_config = { + "conda_auto_init": False, + "tool_dependency_dir": "some_not_none_value", + } + dm = __dependency_manager_for_config(app_config.copy()) + assert dm.to_dict()["use"] + + # but setting it none disables dependency resolution unless explicit resolvers + # are configured. + app_config = { + "conda_auto_init": False, + "tool_dependency_dir": "none", + } + dm = __dependency_manager_for_config(app_config.copy()) + assert not dm.to_dict()["use"] + + +def _first_conda_resolver_options(dm): + return [r for r in dm.to_dict()["resolvers"] if r["resolver_type"] == "conda"][0] + + @contextmanager def __environ(values, remove=[]): """ @@ -652,7 +760,15 @@ def __dependency_manager_for_base_path(default_base_path, conf_file=None): dm = DependencyManager(default_base_path=default_base_path, conf_file=conf_file, app_config={"conda_auto_init": False}) if ROUND_TRIP_TEST_DEPENDENCY_MANAGER_SERIALIZATION: as_dict = dm.to_dict() - dm = build_dependency_manager_from_dict(as_dict) + dm = build_dependency_manager(resolution_config_dict=as_dict) + return dm + + +def __dependency_manager_for_config(app_config, resolution_config=None): + dm = build_dependency_manager(app_config_dict=app_config, resolution_config_dict=resolution_config) + if ROUND_TRIP_TEST_DEPENDENCY_MANAGER_SERIALIZATION: + as_dict = dm.to_dict() + dm = build_dependency_manager(resolution_config_dict=as_dict) return dm diff --git a/test/unit/unittest_utils/galaxy_mock.py b/test/unit/unittest_utils/galaxy_mock.py index aef53950912..ad266b08db6 100644 --- a/test/unit/unittest_utils/galaxy_mock.py +++ b/test/unit/unittest_utils/galaxy_mock.py @@ -151,6 +151,10 @@ class MockAppConfig(Bunch): # set by MockDir self.root = root + @property + def config_dict(self): + return self.dict() + class MockWebapp(object):