From 450ae76ddb297312b226d8bf8a2f8ef58afd993f Mon Sep 17 00:00:00 2001 From: John Chilton Date: Sat, 15 Jun 2019 12:39:12 -0400 Subject: [PATCH 1/3] Re-try dependency manager serialization and embedded config. PR #8118 - adding to_dict to the dependency manager and an embedded loading of dependency resolution options from galaxy.yml - was really quite poorly thought through. Certain options could be embedded - but only the resolvers and the actual dictified representation of the DependencyManager couldn't be actually embedded properly in the app config - but this is exactly what I wanted to do on the Pulsar side and it would make no sense to have the exact same options in both apps and configured differently. So now the actual dictified representation of the whole DependencyManager can be directly embedded in galaxy.yml as the ``dependency_resolution:`` parameter. All the parameters can still be configured at the top level of the YAML file but with 'dependency' in the config option name (some prefixes, some suffixes, etc... a hodgepodge that has grown organically). Give the messiness of the existing options might be better to encourage using ``dependency_resolution:`` in the future. --- lib/galaxy/config.py | 55 ++------ .../model/tool_shed_install/__init__.py | 4 +- lib/galaxy/tool_util/deps/__init__.py | 93 ++++++++++---- lib/galaxy/tool_util/deps/resolvers/conda.py | 3 +- lib/galaxy/tools/toolbox/base.py | 4 +- lib/galaxy/webapps/galaxy/config_schema.yml | 39 +++++- .../galaxy/controllers/admin_toolshed.py | 16 +-- .../galaxy_install/dependency_display.py | 4 +- .../galaxy_install/install_manager.py | 7 +- .../tool_dependencies/recipe/step_handler.py | 2 +- .../galaxy_install/tool_migration_manager.py | 2 +- lib/tool_shed/util/tool_dependency_util.py | 4 +- scripts/manage_tool_dependencies.py | 22 +--- .../admin/tool_shed_repository/common.mako | 2 +- ...install_tool_dependencies_with_update.mako | 4 +- .../uninstall_tool_dependencies.mako | 4 +- test/unit/tools/test_tool_deps.py | 120 +++++++++++++++++- test/unit/unittest_utils/galaxy_mock.py | 4 + 18 files changed, 268 insertions(+), 121 deletions(-) diff --git a/lib/galaxy/config.py b/lib/galaxy/config.py index f81a2293e95..c5db70f5fbc 100644 --- a/lib/galaxy/config.py +++ b/lib/galaxy/config.py @@ -501,24 +501,16 @@ class Configuration(object): self.tool_ngram_maxsize = kwargs.get("tool_ngram_maxsize", 4) default_tool_test_data_directories = os.environ.get("GALAXY_TEST_FILE_DIR", resolve_path("test-data", self.root)) self.tool_test_data_directories = kwargs.get("tool_test_data_directories", default_tool_test_data_directories) - # Location for tool dependencies. - use_tool_dependencies, tool_dependency_dir, use_cached_dependency_manager, tool_dependency_cache_dir, precache_dependencies = \ - parse_dependency_options(kwargs, self.root, self.dependency_resolvers_config_file) - self.use_tool_dependencies = use_tool_dependencies - self.tool_dependency_dir = tool_dependency_dir - self.use_cached_dependency_manager = use_cached_dependency_manager - self.tool_dependency_cache_dir = tool_dependency_cache_dir - self.precache_dependencies = precache_dependencies - self.dependency_resolvers = kwargs.get("dependency_resolvers") # Deployers may either specify a complete list of mapping files or get the default for free and just # specify a local mapping file to adapt and extend the default one. - if "conda_mapping_files" in kwargs: - self.conda_mapping_files = kwargs["conda_mapping_files"] - else: - self.conda_mapping_files = [ + if "conda_mapping_files" not in kwargs: + conda_mapping_files = [ self.local_conda_mapping_file, os.path.join(self.root, "lib", "galaxy", "tool_util", "deps", "resolvers", "default_conda_mapping.yml"), ] + # dependency resolution options are consumed via config_dict - so don't populate + # self, populate config_dict + self.config_dict["conda_mapping_files"] = conda_mapping_files self.enable_beta_mulled_containers = string_as_bool(kwargs.get('enable_beta_mulled_containers', 'False')) containers_resolvers_config_file = kwargs.get('containers_resolvers_config_file', None) @@ -528,7 +520,10 @@ class Configuration(object): involucro_path = kwargs.get('involucro_path', None) if involucro_path is None: - involucro_path = os.path.join(tool_dependency_dir or "database", "involucro") + target_dir = kwargs.get("tool_dependency_dir", "database/dependencies") + if target_dir == "none": + target_dir = "database" + involucro_path = os.path.join(target_dir, "involucro") self.involucro_path = resolve_path(involucro_path, self.root) self.involucro_auto_init = string_as_bool(kwargs.get('involucro_auto_init', True)) mulled_channels = kwargs.get('mulled_channels') @@ -921,34 +916,6 @@ class Configuration(object): return [parse(v) for v in allowed_origin_hostnames if v] -def parse_dependency_options(kwargs, root, dependency_resolvers_config_file): - # Location for tool dependencies. - tool_dependency_dir = kwargs.get("tool_dependency_dir", "database/dependencies") - if tool_dependency_dir.lower() == "none": - tool_dependency_dir = None - - if tool_dependency_dir is not None: - tool_dependency_dir = resolve_path(tool_dependency_dir, root) - # Setting the following flag to true will ultimately cause tool dependencies - # to be located in the shell environment and used by the job that is executing - # the tool. - use_tool_dependencies = True - tool_dependency_cache_dir = kwargs.get('tool_dependency_cache_dir', os.path.join(tool_dependency_dir, '_cache')) - use_cached_dependency_manager = string_as_bool(kwargs.get("use_cached_dependency_manager", 'False')) - precache_dependencies = string_as_bool(kwargs.get("precache_dependencies", 'True')) - else: - tool_dependency_dir = None - if dependency_resolvers_config_file is None: - use_tool_dependencies = bool(kwargs.get("dependency_resolvers", None)) - else: - use_tool_dependencies = os.path.exists(dependency_resolvers_config_file) - tool_dependency_cache_dir = None - precache_dependencies = False - use_cached_dependency_manager = False - - return use_tool_dependencies, tool_dependency_dir, use_cached_dependency_manager, tool_dependency_cache_dir, precache_dependencies - - def get_database_engine_options(kwargs, model_prefix=''): """ Allow options for the SQLAlchemy database engine to be passed by using @@ -1225,3 +1192,7 @@ class ConfiguresGalaxyMixin(object): except Exception: log.info("Waiting for database: attempt %d of %d" % (i, attempts)) time.sleep(pause) + + @property + def tool_dependency_dir(self): + return self.toolbox.dependency_manager.default_base_path diff --git a/lib/galaxy/model/tool_shed_install/__init__.py b/lib/galaxy/model/tool_shed_install/__init__.py index 4aef91b43b7..430642f60dc 100644 --- a/lib/galaxy/model/tool_shed_install/__init__.py +++ b/lib/galaxy/model/tool_shed_install/__init__.py @@ -555,14 +555,14 @@ class ToolDependency(object): def installation_directory(self, app): if self.type == 'package': - return os.path.join(app.config.tool_dependency_dir, + return os.path.join(app.tool_dependency_dir, self.name, self.version, self.tool_shed_repository.owner, self.tool_shed_repository.name, self.tool_shed_repository.installed_changeset_revision) if self.type == 'set_environment': - return os.path.join(app.config.tool_dependency_dir, + return os.path.join(app.tool_dependency_dir, 'environment_settings', self.name, self.tool_shed_repository.owner, diff --git a/lib/galaxy/tool_util/deps/__init__.py b/lib/galaxy/tool_util/deps/__init__.py index 397a9a12faf..61e7f46a1c7 100644 --- a/lib/galaxy/tool_util/deps/__init__.py +++ b/lib/galaxy/tool_util/deps/__init__.py @@ -10,7 +10,8 @@ from collections import OrderedDict from galaxy.util import ( hash_util, - plugin_config + plugin_config, + string_as_bool, ) from galaxy.util.oset import OrderedSet from .container_resolvers import ContainerResolver @@ -31,27 +32,59 @@ log = logging.getLogger(__name__) CONFIG_VAL_NOT_FOUND = object() -def build_dependency_manager(config): - config_dict = { - "use_tool_dependencies": getattr(config, "use_tool_dependencies", False), - "default_base_path": getattr(config, "tool_dependency_dir", None), - "conf_file": getattr(config, "dependency_resolvers_config_file", None), - "cache": getattr(config, "use_cached_dependency_manager", False), - "app_config": config, - "dependency_resolvers": getattr(config, "dependency_resolvers", None), - } - return build_dependency_manager_from_dict(config_dict) +def build_dependency_manager(app_config_dict=None, resolution_config_dict=None, conf_file=None, default_tool_dependency_dir=None): + """Build a DependencyManager object from app and/or resolution config. + If app_config_dict is specified, it should be application configuration information + and configuration options are generally named to identify the context is dependency + management (e.g. conda_prefix not prefix or use_cached_dependency_manager not cache). + resolution_config_dict if specified is assumed to be the to_dict() version of a + DependencyManager and should only contain dependency configuration options. + """ -def build_dependency_manager_from_dict(config_dict): - if config_dict.get("use_tool_dependencies", False): - dependency_manager_kwds = { - 'default_base_path': config_dict.get("default_base_path"), - 'conf_file': config_dict.get("conf_file"), - 'app_config': config_dict.get("app_config"), - "dependency_resolver_dicts": config_dict.get("dependency_resolvers"), + if app_config_dict is None: + app_config_dict = {} + else: + app_config_dict = app_config_dict.copy() + + tool_dependency_dir = app_config_dict.get("tool_dependency_dir", default_tool_dependency_dir) + if tool_dependency_dir and tool_dependency_dir.lower() == "none": + app_config_dict["tool_dependency_dir"] = None + + if resolution_config_dict is None and "dependency_resolution" in app_config_dict: + resolution_config_dict = app_config_dict["dependency_resolution"] + + if resolution_config_dict: + # Convert local to_dict options into global ones. + + # to_dict() has "cache", "cache_dir", "use", "default_base_path", "resolvers", "precache" + app_config_props_from_resolution_config = { + "use_tool_dependencies": resolution_config_dict.get("use", None), + "tool_dependency_dir": resolution_config_dict.get("default_base_path", None), + "dependency_resolvers": resolution_config_dict.get("resolvers", None), + "tool_dependency_cache_dir": resolution_config_dict.get("cache_dir", None), + "precache_dependencies": resolution_config_dict.get("precache", None), + "use_cached_dependency_manager": resolution_config_dict.get("cache", None), } - if config_dict.get("cache"): + + for key, value in app_config_props_from_resolution_config.items(): + if value is not None: + app_config_dict[key] = value + + use_tool_dependencies = app_config_dict.get("use_tool_dependencies", None) + # if we haven't set an explicit True or False, try to infer from config... + if use_tool_dependencies is None: + use_tool_dependencies = app_config_dict.get("tool_dependency_dir", default_tool_dependency_dir) is not None or \ + app_config_dict.get("dependency_resolvers") or \ + (conf_file and os.path.exists(conf_file)) + + if use_tool_dependencies: + dependency_manager_kwds = { + "default_base_path": app_config_dict.get("tool_dependency_dir", default_tool_dependency_dir), + "conf_file": conf_file, + "app_config": app_config_dict, + } + if string_as_bool(app_config_dict.get("use_cached_dependency_manager")): dependency_manager = CachedDependencyManager(**dependency_manager_kwds) else: dependency_manager = DependencyManager(**dependency_manager_kwds) @@ -74,7 +107,7 @@ class DependencyManager(object): """ cached = False - def __init__(self, default_base_path, conf_file=None, app_config={}, dependency_resolver_dicts=None): + def __init__(self, default_base_path, conf_file=None, app_config={}): """ Create a new dependency manager looking for packages under the paths listed in `base_paths`. The default base path is app.config.tool_dependency_dir. @@ -88,6 +121,7 @@ class DependencyManager(object): self.resolver_classes = self.__resolvers_dict() plugin_source = None + dependency_resolver_dicts = app_config.get("dependency_resolvers") if dependency_resolver_dicts is not None: plugin_source = ('dict', dependency_resolver_dicts) else: @@ -139,6 +173,10 @@ class DependencyManager(object): value = default return value + @property + def precache(self): + return string_as_bool(self.get_app_option("precache_dependencies", True)) + def dependency_shell_commands(self, requirements, **kwds): requirements_to_dependencies = self.requirements_to_dependencies(requirements, **kwds) ordered_dependencies = OrderedSet(requirements_to_dependencies.values()) @@ -281,11 +319,12 @@ class DependencyManager(object): def to_dict(self): return { + "use": True, "cache": self.cached, - "use_tool_dependencies": True, + "precache": self.precache, + "cache_dir": getattr(self, "tool_dependency_cache_dir", None), "default_base_path": self.default_base_path, - "dependency_resolvers": [m.to_dict() for m in self.dependency_resolvers], - "tool_dependency_cache_dir": getattr(self, "tool_dependency_cache_dir", None), + "resolvers": [m.to_dict() for m in self.dependency_resolvers], } @@ -294,7 +333,7 @@ class CachedDependencyManager(DependencyManager): def __init__(self, default_base_path, **kwd): super(CachedDependencyManager, self).__init__(default_base_path=default_base_path, **kwd) - self.tool_dependency_cache_dir = self.get_app_option("tool_dependency_cache_dir") + self.tool_dependency_cache_dir = self.get_app_option("tool_dependency_cache_dir") or os.path.join(default_base_path, "_cache") def build_cache(self, requirements, **kwds): resolved_dependencies = self.requirements_to_dependencies(requirements, **kwds) @@ -323,7 +362,7 @@ class CachedDependencyManager(DependencyManager): resolved_dependencies = self.requirements_to_dependencies(requirements, **kwds) cacheable_dependencies = [dep for dep in resolved_dependencies.values() if dep.cacheable] hashed_dependencies_dir = self.get_hashed_dependencies_path(cacheable_dependencies) - if not os.path.exists(hashed_dependencies_dir) and self.get_app_option("precache_dependencies", False): + if not os.path.exists(hashed_dependencies_dir) and self.precache: # Cache not present, try to create it self.build_cache(requirements, **kwds) if os.path.exists(hashed_dependencies_dir): @@ -352,6 +391,7 @@ class CachedDependencyManager(DependencyManager): class NullDependencyManager(DependencyManager): + cached = False def __init__(self, default_base_path=None, conf_file=None, app_config={}): self.__app_config = app_config @@ -359,6 +399,7 @@ class NullDependencyManager(DependencyManager): self.dependency_resolvers = [] self._enabled_container_types = [] self._destination_for_container_type = {} + self.default_base_path = None def uses_tool_shed_dependencies(self): return False @@ -370,4 +411,4 @@ class NullDependencyManager(DependencyManager): return NullDependency(version=version, name=name) def to_dict(self): - return {"use_tool_dependencies": False} + return {"use": False} diff --git a/lib/galaxy/tool_util/deps/resolvers/conda.py b/lib/galaxy/tool_util/deps/resolvers/conda.py index e7f4d3a25f0..745f843b723 100644 --- a/lib/galaxy/tool_util/deps/resolvers/conda.py +++ b/lib/galaxy/tool_util/deps/resolvers/conda.py @@ -65,7 +65,7 @@ log = logging.getLogger(__name__) class CondaDependencyResolver(DependencyResolver, MultipleDependencyResolver, ListableDependencyResolver, InstallableDependencyResolver, SpecificationPatternDependencyResolver, MappableDependencyResolver): - dict_collection_visible_keys = DependencyResolver.dict_collection_visible_keys + ['conda_prefix', 'versionless', 'ensure_channels', 'auto_install', 'auto_init'] + dict_collection_visible_keys = DependencyResolver.dict_collection_visible_keys + ['conda_prefix', 'versionless', 'ensure_channels', 'auto_install', 'auto_init', 'use_local'] resolver_type = "conda" config_options = { 'prefix': None, @@ -127,6 +127,7 @@ class CondaDependencyResolver(DependencyResolver, MultipleDependencyResolver, Li copy_dependencies=copy_dependencies, use_local=use_local, ) + self.use_local = use_local self.ensure_channels = ensure_channels # Conda operations options (these define how resolution will occur) diff --git a/lib/galaxy/tools/toolbox/base.py b/lib/galaxy/tools/toolbox/base.py index a85979cbf76..959332466a2 100644 --- a/lib/galaxy/tools/toolbox/base.py +++ b/lib/galaxy/tools/toolbox/base.py @@ -1139,7 +1139,9 @@ class BaseGalaxyToolBox(AbstractToolBox): return looks_like_a_tool(path, enable_beta_formats=getattr(self.app.config, "enable_beta_tool_formats", False)) def _init_dependency_manager(self): - self.dependency_manager = build_dependency_manager(self.app.config) + app_config_dict = self.app.config.config_dict + conf_file = app_config_dict.get("dependency_resolvers_config_file") + self.dependency_manager = build_dependency_manager(app_config_dict=app_config_dict, conf_file=conf_file, default_tool_dependency_dir="database/dependencies") def reload_dependency_manager(self): self._init_dependency_manager() diff --git a/lib/galaxy/webapps/galaxy/config_schema.yml b/lib/galaxy/webapps/galaxy/config_schema.yml index 91875eff914..2c2ebbc91cb 100644 --- a/lib/galaxy/webapps/galaxy/config_schema.yml +++ b/lib/galaxy/webapps/galaxy/config_schema.yml @@ -219,13 +219,13 @@ mapping: default: database/dependencies required: false desc: | - Path to the directory in which tool dependencies are placed. This is used by - the Tool Shed to install dependencies and can also be used by administrators - to manually install or link to dependencies. For details, see: - https://galaxyproject.org/admin/config/tool-dependencies + Various dependency resolver configuration parameters will have defaults set relative + to this path, such as the default conda prefix, default Galaxy packages path, legacy + tool shed dependencies path, and the dependency cache directory. + Set the string to None to explicitly disable tool dependency handling. If this option is set to none or an invalid path, installing tools with dependencies - from the Tool Shed will fail. + from the Tool Shed or in Conda will fail. dependency_resolvers_config_file: type: str @@ -316,6 +316,10 @@ mapping: used if you experience long waiting times before a job is actually submitted to your cluster. + This only affects tools where some requirements can be resolved but not others, + most modern best practice tools can use prebuilt environments in the Conda + directory. + tool_dependency_cache_dir: type: str default: /_cache @@ -2371,6 +2375,31 @@ mapping: resolvers to enable can be embedded into Galaxy's config with this option. This has no effect if a dependency_resolvers_config_file is used. + dependency_resolution: + type: map + desc: | + Alternative representation of various dependency resolution parameters. Takes the + dictified version of a DependencyManager object - so this is ideal for automating the + configuration of dependency resolution from one application that uses a DependencyManager + to another. + mapping: + cache: + type: bool + default: false + desc: See description of 'use_cached_dependency_manager'. + precache: + type: bool + default: true + desc: See description of 'precache_dependencies'. + cache_dir: + type: str + default: '/_cache' + desc: See description of 'tool_dependency_cache_dir'. + default_base_path: + default: database/dependencies + desc: See description of 'tool_dependency_dir'. + resolvers: + desc: See description of 'dependency_resolvers'. default_job_resubmission_condition: type: str required: false diff --git a/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py b/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py index 001deeb2794..e2678aad8a5 100644 --- a/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py +++ b/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py @@ -524,11 +524,11 @@ class AdminToolshed(AdminGalaxy): (escape(str(repository.name)), updating_to_changeset_revision) self.initiate_tool_dependency_installation(trans, tool_dependencies, message=message, status=status) # Handle tool dependencies check box. - if trans.app.config.tool_dependency_dir is None: + if not trans.app.toolbox.dependency_manager.uses_tool_shed_dependencies: if tool_dependencies_dict: message = ("Tool dependencies defined in this repository can be automatically installed if you set " "the value of your tool_dependency_dir setting in your Galaxy config file " - "(galaxy.ini) and restart your Galaxy server.") + "(galaxy.yml) and restart your Galaxy server.") status = "warning" install_tool_dependencies_check_box_checked = False else: @@ -696,7 +696,7 @@ class AdminToolshed(AdminGalaxy): message = 'No selected tool dependencies can be uninstalled, you may need to use the Repair repository feature.' status = 'error' elif operation == "install": - if trans.app.config.tool_dependency_dir: + if trans.app.toolbox.dependency_manager.uses_tool_shed_dependencies: tool_dependencies_for_installation = [] for tool_dependency_id in tool_dependency_ids: tool_dependency = tool_dependency_util.get_tool_dependency(trans.app, tool_dependency_id) @@ -756,7 +756,7 @@ class AdminToolshed(AdminGalaxy): action='browse_tool_dependency', **kwd)) elif operation == "install": - if trans.app.config.tool_dependency_dir: + if trans.app.toolbox.dependency_manager.uses_tool_shed_dependencies: tool_dependencies_for_installation = [] for tool_dependency_id in tool_dependency_ids: tool_dependency = tool_dependency_util.get_tool_dependency(trans.app, tool_dependency_id) @@ -1048,11 +1048,11 @@ class AdminToolshed(AdminGalaxy): # Merge all containers into a single container. containers_dict = dd.merge_containers_dicts_for_new_install(containers_dicts) # Handle tool dependencies check box. - if trans.app.config.tool_dependency_dir is None: + if not trans.app.toolbox.dependency_manager.uses_tool_shed_dependencies: if includes_tool_dependencies: message = "Tool dependencies defined in this repository can be automatically installed if you set " message += "the value of your tool_dependency_dir setting in your Galaxy config file " - message += "(galaxy.ini) and restart your Galaxy server before installing the repository." + message += "(galaxy.yml) and restart your Galaxy server before installing the repository." status = "warning" install_tool_dependencies_check_box_checked = False else: @@ -1429,10 +1429,10 @@ class AdminToolshed(AdminGalaxy): # Handle repository dependencies check box. install_repository_dependencies_check_box = CheckboxField('install_repository_dependencies', value=True) # Handle tool dependencies check box. - if trans.app.config.tool_dependency_dir is None: + if not trans.app.toolbox.dependency_manager.uses_tool_shed_dependencies: if includes_tool_dependencies: message += "Tool dependencies defined in this repository can be automatically installed if you set the value of your tool_dependency_dir " - message += "setting in your Galaxy config file (galaxy.ini) and restart your Galaxy server before installing the repository. " + message += "setting in your Galaxy config file (galaxy.yml) and restart your Galaxy server before installing the repository. " status = "warning" install_tool_dependencies_check_box_checked = False else: diff --git a/lib/tool_shed/galaxy_install/dependency_display.py b/lib/tool_shed/galaxy_install/dependency_display.py index 2ab5f97b706..44779571903 100644 --- a/lib/tool_shed/galaxy_install/dependency_display.py +++ b/lib/tool_shed/galaxy_install/dependency_display.py @@ -37,8 +37,8 @@ class DependencyDisplayer(object): changeset_revision = requirements_dict.get('changeset_revision', 'unknown') dependency_name = requirements_dict['name'] version = requirements_dict['version'] - if self.app.config.tool_dependency_dir: - root_dir = self.app.config.tool_dependency_dir + if self.app.tool_dependency_dir: + root_dir = self.app.tool_dependency_dir else: root_dir = '' install_dir = os.path.join(root_dir, diff --git a/lib/tool_shed/galaxy_install/install_manager.py b/lib/tool_shed/galaxy_install/install_manager.py index e75d0e0930d..20605e8fce1 100644 --- a/lib/tool_shed/galaxy_install/install_manager.py +++ b/lib/tool_shed/galaxy_install/install_manager.py @@ -899,8 +899,9 @@ class InstallRepositoryManager(object): new_tools = [self.app.toolbox._tools_by_id.get(tool_d['guid'], None) for tool_d in metadata['tools']] new_requirements = set([tool.requirements.packages for tool in new_tools if tool]) [self._view.install_dependencies(r) for r in new_requirements] - if self.app.config.use_cached_dependency_manager: - [self.app.toolbox.dependency_manager.build_cache(r) for r in new_requirements] + dependency_manager = self.app.toolbox.dependency_manager + if dependency_manager.cached: + [dependency_manager.build_cache(r) for r in new_requirements] if install_tool_dependencies and tool_shed_repository.tool_dependencies and 'tool_dependencies' in metadata: work_dir = tempfile.mkdtemp(prefix="tmp-toolshed-itsr") @@ -992,7 +993,7 @@ class InstallRepositoryManager(object): self.install_model.context.flush() def __assert_can_install_dependencies(self): - if self.app.config.tool_dependency_dir is None: + if self.app.tool_dependency_dir is None: no_tool_dependency_dir_message = "Tool dependencies can be automatically installed only if you set " no_tool_dependency_dir_message += "the value of your 'tool_dependency_dir' setting in your Galaxy " no_tool_dependency_dir_message += "configuration file (galaxy.ini) and restart your Galaxy server. " diff --git a/lib/tool_shed/galaxy_install/tool_dependencies/recipe/step_handler.py b/lib/tool_shed/galaxy_install/tool_dependencies/recipe/step_handler.py index b83281196c9..6b7176dee3a 100644 --- a/lib/tool_shed/galaxy_install/tool_dependencies/recipe/step_handler.py +++ b/lib/tool_shed/galaxy_install/tool_dependencies/recipe/step_handler.py @@ -1576,7 +1576,7 @@ class SetupVirtualEnv(Download, RecipeStep): # This class is not currently used during stage 1 of the installation process, so filter_actions # are not affected, and dir is not set. Enhancements can easily be made to this function if this # class is needed in stage 1. - venv_src_directory = os.path.abspath(os.path.join(self.app.config.tool_dependency_dir, '__virtualenv_src')) + venv_src_directory = os.path.abspath(os.path.join(self.app.tool_dependency_dir, '__virtualenv_src')) if not self.install_virtualenv(install_environment, venv_src_directory): log.debug('Unable to install virtualenv') return tool_dependency, None, None diff --git a/lib/tool_shed/galaxy_install/tool_migration_manager.py b/lib/tool_shed/galaxy_install/tool_migration_manager.py index e8de5dd15cb..945b433f111 100644 --- a/lib/tool_shed/galaxy_install/tool_migration_manager.py +++ b/lib/tool_shed/galaxy_install/tool_migration_manager.py @@ -47,7 +47,7 @@ class ToolMigrationManager(object): self.tpm = tool_panel_manager.ToolPanelManager(self.app) # If install_dependencies is True but tool_dependency_dir is not set, do not attempt # to install but print informative error message. - if install_dependencies and app.config.tool_dependency_dir is None: + if install_dependencies and app.tool_dependency_dir is None: message = 'You are attempting to install tool dependencies but do not have a value ' message += 'for "tool_dependency_dir" set in your galaxy.ini file. Set this ' message += 'location value to the path where you want tool dependencies installed and ' diff --git a/lib/tool_shed/util/tool_dependency_util.py b/lib/tool_shed/util/tool_dependency_util.py index 3b83716a515..fcddf11f867 100644 --- a/lib/tool_shed/util/tool_dependency_util.py +++ b/lib/tool_shed/util/tool_dependency_util.py @@ -207,14 +207,14 @@ def get_tool_dependency_ids(as_string=False, **kwd): def get_tool_dependency_install_dir(app, repository_name, repository_owner, repository_changeset_revision, tool_dependency_type, tool_dependency_name, tool_dependency_version): if tool_dependency_type == 'package': - return os.path.abspath(os.path.join(app.config.tool_dependency_dir, + return os.path.abspath(os.path.join(app.tool_dependency_dir, tool_dependency_name, tool_dependency_version, repository_owner, repository_name, repository_changeset_revision)) if tool_dependency_type == 'set_environment': - return os.path.abspath(os.path.join(app.config.tool_dependency_dir, + return os.path.abspath(os.path.join(app.tool_dependency_dir, 'environment_settings', tool_dependency_name, repository_owner, diff --git a/scripts/manage_tool_dependencies.py b/scripts/manage_tool_dependencies.py index e1de8bcf562..57175f4837f 100644 --- a/scripts/manage_tool_dependencies.py +++ b/scripts/manage_tool_dependencies.py @@ -7,9 +7,8 @@ from galaxy.config import ( configure_logging, find_path, find_root, - parse_dependency_options, ) -from galaxy.tool_util.deps import CachedDependencyManager, DependencyManager, NullDependencyManager +from galaxy.tool_util.deps import build_dependency_manager from galaxy.util.script import main_factory DESCRIPTION = "Script to manage tool dependencies (with focus on a Conda environments)." @@ -29,24 +28,7 @@ def _build_dependency_manager_no_config(kwargs): configure_logging(kwargs) root = find_root(kwargs) dependency_resolvers_config_file = find_path(kwargs, "dependency_resolvers_config_file", root) - use_dependencies, tool_dependency_dir, use_cached_dependency_manager, tool_dependency_cache_dir, precache_dependencies = \ - parse_dependency_options(kwargs, root, dependency_resolvers_config_file) - - if not use_dependencies: - dependency_manager = NullDependencyManager() - else: - dependency_manager_kwds = { - 'default_base_path': tool_dependency_dir, - 'conf_file': dependency_resolvers_config_file, - 'app_config': kwargs, - } - - if use_cached_dependency_manager: - dependency_manager_kwds['tool_dependency_cache_dir'] = tool_dependency_cache_dir - dependency_manager = CachedDependencyManager(**dependency_manager_kwds) - else: - dependency_manager = DependencyManager(**dependency_manager_kwds) - + dependency_manager = build_dependency_manager(app_config_dict=kwargs, conf_file=dependency_resolvers_config_file, default_tool_dependency_dir="database/dependencies") return dependency_manager diff --git a/templates/admin/tool_shed_repository/common.mako b/templates/admin/tool_shed_repository/common.mako index 427bb5a9a6f..16f9d793f5b 100644 --- a/templates/admin/tool_shed_repository/common.mako +++ b/templates/admin/tool_shed_repository/common.mako @@ -189,7 +189,7 @@ %if install_tool_dependencies_check_box is not None:
- <% disabled = trans.app.config.tool_dependency_dir is None %> + <% disabled = trans.app.tool_dependency_dir is None %> ${render_checkbox(install_tool_dependencies_check_box, disabled=disabled)}
%if disabled: diff --git a/templates/admin/tool_shed_repository/install_tool_dependencies_with_update.mako b/templates/admin/tool_shed_repository/install_tool_dependencies_with_update.mako index 9b360d10758..366eeb3bd3c 100644 --- a/templates/admin/tool_shed_repository/install_tool_dependencies_with_update.mako +++ b/templates/admin/tool_shed_repository/install_tool_dependencies_with_update.mako @@ -31,7 +31,7 @@ ${render_galaxy_repository_actions( repository )} %if install_tool_dependencies_check_box is not None:
- <% disabled = trans.app.config.tool_dependency_dir is None %> + <% disabled = trans.app.tool_dependency_dir is None %> ${render_checkbox(install_tool_dependencies_check_box, disabled=disabled)}
%if disabled: @@ -63,7 +63,7 @@ ${render_galaxy_repository_actions( repository )} key_name = key_items[ 0 ] key_version = key_items[ 1 ] readme_text = requirements_dict.get( 'readme', None ) - install_dir = os.path.join( trans.app.config.tool_dependency_dir, + install_dir = os.path.join( trans.app.tool_dependency_dir, key_name, key_version, repository.owner, diff --git a/templates/admin/tool_shed_repository/uninstall_tool_dependencies.mako b/templates/admin/tool_shed_repository/uninstall_tool_dependencies.mako index d85700fb28d..321d5f708d2 100644 --- a/templates/admin/tool_shed_repository/uninstall_tool_dependencies.mako +++ b/templates/admin/tool_shed_repository/uninstall_tool_dependencies.mako @@ -26,14 +26,14 @@ ${render_galaxy_repository_actions( repository )} <% if tool_dependency.type == 'package': - install_dir = os.path.join( trans.app.config.tool_dependency_dir, + install_dir = os.path.join( trans.app.tool_dependency_dir, tool_dependency.name, tool_dependency.version, tool_dependency.tool_shed_repository.owner, tool_dependency.tool_shed_repository.name, tool_dependency.tool_shed_repository.installed_changeset_revision ) elif tool_dependency.type == 'set_environment': - install_dir = os.path.join( trans.app.config.tool_dependency_dir, + install_dir = os.path.join( trans.app.tool_dependency_dir, 'environment_settings', tool_dependency.name, tool_dependency.tool_shed_repository.owner, diff --git a/test/unit/tools/test_tool_deps.py b/test/unit/tools/test_tool_deps.py index 0f609201a77..c28818bd526 100644 --- a/test/unit/tools/test_tool_deps.py +++ b/test/unit/tools/test_tool_deps.py @@ -12,7 +12,7 @@ from shutil import rmtree from stat import S_IXUSR from subprocess import PIPE, Popen -from galaxy.tool_util.deps import build_dependency_manager_from_dict, DependencyManager +from galaxy.tool_util.deps import build_dependency_manager, DependencyManager from galaxy.tool_util.deps.requirements import ( ToolRequirement, ToolRequirements @@ -610,6 +610,114 @@ def test_config_module_directory_searcher(): assert module_resolver.module_checker.directories == ["/opt/Modules/modulefiles"] +def test_dependency_manager_config_options_global(): + app_config = { + "tool_dependency_dir": "/tmp", + "tool_dependency_cache_dir": "/tmp", + "conda_auto_init": False, + } + dm = __dependency_manager_for_config(app_config.copy()) + assert not dm.to_dict()["cache"] + + app_config["use_cached_dependency_manager"] = True + dm = __dependency_manager_for_config(app_config.copy()) + assert dm.to_dict()["cache"] + assert dm.to_dict()["precache"] + + app_config["precache_dependencies"] = False + dm = __dependency_manager_for_config(app_config.copy()) + assert not dm.to_dict()["precache"] + + conda_opts = _first_conda_resolver_options(dm) + assert conda_opts["use_local"] is False + + app_config["conda_use_local"] = True + dm = __dependency_manager_for_config(app_config.copy()) + conda_opts = _first_conda_resolver_options(dm) + assert conda_opts["use_local"] is True + + +def test_dependency_manager_config_options_embedded_config(): + app_config = { + "dependency_resolution": { + "default_base_path": "/tmp", + "cache_dir": "/tmp", + }, + "conda_auto_init": False, + } + dm = __dependency_manager_for_config(app_config.copy()) + assert not dm.to_dict()["cache"] + + app_config["dependency_resolution"]["cache"] = True + dm = __dependency_manager_for_config(app_config.copy()) + assert dm.to_dict()["cache"] + assert dm.to_dict()["precache"] + + app_config["dependency_resolution"]["precache"] = False + dm = __dependency_manager_for_config(app_config.copy()) + assert not dm.to_dict()["precache"] + + conda_opts = _first_conda_resolver_options(dm) + assert conda_opts["use_local"] is False + + app_config["conda_use_local"] = True + dm = __dependency_manager_for_config(app_config.copy()) + conda_opts = _first_conda_resolver_options(dm) + assert conda_opts["use_local"] is True + + +def test_dependency_manager_config_options_resolution_config(): + app_config = { + "conda_auto_init": False, + } + resolution_config = { + "default_base_path": "/tmp", + "cache_dir": "/tmp", + } + dm = __dependency_manager_for_config(app_config.copy(), resolution_config=resolution_config.copy()) + assert not dm.to_dict()["cache"] + + resolution_config["cache"] = True + dm = __dependency_manager_for_config(app_config.copy(), resolution_config=resolution_config.copy()) + assert dm.to_dict()["cache"] + assert dm.to_dict()["precache"] + + resolution_config["precache"] = False + dm = __dependency_manager_for_config(app_config.copy(), resolution_config=resolution_config.copy()) + assert not dm.to_dict()["precache"] + + conda_opts = _first_conda_resolver_options(dm) + assert conda_opts["use_local"] is False + + app_config["conda_use_local"] = True + dm = __dependency_manager_for_config(app_config.copy(), resolution_config=resolution_config.copy()) + conda_opts = _first_conda_resolver_options(dm) + assert conda_opts["use_local"] is True + + +def test_dependency_manager_none(): + # by default tool_dependency_dir will be use to create some default resolvers... + app_config = { + "conda_auto_init": False, + "tool_dependency_dir": "some_not_none_value", + } + dm = __dependency_manager_for_config(app_config.copy()) + assert dm.to_dict()["use"] + + # but setting it none disables dependency resolution unless explicit resolvers + # are configured. + app_config = { + "conda_auto_init": False, + "tool_dependency_dir": "none", + } + dm = __dependency_manager_for_config(app_config.copy()) + assert not dm.to_dict()["use"] + + +def _first_conda_resolver_options(dm): + return [r for r in dm.to_dict()["resolvers"] if r["resolver_type"] == "conda"][0] + + @contextmanager def __environ(values, remove=[]): """ @@ -652,7 +760,15 @@ def __dependency_manager_for_base_path(default_base_path, conf_file=None): dm = DependencyManager(default_base_path=default_base_path, conf_file=conf_file, app_config={"conda_auto_init": False}) if ROUND_TRIP_TEST_DEPENDENCY_MANAGER_SERIALIZATION: as_dict = dm.to_dict() - dm = build_dependency_manager_from_dict(as_dict) + dm = build_dependency_manager(resolution_config_dict=as_dict) + return dm + + +def __dependency_manager_for_config(app_config, resolution_config=None): + dm = build_dependency_manager(app_config_dict=app_config, resolution_config_dict=resolution_config) + if ROUND_TRIP_TEST_DEPENDENCY_MANAGER_SERIALIZATION: + as_dict = dm.to_dict() + dm = build_dependency_manager(resolution_config_dict=as_dict) return dm diff --git a/test/unit/unittest_utils/galaxy_mock.py b/test/unit/unittest_utils/galaxy_mock.py index aef53950912..ad266b08db6 100644 --- a/test/unit/unittest_utils/galaxy_mock.py +++ b/test/unit/unittest_utils/galaxy_mock.py @@ -151,6 +151,10 @@ class MockAppConfig(Bunch): # set by MockDir self.root = root + @property + def config_dict(self): + return self.dict() + class MockWebapp(object): From 78d351087e3796497bb563524156ec89a31c2a64 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Sun, 7 Jul 2019 12:44:17 +0200 Subject: [PATCH 2/3] Update lib/galaxy/webapps/galaxy/config_schema.yml Co-Authored-By: Marius van den Beek --- lib/galaxy/webapps/galaxy/config_schema.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/galaxy/webapps/galaxy/config_schema.yml b/lib/galaxy/webapps/galaxy/config_schema.yml index 2c2ebbc91cb..96d50114eb5 100644 --- a/lib/galaxy/webapps/galaxy/config_schema.yml +++ b/lib/galaxy/webapps/galaxy/config_schema.yml @@ -2400,6 +2400,7 @@ mapping: desc: See description of 'tool_dependency_dir'. resolvers: desc: See description of 'dependency_resolvers'. + default_job_resubmission_condition: type: str required: false From ee02ddc226d5c840c2fa7c2591019f869961cf2e Mon Sep 17 00:00:00 2001 From: John Chilton Date: Sun, 7 Jul 2019 12:44:23 +0200 Subject: [PATCH 3/3] Update lib/galaxy/tool_util/deps/__init__.py Co-Authored-By: Marius van den Beek --- lib/galaxy/tool_util/deps/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/tool_util/deps/__init__.py b/lib/galaxy/tool_util/deps/__init__.py index 61e7f46a1c7..af255a818de 100644 --- a/lib/galaxy/tool_util/deps/__init__.py +++ b/lib/galaxy/tool_util/deps/__init__.py @@ -36,7 +36,7 @@ def build_dependency_manager(app_config_dict=None, resolution_config_dict=None, """Build a DependencyManager object from app and/or resolution config. If app_config_dict is specified, it should be application configuration information - and configuration options are generally named to identify the context is dependency + and configuration options are generally named to identify the context of dependency management (e.g. conda_prefix not prefix or use_cached_dependency_manager not cache). resolution_config_dict if specified is assumed to be the to_dict() version of a DependencyManager and should only contain dependency configuration options.