opensource/galaxy
3
0
Fork 0
mirror of https://github.com/galaxyproject/galaxy.git synced 2026-09-24 16:30:27 +08:00
Code Issues Packages Projects Releases Wiki Activity

Re-try dependency manager serialization and embedded config.

Browse Source 
PR #8118 - adding to_dict to the dependency manager and an embedded loading of dependency resolution options from galaxy.yml - was really quite poorly thought through.

Certain options could be embedded - but only the resolvers and the actual dictified representation of the DependencyManager couldn't be actually embedded properly in the app config - but this is exactly what I wanted to do on the Pulsar side and it would make no sense to have the exact same options in both apps and configured differently.

So now the actual dictified representation of the whole DependencyManager can be directly embedded in galaxy.yml as the ``dependency_resolution:`` parameter. All the parameters can still be configured at the top level of the YAML file but with 'dependency' in the config option name (some prefixes, some suffixes, etc... a hodgepodge that has grown organically). Give the messiness of the existing options might be better to encourage using ``dependency_resolution:`` in the future.
This commit is contained in:
John Chilton
2019-06-20 14:36:40 -04:00
parent 1b4ff60f06
commit 450ae76ddb
18 changed files with 268 additions and 121 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/galaxy/config.py
+13 -42
View File
@@ -501,24 +501,16 @@ class Configuration(object):
self.tool_ngram_maxsize = kwargs.get("tool_ngram_maxsize", 4)
default_tool_test_data_directories = os.environ.get("GALAXY_TEST_FILE_DIR", resolve_path("test-data", self.root))
self.tool_test_data_directories = kwargs.get("tool_test_data_directories", default_tool_test_data_directories)
# Location for tool dependencies.
use_tool_dependencies, tool_dependency_dir, use_cached_dependency_manager, tool_dependency_cache_dir, precache_dependencies = \
parse_dependency_options(kwargs, self.root, self.dependency_resolvers_config_file)
self.use_tool_dependencies = use_tool_dependencies
self.tool_dependency_dir = tool_dependency_dir
self.use_cached_dependency_manager = use_cached_dependency_manager
self.tool_dependency_cache_dir = tool_dependency_cache_dir
self.precache_dependencies = precache_dependencies
self.dependency_resolvers = kwargs.get("dependency_resolvers")
# Deployers may either specify a complete list of mapping files or get the default for free and just
# specify a local mapping file to adapt and extend the default one.
if "conda_mapping_files" in kwargs:
self.conda_mapping_files = kwargs["conda_mapping_files"]
else:
self.conda_mapping_files = [
if "conda_mapping_files" not in kwargs:
conda_mapping_files = [
self.local_conda_mapping_file,
os.path.join(self.root, "lib", "galaxy", "tool_util", "deps", "resolvers", "default_conda_mapping.yml"),
]
# dependency resolution options are consumed via config_dict - so don't populate
# self, populate config_dict
self.config_dict["conda_mapping_files"] = conda_mapping_files
self.enable_beta_mulled_containers = string_as_bool(kwargs.get('enable_beta_mulled_containers', 'False'))
containers_resolvers_config_file = kwargs.get('containers_resolvers_config_file', None)
@@ -528,7 +520,10 @@ class Configuration(object):
involucro_path = kwargs.get('involucro_path', None)
if involucro_path is None:
involucro_path = os.path.join(tool_dependency_dir or "database", "involucro")
target_dir = kwargs.get("tool_dependency_dir", "database/dependencies")
if target_dir == "none":
target_dir = "database"
involucro_path = os.path.join(target_dir, "involucro")
self.involucro_path = resolve_path(involucro_path, self.root)
self.involucro_auto_init = string_as_bool(kwargs.get('involucro_auto_init', True))
mulled_channels = kwargs.get('mulled_channels')
@@ -921,34 +916,6 @@ class Configuration(object):
return [parse(v) for v in allowed_origin_hostnames if v]
def parse_dependency_options(kwargs, root, dependency_resolvers_config_file):
# Location for tool dependencies.
tool_dependency_dir = kwargs.get("tool_dependency_dir", "database/dependencies")
if tool_dependency_dir.lower() == "none":
tool_dependency_dir = None
if tool_dependency_dir is not None:
tool_dependency_dir = resolve_path(tool_dependency_dir, root)
# Setting the following flag to true will ultimately cause tool dependencies
# to be located in the shell environment and used by the job that is executing
# the tool.
use_tool_dependencies = True
tool_dependency_cache_dir = kwargs.get('tool_dependency_cache_dir', os.path.join(tool_dependency_dir, '_cache'))
use_cached_dependency_manager = string_as_bool(kwargs.get("use_cached_dependency_manager", 'False'))
precache_dependencies = string_as_bool(kwargs.get("precache_dependencies", 'True'))
else:
tool_dependency_dir = None
if dependency_resolvers_config_file is None:
use_tool_dependencies = bool(kwargs.get("dependency_resolvers", None))
else:
use_tool_dependencies = os.path.exists(dependency_resolvers_config_file)
tool_dependency_cache_dir = None
precache_dependencies = False
use_cached_dependency_manager = False
return use_tool_dependencies, tool_dependency_dir, use_cached_dependency_manager, tool_dependency_cache_dir, precache_dependencies
def get_database_engine_options(kwargs, model_prefix=''):
"""
Allow options for the SQLAlchemy database engine to be passed by using
@@ -1225,3 +1192,7 @@ class ConfiguresGalaxyMixin(object):
except Exception:
log.info("Waiting for database: attempt %d of %d" % (i, attempts))
time.sleep(pause)
@property
def tool_dependency_dir(self):
return self.toolbox.dependency_manager.default_base_path
lib/galaxy/model/tool_shed_install/__init__.py
+2 -2
View File
@@ -555,14 +555,14 @@ class ToolDependency(object):
def installation_directory(self, app):
if self.type == 'package':
return os.path.join(app.config.tool_dependency_dir,
return os.path.join(app.tool_dependency_dir,
self.name,
self.version,
self.tool_shed_repository.owner,
self.tool_shed_repository.name,
self.tool_shed_repository.installed_changeset_revision)
if self.type == 'set_environment':
return os.path.join(app.config.tool_dependency_dir,
return os.path.join(app.tool_dependency_dir,
'environment_settings',
self.name,
self.tool_shed_repository.owner,
lib/galaxy/tool_util/deps/__init__.py
+67 -26
View File
@@ -10,7 +10,8 @@ from collections import OrderedDict
from galaxy.util import (
hash_util,
plugin_config
plugin_config,
string_as_bool,
)
from galaxy.util.oset import OrderedSet
from .container_resolvers import ContainerResolver
@@ -31,27 +32,59 @@ log = logging.getLogger(__name__)
CONFIG_VAL_NOT_FOUND = object()
def build_dependency_manager(config):
config_dict = {
"use_tool_dependencies": getattr(config, "use_tool_dependencies", False),
"default_base_path": getattr(config, "tool_dependency_dir", None),
"conf_file": getattr(config, "dependency_resolvers_config_file", None),
"cache": getattr(config, "use_cached_dependency_manager", False),
"app_config": config,
"dependency_resolvers": getattr(config, "dependency_resolvers", None),
}
return build_dependency_manager_from_dict(config_dict)
def build_dependency_manager(app_config_dict=None, resolution_config_dict=None, conf_file=None, default_tool_dependency_dir=None):
"""Build a DependencyManager object from app and/or resolution config.
If app_config_dict is specified, it should be application configuration information
and configuration options are generally named to identify the context is dependency
management (e.g. conda_prefix not prefix or use_cached_dependency_manager not cache).
resolution_config_dict if specified is assumed to be the to_dict() version of a
DependencyManager and should only contain dependency configuration options.
"""
def build_dependency_manager_from_dict(config_dict):
if config_dict.get("use_tool_dependencies", False):
dependency_manager_kwds = {
'default_base_path': config_dict.get("default_base_path"),
'conf_file': config_dict.get("conf_file"),
'app_config': config_dict.get("app_config"),
"dependency_resolver_dicts": config_dict.get("dependency_resolvers"),
if app_config_dict is None:
app_config_dict = {}
else:
app_config_dict = app_config_dict.copy()
tool_dependency_dir = app_config_dict.get("tool_dependency_dir", default_tool_dependency_dir)
if tool_dependency_dir and tool_dependency_dir.lower() == "none":
app_config_dict["tool_dependency_dir"] = None
if resolution_config_dict is None and "dependency_resolution" in app_config_dict:
resolution_config_dict = app_config_dict["dependency_resolution"]
if resolution_config_dict:
# Convert local to_dict options into global ones.
# to_dict() has "cache", "cache_dir", "use", "default_base_path", "resolvers", "precache"
app_config_props_from_resolution_config = {
"use_tool_dependencies": resolution_config_dict.get("use", None),
"tool_dependency_dir": resolution_config_dict.get("default_base_path", None),
"dependency_resolvers": resolution_config_dict.get("resolvers", None),
"tool_dependency_cache_dir": resolution_config_dict.get("cache_dir", None),
"precache_dependencies": resolution_config_dict.get("precache", None),
"use_cached_dependency_manager": resolution_config_dict.get("cache", None),
}
if config_dict.get("cache"):
for key, value in app_config_props_from_resolution_config.items():
if value is not None:
app_config_dict[key] = value
use_tool_dependencies = app_config_dict.get("use_tool_dependencies", None)
# if we haven't set an explicit True or False, try to infer from config...
if use_tool_dependencies is None:
use_tool_dependencies = app_config_dict.get("tool_dependency_dir", default_tool_dependency_dir) is not None or \
app_config_dict.get("dependency_resolvers") or \
(conf_file and os.path.exists(conf_file))
if use_tool_dependencies:
dependency_manager_kwds = {
"default_base_path": app_config_dict.get("tool_dependency_dir", default_tool_dependency_dir),
"conf_file": conf_file,
"app_config": app_config_dict,
}
if string_as_bool(app_config_dict.get("use_cached_dependency_manager")):
dependency_manager = CachedDependencyManager(**dependency_manager_kwds)
else:
dependency_manager = DependencyManager(**dependency_manager_kwds)
@@ -74,7 +107,7 @@ class DependencyManager(object):
"""
cached = False
def __init__(self, default_base_path, conf_file=None, app_config={}, dependency_resolver_dicts=None):
def __init__(self, default_base_path, conf_file=None, app_config={}):
"""
Create a new dependency manager looking for packages under the paths listed
in `base_paths`. The default base path is app.config.tool_dependency_dir.
@@ -88,6 +121,7 @@ class DependencyManager(object):
self.resolver_classes = self.__resolvers_dict()
plugin_source = None
dependency_resolver_dicts = app_config.get("dependency_resolvers")
if dependency_resolver_dicts is not None:
plugin_source = ('dict', dependency_resolver_dicts)
else:
@@ -139,6 +173,10 @@ class DependencyManager(object):
value = default
return value
@property
def precache(self):
return string_as_bool(self.get_app_option("precache_dependencies", True))
def dependency_shell_commands(self, requirements, **kwds):
requirements_to_dependencies = self.requirements_to_dependencies(requirements, **kwds)
ordered_dependencies = OrderedSet(requirements_to_dependencies.values())
@@ -281,11 +319,12 @@ class DependencyManager(object):
def to_dict(self):
return {
"use": True,
"cache": self.cached,
"use_tool_dependencies": True,
"precache": self.precache,
"cache_dir": getattr(self, "tool_dependency_cache_dir", None),
"default_base_path": self.default_base_path,
"dependency_resolvers": [m.to_dict() for m in self.dependency_resolvers],
"tool_dependency_cache_dir": getattr(self, "tool_dependency_cache_dir", None),
"resolvers": [m.to_dict() for m in self.dependency_resolvers],
}
@@ -294,7 +333,7 @@ class CachedDependencyManager(DependencyManager):
def __init__(self, default_base_path, **kwd):
super(CachedDependencyManager, self).__init__(default_base_path=default_base_path, **kwd)
self.tool_dependency_cache_dir = self.get_app_option("tool_dependency_cache_dir")
self.tool_dependency_cache_dir = self.get_app_option("tool_dependency_cache_dir") or os.path.join(default_base_path, "_cache")
def build_cache(self, requirements, **kwds):
resolved_dependencies = self.requirements_to_dependencies(requirements, **kwds)
@@ -323,7 +362,7 @@ class CachedDependencyManager(DependencyManager):
resolved_dependencies = self.requirements_to_dependencies(requirements, **kwds)
cacheable_dependencies = [dep for dep in resolved_dependencies.values() if dep.cacheable]
hashed_dependencies_dir = self.get_hashed_dependencies_path(cacheable_dependencies)
if not os.path.exists(hashed_dependencies_dir) and self.get_app_option("precache_dependencies", False):
if not os.path.exists(hashed_dependencies_dir) and self.precache:
# Cache not present, try to create it
self.build_cache(requirements, **kwds)
if os.path.exists(hashed_dependencies_dir):
@@ -352,6 +391,7 @@ class CachedDependencyManager(DependencyManager):
class NullDependencyManager(DependencyManager):
cached = False
def __init__(self, default_base_path=None, conf_file=None, app_config={}):
self.__app_config = app_config
@@ -359,6 +399,7 @@ class NullDependencyManager(DependencyManager):
self.dependency_resolvers = []
self._enabled_container_types = []
self._destination_for_container_type = {}
self.default_base_path = None
def uses_tool_shed_dependencies(self):
return False
@@ -370,4 +411,4 @@ class NullDependencyManager(DependencyManager):
return NullDependency(version=version, name=name)
def to_dict(self):
return {"use_tool_dependencies": False}
return {"use": False}
lib/galaxy/tool_util/deps/resolvers/conda.py
+2 -1
View File
@@ -65,7 +65,7 @@ log = logging.getLogger(__name__)
class CondaDependencyResolver(DependencyResolver, MultipleDependencyResolver, ListableDependencyResolver, InstallableDependencyResolver, SpecificationPatternDependencyResolver, MappableDependencyResolver):
dict_collection_visible_keys = DependencyResolver.dict_collection_visible_keys + ['conda_prefix', 'versionless', 'ensure_channels', 'auto_install', 'auto_init']
dict_collection_visible_keys = DependencyResolver.dict_collection_visible_keys + ['conda_prefix', 'versionless', 'ensure_channels', 'auto_install', 'auto_init', 'use_local']
resolver_type = "conda"
config_options = {
'prefix': None,
@@ -127,6 +127,7 @@ class CondaDependencyResolver(DependencyResolver, MultipleDependencyResolver, Li
copy_dependencies=copy_dependencies,
use_local=use_local,
)
self.use_local = use_local
self.ensure_channels = ensure_channels
# Conda operations options (these define how resolution will occur)
lib/galaxy/tools/toolbox/base.py
+3 -1
View File
@@ -1139,7 +1139,9 @@ class BaseGalaxyToolBox(AbstractToolBox):
return looks_like_a_tool(path, enable_beta_formats=getattr(self.app.config, "enable_beta_tool_formats", False))
def _init_dependency_manager(self):
self.dependency_manager = build_dependency_manager(self.app.config)
app_config_dict = self.app.config.config_dict
conf_file = app_config_dict.get("dependency_resolvers_config_file")
self.dependency_manager = build_dependency_manager(app_config_dict=app_config_dict, conf_file=conf_file, default_tool_dependency_dir="database/dependencies")
def reload_dependency_manager(self):
self._init_dependency_manager()
lib/galaxy/webapps/galaxy/config_schema.yml
+34 -5
View File
@@ -219,13 +219,13 @@ mapping:
default: database/dependencies
required: false
desc: |
Path to the directory in which tool dependencies are placed. This is used by
the Tool Shed to install dependencies and can also be used by administrators
to manually install or link to dependencies. For details, see:
https://galaxyproject.org/admin/config/tool-dependencies
Various dependency resolver configuration parameters will have defaults set relative
to this path, such as the default conda prefix, default Galaxy packages path, legacy
tool shed dependencies path, and the dependency cache directory.
Set the string to None to explicitly disable tool dependency handling.
If this option is set to none or an invalid path, installing tools with dependencies
from the Tool Shed will fail.
from the Tool Shed or in Conda will fail.
dependency_resolvers_config_file:
type: str
@@ -316,6 +316,10 @@ mapping:
used if you experience long waiting times before a job is actually submitted
to your cluster.
This only affects tools where some requirements can be resolved but not others,
most modern best practice tools can use prebuilt environments in the Conda
directory.
tool_dependency_cache_dir:
type: str
default: <tool_dependency_dir>/_cache
@@ -2371,6 +2375,31 @@ mapping:
resolvers to enable can be embedded into Galaxy's config with this option.
This has no effect if a dependency_resolvers_config_file is used.
dependency_resolution:
type: map
desc: |
Alternative representation of various dependency resolution parameters. Takes the
dictified version of a DependencyManager object - so this is ideal for automating the
configuration of dependency resolution from one application that uses a DependencyManager
to another.
mapping:
cache:
type: bool
default: false
desc: See description of 'use_cached_dependency_manager'.
precache:
type: bool
default: true
desc: See description of 'precache_dependencies'.
cache_dir:
type: str
default: '<tool_dependency_cache_dir>/_cache'
desc: See description of 'tool_dependency_cache_dir'.
default_base_path:
default: database/dependencies
desc: See description of 'tool_dependency_dir'.
resolvers:
desc: See description of 'dependency_resolvers'.
default_job_resubmission_condition:
type: str
required: false
lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py
+8 -8
View File
@@ -524,11 +524,11 @@ class AdminToolshed(AdminGalaxy):
(escape(str(repository.name)), updating_to_changeset_revision)
self.initiate_tool_dependency_installation(trans, tool_dependencies, message=message, status=status)
# Handle tool dependencies check box.
if trans.app.config.tool_dependency_dir is None:
if not trans.app.toolbox.dependency_manager.uses_tool_shed_dependencies:
if tool_dependencies_dict:
message = ("Tool dependencies defined in this repository can be automatically installed if you set "
"the value of your <b>tool_dependency_dir</b> setting in your Galaxy config file "
"(galaxy.ini) and restart your Galaxy server.")
"(galaxy.yml) and restart your Galaxy server.")
status = "warning"
install_tool_dependencies_check_box_checked = False
else:
@@ -696,7 +696,7 @@ class AdminToolshed(AdminGalaxy):
message = 'No selected tool dependencies can be uninstalled, you may need to use the <b>Repair repository</b> feature.'
status = 'error'
elif operation == "install":
if trans.app.config.tool_dependency_dir:
if trans.app.toolbox.dependency_manager.uses_tool_shed_dependencies:
tool_dependencies_for_installation = []
for tool_dependency_id in tool_dependency_ids:
tool_dependency = tool_dependency_util.get_tool_dependency(trans.app, tool_dependency_id)
@@ -756,7 +756,7 @@ class AdminToolshed(AdminGalaxy):
action='browse_tool_dependency',
**kwd))
elif operation == "install":
if trans.app.config.tool_dependency_dir:
if trans.app.toolbox.dependency_manager.uses_tool_shed_dependencies:
tool_dependencies_for_installation = []
for tool_dependency_id in tool_dependency_ids:
tool_dependency = tool_dependency_util.get_tool_dependency(trans.app, tool_dependency_id)
@@ -1048,11 +1048,11 @@ class AdminToolshed(AdminGalaxy):
# Merge all containers into a single container.
containers_dict = dd.merge_containers_dicts_for_new_install(containers_dicts)
# Handle tool dependencies check box.
if trans.app.config.tool_dependency_dir is None:
if not trans.app.toolbox.dependency_manager.uses_tool_shed_dependencies:
if includes_tool_dependencies:
message = "Tool dependencies defined in this repository can be automatically installed if you set "
message += "the value of your <b>tool_dependency_dir</b> setting in your Galaxy config file "
message += "(galaxy.ini) and restart your Galaxy server before installing the repository."
message += "(galaxy.yml) and restart your Galaxy server before installing the repository."
status = "warning"
install_tool_dependencies_check_box_checked = False
else:
@@ -1429,10 +1429,10 @@ class AdminToolshed(AdminGalaxy):
# Handle repository dependencies check box.
install_repository_dependencies_check_box = CheckboxField('install_repository_dependencies', value=True)
# Handle tool dependencies check box.
if trans.app.config.tool_dependency_dir is None:
if not trans.app.toolbox.dependency_manager.uses_tool_shed_dependencies:
if includes_tool_dependencies:
message += "Tool dependencies defined in this repository can be automatically installed if you set the value of your <b>tool_dependency_dir</b> "
message += "setting in your Galaxy config file (galaxy.ini) and restart your Galaxy server before installing the repository. "
message += "setting in your Galaxy config file (galaxy.yml) and restart your Galaxy server before installing the repository. "
status = "warning"
install_tool_dependencies_check_box_checked = False
else:
lib/tool_shed/galaxy_install/dependency_display.py
+2 -2
View File
@@ -37,8 +37,8 @@ class DependencyDisplayer(object):
changeset_revision = requirements_dict.get('changeset_revision', 'unknown')
dependency_name = requirements_dict['name']
version = requirements_dict['version']
if self.app.config.tool_dependency_dir:
root_dir = self.app.config.tool_dependency_dir
if self.app.tool_dependency_dir:
root_dir = self.app.tool_dependency_dir
else:
root_dir = '<set your tool_dependency_dir in your Galaxy configuration file>'
install_dir = os.path.join(root_dir,
lib/tool_shed/galaxy_install/install_manager.py
+4 -3
View File
@@ -899,8 +899,9 @@ class InstallRepositoryManager(object):
new_tools = [self.app.toolbox._tools_by_id.get(tool_d['guid'], None) for tool_d in metadata['tools']]
new_requirements = set([tool.requirements.packages for tool in new_tools if tool])
[self._view.install_dependencies(r) for r in new_requirements]
if self.app.config.use_cached_dependency_manager:
[self.app.toolbox.dependency_manager.build_cache(r) for r in new_requirements]
dependency_manager = self.app.toolbox.dependency_manager
if dependency_manager.cached:
[dependency_manager.build_cache(r) for r in new_requirements]
if install_tool_dependencies and tool_shed_repository.tool_dependencies and 'tool_dependencies' in metadata:
work_dir = tempfile.mkdtemp(prefix="tmp-toolshed-itsr")
@@ -992,7 +993,7 @@ class InstallRepositoryManager(object):
self.install_model.context.flush()
def __assert_can_install_dependencies(self):
if self.app.config.tool_dependency_dir is None:
if self.app.tool_dependency_dir is None:
no_tool_dependency_dir_message = "Tool dependencies can be automatically installed only if you set "
no_tool_dependency_dir_message += "the value of your 'tool_dependency_dir' setting in your Galaxy "
no_tool_dependency_dir_message += "configuration file (galaxy.ini) and restart your Galaxy server. "
lib/tool_shed/galaxy_install/tool_dependencies/recipe/step_handler.py
+1 -1
View File
@@ -1576,7 +1576,7 @@ class SetupVirtualEnv(Download, RecipeStep):
# This class is not currently used during stage 1 of the installation process, so filter_actions
# are not affected, and dir is not set. Enhancements can easily be made to this function if this
# class is needed in stage 1.
venv_src_directory = os.path.abspath(os.path.join(self.app.config.tool_dependency_dir, '__virtualenv_src'))
venv_src_directory = os.path.abspath(os.path.join(self.app.tool_dependency_dir, '__virtualenv_src'))
if not self.install_virtualenv(install_environment, venv_src_directory):
log.debug('Unable to install virtualenv')
return tool_dependency, None, None
lib/tool_shed/galaxy_install/tool_migration_manager.py
+1 -1
View File
@@ -47,7 +47,7 @@ class ToolMigrationManager(object):
self.tpm = tool_panel_manager.ToolPanelManager(self.app)
# If install_dependencies is True but tool_dependency_dir is not set, do not attempt
# to install but print informative error message.
if install_dependencies and app.config.tool_dependency_dir is None:
if install_dependencies and app.tool_dependency_dir is None:
message = 'You are attempting to install tool dependencies but do not have a value '
message += 'for "tool_dependency_dir" set in your galaxy.ini file. Set this '
message += 'location value to the path where you want tool dependencies installed and '
lib/tool_shed/util/tool_dependency_util.py
+2 -2
View File
@@ -207,14 +207,14 @@ def get_tool_dependency_ids(as_string=False, **kwd):
def get_tool_dependency_install_dir(app, repository_name, repository_owner, repository_changeset_revision, tool_dependency_type,
tool_dependency_name, tool_dependency_version):
if tool_dependency_type == 'package':
return os.path.abspath(os.path.join(app.config.tool_dependency_dir,
return os.path.abspath(os.path.join(app.tool_dependency_dir,
tool_dependency_name,
tool_dependency_version,
repository_owner,
repository_name,
repository_changeset_revision))
if tool_dependency_type == 'set_environment':
return os.path.abspath(os.path.join(app.config.tool_dependency_dir,
return os.path.abspath(os.path.join(app.tool_dependency_dir,
'environment_settings',
tool_dependency_name,
repository_owner,
scripts/manage_tool_dependencies.py
+2 -20
View File
@@ -7,9 +7,8 @@ from galaxy.config import (
configure_logging,
find_path,
find_root,
parse_dependency_options,
)
from galaxy.tool_util.deps import CachedDependencyManager, DependencyManager, NullDependencyManager
from galaxy.tool_util.deps import build_dependency_manager
from galaxy.util.script import main_factory
DESCRIPTION = "Script to manage tool dependencies (with focus on a Conda environments)."
@@ -29,24 +28,7 @@ def _build_dependency_manager_no_config(kwargs):
configure_logging(kwargs)
root = find_root(kwargs)
dependency_resolvers_config_file = find_path(kwargs, "dependency_resolvers_config_file", root)
use_dependencies, tool_dependency_dir, use_cached_dependency_manager, tool_dependency_cache_dir, precache_dependencies = \
parse_dependency_options(kwargs, root, dependency_resolvers_config_file)
if not use_dependencies:
dependency_manager = NullDependencyManager()
else:
dependency_manager_kwds = {
'default_base_path': tool_dependency_dir,
'conf_file': dependency_resolvers_config_file,
'app_config': kwargs,
}
if use_cached_dependency_manager:
dependency_manager_kwds['tool_dependency_cache_dir'] = tool_dependency_cache_dir
dependency_manager = CachedDependencyManager(**dependency_manager_kwds)
else:
dependency_manager = DependencyManager(**dependency_manager_kwds)
dependency_manager = build_dependency_manager(app_config_dict=kwargs, conf_file=dependency_resolvers_config_file, default_tool_dependency_dir="database/dependencies")
return dependency_manager
templates/admin/tool_shed_repository/common.mako
+1 -1
View File
@@ -189,7 +189,7 @@
%if install_tool_dependencies_check_box is not None:
<div class="form-row">
<label>When available, install Tool Shed managed tool dependencies?</label>
<% disabled = trans.app.config.tool_dependency_dir is None %>
<% disabled = trans.app.tool_dependency_dir is None %>
${render_checkbox(install_tool_dependencies_check_box, disabled=disabled)}
<div class="toolParamHelp" style="clear: both;">
%if disabled:
templates/admin/tool_shed_repository/install_tool_dependencies_with_update.mako
+2 -2
View File
@@ -31,7 +31,7 @@ ${render_galaxy_repository_actions( repository )}
%if install_tool_dependencies_check_box is not None:
<div class="form-row">
<label>Handle tool dependencies?</label>
<% disabled = trans.app.config.tool_dependency_dir is None %>
<% disabled = trans.app.tool_dependency_dir is None %>
${render_checkbox(install_tool_dependencies_check_box, disabled=disabled)}
<div class="toolParamHelp" style="clear: both;">
%if disabled:
@@ -63,7 +63,7 @@ ${render_galaxy_repository_actions( repository )}
key_name = key_items[ 0 ]
key_version = key_items[ 1 ]
readme_text = requirements_dict.get( 'readme', None )
install_dir = os.path.join( trans.app.config.tool_dependency_dir,
install_dir = os.path.join( trans.app.tool_dependency_dir,
key_name,
key_version,
repository.owner,
templates/admin/tool_shed_repository/uninstall_tool_dependencies.mako
+2 -2
View File
@@ -26,14 +26,14 @@ ${render_galaxy_repository_actions( repository )}
<input type="hidden" name="tool_dependency_ids" value="${trans.security.encode_id( tool_dependency.id )}"/>
<%
if tool_dependency.type == 'package':
install_dir = os.path.join( trans.app.config.tool_dependency_dir,
install_dir = os.path.join( trans.app.tool_dependency_dir,
tool_dependency.name,
tool_dependency.version,
tool_dependency.tool_shed_repository.owner,
tool_dependency.tool_shed_repository.name,
tool_dependency.tool_shed_repository.installed_changeset_revision )
elif tool_dependency.type == 'set_environment':
install_dir = os.path.join( trans.app.config.tool_dependency_dir,
install_dir = os.path.join( trans.app.tool_dependency_dir,
'environment_settings',
tool_dependency.name,
tool_dependency.tool_shed_repository.owner,
test/unit/tools/test_tool_deps.py
+118 -2
View File
@@ -12,7 +12,7 @@ from shutil import rmtree
from stat import S_IXUSR
from subprocess import PIPE, Popen
from galaxy.tool_util.deps import build_dependency_manager_from_dict, DependencyManager
from galaxy.tool_util.deps import build_dependency_manager, DependencyManager
from galaxy.tool_util.deps.requirements import (
ToolRequirement,
ToolRequirements
@@ -610,6 +610,114 @@ def test_config_module_directory_searcher():
assert module_resolver.module_checker.directories == ["/opt/Modules/modulefiles"]
def test_dependency_manager_config_options_global():
app_config = {
"tool_dependency_dir": "/tmp",
"tool_dependency_cache_dir": "/tmp",
"conda_auto_init": False,
}
dm = __dependency_manager_for_config(app_config.copy())
assert not dm.to_dict()["cache"]
app_config["use_cached_dependency_manager"] = True
dm = __dependency_manager_for_config(app_config.copy())
assert dm.to_dict()["cache"]
assert dm.to_dict()["precache"]
app_config["precache_dependencies"] = False
dm = __dependency_manager_for_config(app_config.copy())
assert not dm.to_dict()["precache"]
conda_opts = _first_conda_resolver_options(dm)
assert conda_opts["use_local"] is False
app_config["conda_use_local"] = True
dm = __dependency_manager_for_config(app_config.copy())
conda_opts = _first_conda_resolver_options(dm)
assert conda_opts["use_local"] is True
def test_dependency_manager_config_options_embedded_config():
app_config = {
"dependency_resolution": {
"default_base_path": "/tmp",
"cache_dir": "/tmp",
},
"conda_auto_init": False,
}
dm = __dependency_manager_for_config(app_config.copy())
assert not dm.to_dict()["cache"]
app_config["dependency_resolution"]["cache"] = True
dm = __dependency_manager_for_config(app_config.copy())
assert dm.to_dict()["cache"]
assert dm.to_dict()["precache"]
app_config["dependency_resolution"]["precache"] = False
dm = __dependency_manager_for_config(app_config.copy())
assert not dm.to_dict()["precache"]
conda_opts = _first_conda_resolver_options(dm)
assert conda_opts["use_local"] is False
app_config["conda_use_local"] = True
dm = __dependency_manager_for_config(app_config.copy())
conda_opts = _first_conda_resolver_options(dm)
assert conda_opts["use_local"] is True
def test_dependency_manager_config_options_resolution_config():
app_config = {
"conda_auto_init": False,
}
resolution_config = {
"default_base_path": "/tmp",
"cache_dir": "/tmp",
}
dm = __dependency_manager_for_config(app_config.copy(), resolution_config=resolution_config.copy())
assert not dm.to_dict()["cache"]
resolution_config["cache"] = True
dm = __dependency_manager_for_config(app_config.copy(), resolution_config=resolution_config.copy())
assert dm.to_dict()["cache"]
assert dm.to_dict()["precache"]
resolution_config["precache"] = False
dm = __dependency_manager_for_config(app_config.copy(), resolution_config=resolution_config.copy())
assert not dm.to_dict()["precache"]
conda_opts = _first_conda_resolver_options(dm)
assert conda_opts["use_local"] is False
app_config["conda_use_local"] = True
dm = __dependency_manager_for_config(app_config.copy(), resolution_config=resolution_config.copy())
conda_opts = _first_conda_resolver_options(dm)
assert conda_opts["use_local"] is True
def test_dependency_manager_none():
# by default tool_dependency_dir will be use to create some default resolvers...
app_config = {
"conda_auto_init": False,
"tool_dependency_dir": "some_not_none_value",
}
dm = __dependency_manager_for_config(app_config.copy())
assert dm.to_dict()["use"]
# but setting it none disables dependency resolution unless explicit resolvers
# are configured.
app_config = {
"conda_auto_init": False,
"tool_dependency_dir": "none",
}
dm = __dependency_manager_for_config(app_config.copy())
assert not dm.to_dict()["use"]
def _first_conda_resolver_options(dm):
return [r for r in dm.to_dict()["resolvers"] if r["resolver_type"] == "conda"][0]
@contextmanager
def __environ(values, remove=[]):
"""
@@ -652,7 +760,15 @@ def __dependency_manager_for_base_path(default_base_path, conf_file=None):
dm = DependencyManager(default_base_path=default_base_path, conf_file=conf_file, app_config={"conda_auto_init": False})
if ROUND_TRIP_TEST_DEPENDENCY_MANAGER_SERIALIZATION:
as_dict = dm.to_dict()
dm = build_dependency_manager_from_dict(as_dict)
dm = build_dependency_manager(resolution_config_dict=as_dict)
return dm
def __dependency_manager_for_config(app_config, resolution_config=None):
dm = build_dependency_manager(app_config_dict=app_config, resolution_config_dict=resolution_config)
if ROUND_TRIP_TEST_DEPENDENCY_MANAGER_SERIALIZATION:
as_dict = dm.to_dict()
dm = build_dependency_manager(resolution_config_dict=as_dict)
return dm
test/unit/unittest_utils/galaxy_mock.py
+4
View File
@@ -151,6 +151,10 @@ class MockAppConfig(Bunch):
# set by MockDir
self.root = root
@property
def config_dict(self):
return self.dict()
class MockWebapp(object):