opensource/galaxy
3
0
Fork 0
mirror of https://github.com/galaxyproject/galaxy.git synced 2026-09-24 16:30:27 +08:00
Code Issues Packages Projects Releases Wiki Activity

More modular config option handling for dependency resolvers.

Browse Source 
The handling of global configuration options for Conda specific options was hacky and a poor design. This is much more modular and allows the elegant cascading option sources for all resolvers.
This commit is contained in:
John Chilton
2017-01-23 09:47:17 -05:00
parent 330d71d37c
commit baf9b82847
3 changed files with 38 additions and 41 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/galaxy/tools/deps/__init__.py
+27 -28
View File
@@ -19,26 +19,12 @@ from .requirements import (
ToolRequirements
)
from .resolvers import NullDependency
from .resolvers.conda import CondaDependencyResolver, DEFAULT_ENSURE_CHANNELS
from .resolvers.conda import CondaDependencyResolver
from .resolvers.galaxy_packages import GalaxyPackageDependencyResolver
from .resolvers.tool_shed_packages import ToolShedPackageDependencyResolver
log = logging.getLogger( __name__ )
# TODO: Load these from the plugins. Would require a two step initialization of
# DependencyManager - where the plugins are loaded first and then the config
# is parsed and sent through.
EXTRA_CONFIG_KWDS = {
'conda_prefix': None,
'conda_exec': None,
'conda_debug': None,
'conda_ensure_channels': DEFAULT_ENSURE_CHANNELS,
'conda_auto_install': False,
'conda_auto_init': False,
'conda_copy_dependencies': False,
'precache_dependencies': True,
}
CONFIG_VAL_NOT_FOUND = object()
@@ -47,15 +33,9 @@ def build_dependency_manager( config ):
dependency_manager_kwds = {
'default_base_path': config.tool_dependency_dir,
'conf_file': config.dependency_resolvers_config_file,
'app_config': config,
}
for key, default_value in EXTRA_CONFIG_KWDS.items():
value = getattr(config, key, CONFIG_VAL_NOT_FOUND)
if value is CONFIG_VAL_NOT_FOUND and hasattr(config, "config_dict"):
value = config.config_dict.get(key, CONFIG_VAL_NOT_FOUND)
if value is CONFIG_VAL_NOT_FOUND:
value = default_value
dependency_manager_kwds[key] = value
if config.use_cached_dependency_manager:
if getattr(config, "use_cached_dependency_manager", False):
dependency_manager_kwds['tool_dependency_cache_dir'] = config.tool_dependency_cache_dir
dependency_manager = CachedDependencyManager(**dependency_manager_kwds)
else:
@@ -90,7 +70,7 @@ class DependencyManager( object ):
and should each contain a file 'env.sh' which can be sourced to make the
dependency available in the current shell environment.
"""
def __init__( self, default_base_path, conf_file=None, **extra_config ):
def __init__( self, default_base_path, conf_file=None, app_config={} ):
"""
Create a new dependency manager looking for packages under the paths listed
in `base_paths`. The default base path is app.config.tool_dependency_dir.
@@ -99,11 +79,30 @@ class DependencyManager( object ):
log.warning( "Path '%s' does not exist, ignoring", default_base_path )
if not os.path.isdir( default_base_path ):
log.warning( "Path '%s' is not directory, ignoring", default_base_path )
self.extra_config = extra_config
self.__app_config = app_config
self.default_base_path = os.path.abspath( default_base_path )
self.resolver_classes = self.__resolvers_dict()
self.dependency_resolvers = self.__build_dependency_resolvers( conf_file )
def get_resolver_option(self, resolver, key, explicit_resolver_options={}):
"""Look in resolver-specific settings for option and then fallback to global settings.
"""
default = resolver.config_options.get(key)
config_prefix = resolver.resolver_type
global_key = "%s_%s" % (config_prefix, key)
value = explicit_resolver_options.get(key, CONFIG_VAL_NOT_FOUND)
if value is CONFIG_VAL_NOT_FOUND:
if isinstance(self.__app_config, dict):
value = self.__app_config.get(global_key, CONFIG_VAL_NOT_FOUND)
else:
value = getattr(self.__app_config, global_key, CONFIG_VAL_NOT_FOUND)
if value is CONFIG_VAL_NOT_FOUND and hasattr(self.__app_config, "config_dict"):
value = self.__app_config.config_dict.get(global_key, CONFIG_VAL_NOT_FOUND)
if value is CONFIG_VAL_NOT_FOUND:
value = default
return value
def dependency_shell_commands( self, requirements, **kwds ):
requirement_to_dependency = self.requirements_to_dependencies(requirements, **kwds)
return [dependency.shell_commands(requirement) for requirement, dependency in requirement_to_dependency.items()]
@@ -221,8 +220,8 @@ class DependencyManager( object ):
class CachedDependencyManager(DependencyManager):
def __init__(self, default_base_path, conf_file=None, **extra_config):
super(CachedDependencyManager, self).__init__(default_base_path=default_base_path, conf_file=conf_file, **extra_config)
def __init__(self, default_base_path, conf_file=None, app_config={}):
super(CachedDependencyManager, self).__init__(default_base_path=default_base_path, conf_file=conf_file, app_config=app_config)
def build_cache(self, requirements, **kwds):
resolved_dependencies = self.requirements_to_dependencies(requirements, **kwds)
@@ -251,7 +250,7 @@ class CachedDependencyManager(DependencyManager):
resolved_dependencies = self.requirements_to_dependencies(requirements, **kwds)
cacheable_dependencies = [dep for dep in resolved_dependencies.values() if dep.cacheable]
hashed_dependencies_dir = self.get_hashed_dependencies_path(cacheable_dependencies)
if not os.path.exists(hashed_dependencies_dir) and self.extra_config['precache_dependencies']:
if not os.path.exists(hashed_dependencies_dir) and self.__app_config.getattr("precache_dependencies", False):
# Cache not present, try to create it
self.build_cache(requirements, **kwds)
if os.path.exists(hashed_dependencies_dir):
lib/galaxy/tools/deps/resolvers/__init__.py
+1 -12
View File
@@ -22,6 +22,7 @@ class DependencyResolver(Dictifiable, object):
# resolution.
disabled = False
resolves_simple_dependencies = True
config_options = {}
__metaclass__ = ABCMeta
@abstractmethod
@@ -37,18 +38,6 @@ class DependencyResolver(Dictifiable, object):
request version is 'default'.)
"""
def _get_config_option(self, key, dependency_resolver, default=None, config_prefix=None, **kwds):
""" Look in resolver-specific settings for option and then fallback to
global settings.
"""
global_key = "%s_%s" % (config_prefix, key)
if key in kwds:
return kwds.get(key)
elif global_key in dependency_resolver.extra_config:
return dependency_resolver.extra_config.get(global_key)
else:
return default
class ListableDependencyResolver:
""" Mix this into a ``DependencyResolver`` and implement to indicate
lib/galaxy/tools/deps/resolvers/conda.py
+10 -1
View File
@@ -44,6 +44,15 @@ log = logging.getLogger(__name__)
class CondaDependencyResolver(DependencyResolver, ListableDependencyResolver, InstallableDependencyResolver, SpecificationPatternDependencyResolver):
dict_collection_visible_keys = DependencyResolver.dict_collection_visible_keys + ['conda_prefix', 'versionless', 'ensure_channels', 'auto_install']
resolver_type = "conda"
config_options = {
'prefix': None,
'exec': None,
'debug': None,
'ensure_channels': DEFAULT_ENSURE_CHANNELS,
'auto_install': False,
'auto_init': False,
'copy_dependencies': False,
}
_specification_pattern = re.compile(r"https\:\/\/anaconda.org\/\w+\/\w+")
def __init__(self, dependency_manager, **kwds):
@@ -51,7 +60,7 @@ class CondaDependencyResolver(DependencyResolver, ListableDependencyResolver, In
self.dependency_manager = dependency_manager
def get_option(name):
return self._get_config_option(name, dependency_manager, config_prefix="conda", **kwds)
return dependency_manager.get_resolver_option(self, name, explicit_resolver_options=kwds)
# Conda context options (these define the environment)
conda_prefix = get_option("prefix")