opensource/galaxy
3
0
Fork 0
mirror of https://github.com/galaxyproject/galaxy.git synced 2026-10-25 10:24:38 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #1409 from jmchilton/dep_resolution_api

Browse Source 
Evolving from dependency resolution to dependency management.
This commit is contained in:
Björn Grüning
2016-01-03 19:05:11 +01:00
parent 0aa1c5133e 4650d77bd9
commit 5ca71d6f29
17 changed files with 545 additions and 58 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/galaxy/tools/__init__.py
+2 -7
View File
@@ -25,7 +25,6 @@ from galaxy.tools.actions import DefaultToolAction
from galaxy.tools.actions.upload import UploadToolAction
from galaxy.tools.actions.data_source import DataSourceToolAction
from galaxy.tools.actions.data_manager import DataManagerToolAction
from galaxy.tools.deps import build_dependency_manager
from galaxy.tools.parameters import params_to_incoming, check_param, params_from_strings, params_to_strings, visit_input_values
from galaxy.tools.parameters import output_collect
from galaxy.tools.parameters.basic import (BaseURLToolParameter,
@@ -39,7 +38,7 @@ from galaxy.tools.test import parse_tests
from galaxy.tools.parser import get_tool_source
from galaxy.tools.parser.xml import XmlPageSource
from galaxy.tools.parser import ToolOutputCollectionPart
from galaxy.tools.toolbox import AbstractToolBox
from galaxy.tools.toolbox import BaseGalaxyToolBox
from galaxy.util import rst_to_html, string_as_bool
from galaxy.util import ExecutionTimer
from galaxy.util import listify
@@ -98,7 +97,7 @@ class ToolNotFoundException( Exception ):
pass
class ToolBox( AbstractToolBox ):
class ToolBox( BaseGalaxyToolBox ):
""" A derivative of AbstractToolBox with knowledge about Tool internals -
how to construct them, action types, dependency management, etc....
"""
@@ -109,7 +108,6 @@ class ToolBox( AbstractToolBox ):
tool_root_dir=tool_root_dir,
app=app,
)
self._init_dependency_manager()
@property
def tools_by_id( self ):
@@ -158,9 +156,6 @@ class ToolBox( AbstractToolBox ):
tool = ToolClass( config_file, tool_source, self.app, guid=guid, repository_id=repository_id, **kwds )
return tool
def _init_dependency_manager( self ):
self.dependency_manager = build_dependency_manager( self.app.config )
def handle_datatypes_changed( self ):
""" Refresh upload tools when new datatypes are added. """
for tool_id in self._tools_by_id:
lib/galaxy/tools/deps/__init__.py
+9 -1
View File
@@ -49,6 +49,7 @@ def build_dependency_manager( config ):
class NullDependencyManager( object ):
dependency_resolvers = []
def uses_tool_shed_dependencies(self):
return False
@@ -107,10 +108,17 @@ class DependencyManager( object ):
def find_dep( self, name, version=None, type='package', **kwds ):
log.debug('Find dependency %s version %s' % (name, version))
for resolver in self.dependency_resolvers:
index = kwds.get('index', None)
require_exact = kwds.get('exact', False)
for i, resolver in enumerate(self.dependency_resolvers):
if index is not None and i != index:
continue
dependency = resolver.resolve( name, version, type, **kwds )
log.debug('Resolver %s returned %s (isnull? %s)' % (resolver.resolver_type, dependency,
dependency == INDETERMINATE_DEPENDENCY))
if require_exact and not dependency.exact:
dependency = INDETERMINATE_DEPENDENCY
if dependency != INDETERMINATE_DEPENDENCY:
return dependency
return INDETERMINATE_DEPENDENCY
lib/galaxy/tools/deps/conda_util.py
+37 -8
View File
@@ -22,6 +22,7 @@ IS_OS_X = _platform == "darwin"
CONDA_LICENSE = "http://docs.continuum.io/anaconda/eula"
VERSIONED_ENV_DIR_NAME = re.compile(r"__package__(.*)@__version__(.*)")
UNVERSIONED_ENV_DIR_NAME = re.compile(r"__package__(.*)@__unversioned__")
USE_PATH_EXEC_DEFAULT = False
def conda_link():
@@ -44,8 +45,11 @@ def find_conda_prefix(conda_prefix=None):
class CondaContext(object):
def __init__(self, conda_prefix=None, conda_exec=None,
shell_exec=None, debug=False, ensure_channels=''):
conda_exec = conda_exec or commands.which("conda")
shell_exec=None, debug=False, ensure_channels='',
condarc_override=None, use_path_exec=USE_PATH_EXEC_DEFAULT):
self.condarc_override = condarc_override
if not conda_exec and use_path_exec:
conda_exec = commands.which("conda")
if conda_exec:
conda_exec = os.path.normpath(conda_exec)
self.conda_exec = conda_exec
@@ -65,13 +69,21 @@ class CondaContext(object):
if ensure_channels:
if not isinstance(ensure_channels, list):
ensure_channels = [c for c in ensure_channels.split(",") if c]
else:
ensure_channels = None
self.ensure_channels = ensure_channels
self.ensured_channels = False
def ensure_channels_configured(self):
if not self.ensured_channels:
self.ensured_channels = True
changed = False
conda_conf = self.load_condarc()
if "channels" not in conda_conf:
conda_conf["channels"] = []
channels = conda_conf["channels"]
for channel in ensure_channels:
for channel in self.ensure_channels:
if channel not in channels:
changed = True
channels.append(channel)
@@ -97,12 +109,23 @@ class CondaContext(object):
def save_condarc(self, conf):
condarc = self.condarc
with open(condarc, "w") as f:
return yaml.safe_dump(conf, f)
try:
with open(condarc, "w") as f:
return yaml.safe_dump(conf, f)
except IOError:
template = ("Failed to update write to path [%s] while attempting to update conda configuration, "
"please update the configuration to override the condarc location or "
"grant this application write to the parent directory.")
message = template % condarc
raise Exception(message)
@property
def condarc(self):
return os.path.join(os.path.expanduser("~"), ".condarc")
if self.condarc_override:
return self.condarc_override
else:
home = os.path.expanduser("~")
return os.path.join(home, ".condarc")
def command(self, operation, args):
if isinstance(args, list):
@@ -114,7 +137,11 @@ class CondaContext(object):
def exec_command(self, operation, args):
command = self.command(operation, args)
self.shell_exec(command)
env = {}
condarc_override = self.condarc_override
if condarc_override:
env["CONDARC"] = condarc_override
self.shell_exec(command, env=env)
def exec_create(self, args):
create_base_args = [
@@ -161,7 +188,8 @@ class CondaContext(object):
def installed_conda_targets(conda_context):
envs_path = conda_context.envs_path
for name in os.listdir(envs_path):
dir_contents = os.listdir(envs_path) if os.path.exists(envs_path) else []
for name in dir_contents:
versioned_match = VERSIONED_ENV_DIR_NAME.match(name)
if versioned_match:
yield CondaTarget(versioned_match.group(1), versioned_match.group(2))
@@ -249,6 +277,7 @@ def install_conda_target(conda_target, conda_context=None):
""" Install specified target into a its own environment.
"""
conda_context = _ensure_conda_context(conda_context)
conda_context.ensure_channels_configured()
create_args = [
"--name", conda_target.install_environment, # enviornment for package
conda_target.package_specifier,
lib/galaxy/tools/deps/resolvers/__init__.py
+55 -3
View File
@@ -1,7 +1,19 @@
from abc import ABCMeta, abstractmethod
from galaxy.util.dictifiable import Dictifiable
from ..requirements import ToolRequirement
from abc import ABCMeta, abstractmethod, abstractproperty
class DependencyResolver( object ):
class DependencyResolver(Dictifiable, object):
# Keys for dictification.
dict_collection_visible_keys = ['resolver_type', 'resolves_simple_dependencies']
# A "simple" dependency is one that does not depend on the the tool
# resolving the dependency. Classic tool shed dependencies are non-simple
# because the repository install context is used in dependency resolution
# so the same requirement tags in different tools will have very different
# resolution.
resolves_simple_dependencies = True
__metaclass__ = ABCMeta
@abstractmethod
@@ -28,7 +40,38 @@ class DependencyResolver( object ):
return default
class Dependency( object ):
class ListableDependencyResolver:
""" Mix this into a ``DependencyResolver`` and implement to indicate
the dependency resolver can iterate over its dependencies and generate
requirements.
"""
__metaclass__ = ABCMeta
@abstractmethod
def list_dependencies(self):
""" List the "simple" requirements that may be resolved "exact"-ly
by this dependency resolver.
"""
def _to_requirement(self, name, version=None):
return ToolRequirement(name=name, type="package", version=version)
class InstallableDependencyResolver:
""" Mix this into a ``DependencyResolver`` and implement to indicate
the dependency resolver can attempt to install new dependencies.
"""
__metaclass__ = ABCMeta
@abstractmethod
def install_dependency(self, name, version, type, **kwds):
""" Attempt to install this dependency if a recipe to do so
has been registered in some way.
"""
class Dependency(Dictifiable, object):
dict_collection_visible_keys = ['dependency_type', 'exact']
__metaclass__ = ABCMeta
@abstractmethod
@@ -37,10 +80,19 @@ class Dependency( object ):
Return shell commands to enable this dependency.
"""
@abstractproperty
def exact( self ):
""" Return true if version information wasn't discarded to resolve
the dependency.
"""
class NullDependency( Dependency ):
dependency_type = None
exact = True
def shell_commands( self, requirement ):
return None
INDETERMINATE_DEPENDENCY = NullDependency()
lib/galaxy/tools/deps/resolvers/conda.py
+55 -3
View File
@@ -6,10 +6,14 @@ incompatible changes coming.
import os
from galaxy.exceptions import NotImplemented
from ..resolvers import (
DependencyResolver,
INDETERMINATE_DEPENDENCY,
Dependency,
ListableDependencyResolver,
InstallableDependencyResolver,
)
from ..conda_util import (
CondaContext,
@@ -18,16 +22,20 @@ from ..conda_util import (
is_conda_target_installed,
install_conda_target,
build_isolated_environment,
installed_conda_targets,
USE_PATH_EXEC_DEFAULT,
)
DEFAULT_BASE_PATH_DIRECTORY = "_conda"
DEFAULT_CONDARC_OVERRIDE = "_condarc"
DEFAULT_ENSURE_CHANNELS = "r,bioconda"
import logging
log = logging.getLogger(__name__)
class CondaDependencyResolver(DependencyResolver):
class CondaDependencyResolver(DependencyResolver, ListableDependencyResolver, InstallableDependencyResolver):
dict_collection_visible_keys = DependencyResolver.dict_collection_visible_keys + ['conda_prefix', 'versionless', 'ensure_channels', 'auto_install']
resolver_type = "conda"
def __init__(self, dependency_manager, **kwds):
@@ -43,9 +51,20 @@ class CondaDependencyResolver(DependencyResolver):
dependency_manager.default_base_path, DEFAULT_BASE_PATH_DIRECTORY
)
condarc_override = get_option("condarc_override")
if condarc_override is None:
condarc_override = os.path.join(
dependency_manager.default_base_path, DEFAULT_CONDARC_OVERRIDE
)
conda_exec = get_option("exec")
debug = _string_as_bool(get_option("debug"))
ensure_channels = get_option("ensure_channels")
use_path_exec = get_option("use_path_exec")
if use_path_exec is None:
use_path_exec = USE_PATH_EXEC_DEFAULT
else:
use_path_exec = _string_as_bool(use_path_exec)
if ensure_channels is None:
ensure_channels = DEFAULT_ENSURE_CHANNELS
@@ -54,7 +73,10 @@ class CondaDependencyResolver(DependencyResolver):
conda_exec=conda_exec,
debug=debug,
ensure_channels=ensure_channels,
condarc_override=condarc_override,
use_path_exec=use_path_exec,
)
self.ensure_channels = ensure_channels
# Conda operations options (these define how resolution will occur)
auto_init = _string_as_bool(get_option("auto_init"))
@@ -83,6 +105,7 @@ class CondaDependencyResolver(DependencyResolver):
log.warn("Conda dependency resolver not sent job directory.")
return INDETERMINATE_DEPENDENCY
exact = not self.versionless or version is None
if self.versionless:
version = None
@@ -112,17 +135,46 @@ class CondaDependencyResolver(DependencyResolver):
if not exit_code:
return CondaDepenency(
self.conda_context.activate,
conda_environment
conda_environment,
exact,
)
else:
raise Exception("Conda dependency seemingly installed but failed to build job environment.")
def list_dependencies(self):
for install_target in installed_conda_targets(self.conda_context):
name = install_target.package
version = install_target.version
yield self._to_requirement(name, version)
def install_dependency(self, name, version, type, **kwds):
if type != "package":
raise NotImplemented("Can only install dependencies of type '%s'" % type)
if self.versionless:
version = None
conda_target = CondaTarget(name, version=version)
if install_conda_target(conda_target, conda_context=self.conda_context):
raise Exception("Failed to install conda recipe.")
@property
def prefix(self):
return self.conda_context.conda_prefix
class CondaDepenency(Dependency):
dict_collection_visible_keys = Dependency.dict_collection_visible_keys + ['environment_path']
dependency_type = 'conda'
def __init__(self, activate, environment_path):
def __init__(self, activate, environment_path, exact):
self.activate = activate
self.environment_path = environment_path
self._exact = exact
@property
def exact(self):
return self._exact
def shell_commands(self, requirement):
return """[ "$CONDA_DEFAULT_ENV" = "%s" ] || source %s '%s'""" % (
lib/galaxy/tools/deps/resolvers/galaxy_packages.py
+46 -12
View File
@@ -1,14 +1,20 @@
from os import listdir
from os.path import join, islink, realpath, basename, exists
from ..resolvers import DependencyResolver, INDETERMINATE_DEPENDENCY, Dependency
from ..resolvers import (
DependencyResolver,
INDETERMINATE_DEPENDENCY,
Dependency,
ListableDependencyResolver,
)
from .resolver_mixins import UsesToolDependencyDirMixin
import logging
log = logging.getLogger( __name__ )
class GalaxyPackageDependencyResolver(DependencyResolver, UsesToolDependencyDirMixin):
resolver_type = "galaxy_packages"
class BaseGalaxyPackageDependencyResolver(DependencyResolver, UsesToolDependencyDirMixin):
dict_collection_visible_keys = DependencyResolver.dict_collection_visible_keys + ['base_path', 'versionless']
def __init__(self, dependency_manager, **kwds):
# Galaxy tool shed requires explicit versions on XML elements,
@@ -25,40 +31,68 @@ class GalaxyPackageDependencyResolver(DependencyResolver, UsesToolDependencyDirM
symbolic link (if found). Returns a triple of: env_script, base_path, real_version
"""
if version is None or self.versionless:
return self._find_dep_default( name, type=type, **kwds )
exact = not self.versionless or version is None
return self._find_dep_default( name, type=type, exact=exact, **kwds )
else:
return self._find_dep_versioned( name, version, type=type, **kwds )
def _find_dep_versioned( self, name, version, type='package', **kwds ):
base_path = self.base_path
path = join( base_path, name, version )
return self._galaxy_package_dep(path, version)
return self._galaxy_package_dep(path, version, True)
def _find_dep_default( self, name, type='package', **kwds ):
def _find_dep_default( self, name, type='package', exact=True, **kwds ):
base_path = self.base_path
path = join( base_path, name, 'default' )
if islink( path ):
real_path = realpath( path )
real_version = basename( real_path )
return self._galaxy_package_dep(real_path, real_version)
return self._galaxy_package_dep(real_path, real_version, exact)
else:
return INDETERMINATE_DEPENDENCY
def _galaxy_package_dep( self, path, version ):
def _galaxy_package_dep( self, path, version, exact ):
script = join( path, 'env.sh' )
if exists( script ):
return GalaxyPackageDependency(script, path, version)
return GalaxyPackageDependency(script, path, version, exact)
elif exists( join( path, 'bin' ) ):
return GalaxyPackageDependency(None, path, version)
return GalaxyPackageDependency(None, path, version, exact)
return INDETERMINATE_DEPENDENCY
class GalaxyPackageDependency(Dependency):
class GalaxyPackageDependencyResolver(BaseGalaxyPackageDependencyResolver, ListableDependencyResolver):
resolver_type = "galaxy_packages"
def __init__( self, script, path, version ):
def list_dependencies(self):
base_path = self.base_path
for package_name in listdir(base_path):
package_dir = join(base_path, package_name)
for version in listdir(package_dir):
version_dir = join(package_dir, version)
if version == "default":
version = None
valid_dependency = _is_dependency_directory(version_dir)
if valid_dependency:
yield self._to_requirement(package_name, version)
def _is_dependency_directory(directory):
return exists(join(directory, 'env.sh')) or exists(join(directory, 'bin'))
class GalaxyPackageDependency(Dependency):
dict_collection_visible_keys = Dependency.dict_collection_visible_keys + ['script', 'path', 'version']
dependency_type = 'galaxy_package'
def __init__( self, script, path, version, exact=True ):
self.script = script
self.path = path
self.version = version
self._exact = exact
@property
def exact(self):
return self._exact
def shell_commands( self, requirement ):
base_path = self.path
lib/galaxy/tools/deps/resolvers/modules.py
+12 -3
View File
@@ -24,6 +24,7 @@ UNKNOWN_FIND_BY_MESSAGE = "ModuleDependencyResolver does not know how to find mo
class ModuleDependencyResolver(DependencyResolver):
dict_collection_visible_keys = DependencyResolver.dict_collection_visible_keys + ['base_path', 'modulepath']
resolver_type = "modules"
def __init__(self, dependency_manager, **kwds):
@@ -54,9 +55,9 @@ class ModuleDependencyResolver(DependencyResolver):
return INDETERMINATE_DEPENDENCY
if self.__has_module(name, version):
return ModuleDependency(self, name, version)
return ModuleDependency(self, name, version, exact=True)
elif self.versionless and self.__has_module(name, None):
return ModuleDependency(self, name, None)
return ModuleDependency(self, name, None, exact=False)
return INDETERMINATE_DEPENDENCY
@@ -151,10 +152,18 @@ class ModuleDependency(Dependency):
Using Environment Modules' 'modulecmd' (specifically 'modulecmd sh load') to
convert module specifications into shell expressions for inclusion in
the script used to run a tool in Galaxy."""
def __init__(self, module_dependency_resolver, module_name, module_version=None):
dict_collection_visible_keys = Dependency.dict_collection_visible_keys + ['module_name', 'module_version']
dependency_type = 'module'
def __init__(self, module_dependency_resolver, module_name, module_version=None, exact=True):
self.module_dependency_resolver = module_dependency_resolver
self.module_name = module_name
self.module_version = module_version
self._exact = exact
@property
def exact(self):
return self._exact
def shell_commands(self, requirement):
module_to_load = self.module_name
lib/galaxy/tools/deps/resolvers/resolver_mixins.py
+7 -2
View File
@@ -28,7 +28,7 @@ class UsesHomebrewMixin:
# Just grab newest installed version - may make sense some day to find
# the linked version instead.
default_version = sorted(installed_versions, reverse=True)[0]
return self._find_dep_versioned(name, default_version)
return self._find_dep_versioned(name, default_version, exact=version is None)
def _installed_versions(self, recipe):
recipe_base_path = os.path.join(self.cellar_root, recipe)
@@ -62,8 +62,13 @@ class UsesInstalledRepositoriesMixin:
class HomebrewDependency(Dependency):
def __init__(self, commands):
def __init__(self, commands, exact=True):
self.commands = commands
self._exact = exact
@property
def exact(self):
return self._exact
def shell_commands(self, requirement):
raw_commands = self.commands.replace("\n", ";")
lib/galaxy/tools/deps/resolvers/tool_shed_packages.py
+9 -5
View File
@@ -1,12 +1,16 @@
from os.path import abspath, join, exists
from .resolver_mixins import UsesInstalledRepositoriesMixin
from .galaxy_packages import GalaxyPackageDependencyResolver, GalaxyPackageDependency
from .galaxy_packages import BaseGalaxyPackageDependencyResolver, GalaxyPackageDependency
from ..resolvers import INDETERMINATE_DEPENDENCY
class ToolShedPackageDependencyResolver(GalaxyPackageDependencyResolver, UsesInstalledRepositoriesMixin):
class ToolShedPackageDependencyResolver(BaseGalaxyPackageDependencyResolver, UsesInstalledRepositoriesMixin):
resolver_type = "tool_shed_packages"
# Resolution of these dependencies depends on more than just the requirement
# tag, it depends on the tool installation context - therefore these are
# non-simple.
resolves_simple_dependencies = False
def __init__(self, dependency_manager, **kwds):
super(ToolShedPackageDependencyResolver, self).__init__(dependency_manager, **kwds)
@@ -15,7 +19,7 @@ class ToolShedPackageDependencyResolver(GalaxyPackageDependencyResolver, UsesIns
installed_tool_dependency = self._get_installed_dependency( name, type, version=version, **kwds )
if installed_tool_dependency:
path = self._get_package_installed_dependency_path( installed_tool_dependency, name, version )
return self._galaxy_package_dep(path, version)
return self._galaxy_package_dep(path, version, True)
else:
return INDETERMINATE_DEPENDENCY
@@ -28,7 +32,7 @@ class ToolShedPackageDependencyResolver(GalaxyPackageDependencyResolver, UsesIns
has_script_dep = is_galaxy_dep and dependency.script and dependency.path
if has_script_dep:
# Environment settings do not use versions.
return GalaxyPackageDependency(dependency.script, dependency.path, None)
return GalaxyPackageDependency(dependency.script, dependency.path, None, True)
return INDETERMINATE_DEPENDENCY
def _get_package_installed_dependency_path( self, installed_tool_dependency, name, version ):
@@ -54,7 +58,7 @@ class ToolShedPackageDependencyResolver(GalaxyPackageDependencyResolver, UsesIns
tool_shed_repository.installed_changeset_revision ) )
if exists( path ):
script = join( path, 'env.sh' )
return GalaxyPackageDependency(script, path, None)
return GalaxyPackageDependency(script, path, None, True)
return INDETERMINATE_DEPENDENCY
lib/galaxy/tools/deps/resolvers/unlinked_tool_shed_packages.py
+12 -4
View File
@@ -22,8 +22,8 @@ See bottom for instructions on how to add this resolver.
from os import listdir
from os.path import join, exists, getmtime
from .galaxy_packages import GalaxyPackageDependencyResolver
from ..resolvers import INDETERMINATE_DEPENDENCY
from .galaxy_packages import BaseGalaxyPackageDependencyResolver
from ..resolvers import INDETERMINATE_DEPENDENCY, Dependency
import logging
log = logging.getLogger( __name__ )
@@ -32,7 +32,8 @@ MANUAL = "manual"
PREFERRED_OWNERS = MANUAL + ",iuc,devteam"
class UnlinkedToolShedPackageDependencyResolver(GalaxyPackageDependencyResolver):
class UnlinkedToolShedPackageDependencyResolver(BaseGalaxyPackageDependencyResolver):
dict_collection_visible_keys = BaseGalaxyPackageDependencyResolver.dict_collection_visible_keys + ['preferred_owners', 'select_by_owner']
resolver_type = "unlinked_tool_shed_packages"
def __init__(self, dependency_manager, **kwds):
@@ -132,7 +133,14 @@ class UnlinkedToolShedPackageDependencyResolver(GalaxyPackageDependencyResolver)
"""
class CandidateDepenency():
class CandidateDepenency(Dependency):
dict_collection_visible_keys = Dependency.dict_collection_visible_keys + ['dependency', 'path', 'owner']
dependency_type = 'unlinked_tool_shed_package'
_exact = True
@property
def exact(self):
return self._exact
def __init__(self, dependency, path, owner=MANUAL):
self.dependency = dependency
lib/galaxy/tools/deps/views.py
+97
View File
@@ -0,0 +1,97 @@
from galaxy.exceptions import (
RequestParameterMissingException,
NotImplemented
)
class DependencyResolversView(object):
""" Provide a RESTfulish/JSONy interface to a galaxy.tools.deps.DependencyResolver
object. This can be adapted by the Galaxy web framework or other web apps.
"""
def __init__(self, app):
self._app = app
def index(self):
return map(lambda r: r.to_dict(), self._dependency_resolvers)
def show(self, index):
return self._dependency_resolver(index).to_dict()
def reload(self):
self.toolbox.reload_dependency_manager()
def manager_requirements(self):
requirements = []
for index, resolver in enumerate(self._dependency_resolvers):
if not hasattr(resolver, "list_dependencies"):
continue
for requirement in resolver.list_dependencies():
requirements.append({"index": index, "requirement": requirement.to_dict()})
return requirements
def resolver_requirements(self, index):
requirements = []
resolver = self._dependency_resolver(index)
if not hasattr(resolver, "list_dependencies"):
raise NotImplemented()
for requirement in resolver.list_dependencies():
requirements.append(requirement.to_dict())
return requirements
def manager_dependency(self, **kwds):
return self._dependency(**kwds)
def resolver_dependency(self, index, **kwds):
return self._dependency(**kwds)
def install_dependency(self, index, payload):
resolver = self._dependency_resolver(index)
if not hasattr(resolver, "install_dependency"):
raise NotImplemented()
name, version, type, extra_kwds = self._parse_dependency_info(payload)
return resolver.install_dependency(
name=name,
version=version,
type=type,
**extra_kwds
)
def _dependency(self, index=None, **kwds):
if index is not None:
index = int(index)
name, version, type, extra_kwds = self._parse_dependency_info(kwds)
resolve_kwds = dict(
job_directory="/path/to/example/job_directory",
index=index,
**extra_kwds
)
dependency = self._dependency_manager.find_dep(
name, version=version, type=type, **resolve_kwds
)
return dependency.to_dict()
def _parse_dependency_info(self, kwds):
extra_kwds = kwds.copy()
name = extra_kwds.pop("name", None)
if name is None:
raise RequestParameterMissingException("Missing 'name' parameter required for resolution.")
version = extra_kwds.pop("version", None)
type = extra_kwds.pop("type", "package")
return name, version, type, extra_kwds
def _dependency_resolver(self, index):
index = int(index)
return self._dependency_resolvers[index]
@property
def _dependency_manager(self):
return self._app.toolbox.dependency_manager
@property
def _dependency_resolvers(self):
dependency_manager = self._dependency_manager
dependency_resolvers = dependency_manager.dependency_resolvers
return dependency_resolvers
lib/galaxy/tools/toolbox/__init__.py
+2
View File
@@ -6,10 +6,12 @@ from .panel import ToolSection
from .panel import ToolSectionLabel
from .base import AbstractToolBox
from .base import BaseGalaxyToolBox
__all__ = [
"ToolSection",
"ToolSectionLabel",
"panel_item_types",
"AbstractToolBox",
"BaseGalaxyToolBox"
]
lib/galaxy/tools/toolbox/base.py
+35 -10
View File
@@ -23,14 +23,17 @@ from .panel import ToolSection
from .panel import panel_item_types
from .integrated_panel import ManagesIntegratedToolPanelMixin
from galaxy.tools.loader_directory import looks_like_a_tool
from .lineages import LineageMap
from .tags import tool_tag_manager
from .filters import FilterFactory
from .watcher import get_watcher
# Extra tool dependency not used by AbstractToolBox but by
# BaseGalaxyToolBox
from galaxy.tools.loader_directory import looks_like_a_tool
from galaxy.tools.deps import build_dependency_manager
log = logging.getLogger( __name__ )
@@ -693,7 +696,7 @@ class AbstractToolBox( Dictifiable, ManagesIntegratedToolPanelMixin, object ):
child_path = os.path.join(directory, name)
if os.path.isdir(child_path) and recursive:
self.__watch_directory(child_path, elems, integrated_elems, load_panel_dict, recursive)
elif looks_like_a_tool(child_path, enable_beta_formats=getattr(self.app.config, "enable_beta_tool_formats", False)):
elif self._looks_like_a_tool(child_path):
quick_load( child_path, async=False )
tool_loaded = True
if tool_loaded or force_watch:
@@ -957,13 +960,6 @@ class AbstractToolBox( Dictifiable, ManagesIntegratedToolPanelMixin, object ):
stored = self.app.model.context.query( self.app.model.StoredWorkflow ).get( id )
return stored.latest_workflow
@property
def sa_session( self ):
"""
Returns a SQLAlchemy session
"""
return self.app.model.context
def tool_panel_contents( self, trans, **kwds ):
""" Filter tool_panel contents for displaying for user.
"""
@@ -1096,3 +1092,32 @@ def _filter_for_panel( item, item_type, filters, context ):
return copy
return None
class BaseGalaxyToolBox(AbstractToolBox):
"""
Extend the AbstractToolBox with more Galaxy tooling-specific
functionality. Adds dependencies on dependency resolution and
tool loading modules, that an abstract description of panels
shouldn't really depend on.
"""
def __init__(self, config_filenames, tool_root_dir, app):
super(BaseGalaxyToolBox, self).__init__(config_filenames, tool_root_dir, app)
self._init_dependency_manager()
@property
def sa_session( self ):
"""
Returns a SQLAlchemy session
"""
return self.app.model.context
def _looks_like_a_tool(self, path):
return looks_like_a_tool(path, enable_beta_formats=getattr(self.app.config, "enable_beta_tool_formats", False))
def _init_dependency_manager( self ):
self.dependency_manager = build_dependency_manager( self.app.config )
def reload_dependency_manager(self):
self._init_dependency_manager()
lib/galaxy/webapps/galaxy/api/tool_dependencies.py
+135
View File
@@ -0,0 +1,135 @@
"""
API operations allowing clients to determine datatype supported by Galaxy.
"""
from galaxy.web import _future_expose_api as expose_api
from galaxy.web import require_admin
from galaxy.web.base.controller import BaseAPIController
from galaxy.tools.deps import views
import logging
log = logging.getLogger( __name__ )
class ToolDependenciesAPIController( BaseAPIController ):
def __init__(self, app):
super(ToolDependenciesAPIController, self).__init__(app)
self._view = views.DependencyResolversView(app)
@expose_api
@require_admin
def index(self, trans):
"""
GET /api/dependencies_resolvers
"""
return self._view.index()
@expose_api
@require_admin
def show(self, trans, id):
"""
GET /api/dependencies_resolver/<id>
"""
return self._view.show(id)
@expose_api
@require_admin
def update(self, trans):
"""
PUT /api/dependencies_resolvers
Reload tool dependency resolution configuration.
"""
return self._view.reload()
@expose_api
@require_admin
def resolver_dependency(self, trans, id, **kwds):
"""
GET /api/dependencies_resolver/{index}/dependency
Resolve described requirement against specified dependency resolver.
:type index: int
:param index: index of the dependency resolver
:type kwds: dict
:param kwds: dictionary structure containing extra parameters
:type name: str
:param name: name of the requirement to find a dependency for (required)
:type version: str
:param version: version of the requirement to find a dependency for (required)
:type exact: bool
:param exact: require an exact match to specify requirement (do not discard
version information to resolve dependency).
:rtype: dict
:returns: a dictified description of the dependency, with attribute
``dependency_type: null`` if no match was found.
"""
return self._view.resolver_dependency(id, **kwds)
def manager_dependency(self, trans, **kwds):
"""
GET /api/dependencies_resolvers/dependency
Resolve described requirement against all dependency resolvers, returning
the match with highest priority.
:type index: int
:param index: index of the dependency resolver
:type kwds: dict
:param kwds: dictionary structure containing extra parameters
:type name: str
:param name: name of the requirement to find a dependency for (required)
:type version: str
:param version: version of the requirement to find a dependency for (required)
:type exact: bool
:param exact: require an exact match to specify requirement (do not discard
version information to resolve dependency).
:rtype: dict
:returns: a dictified description of the dependency, with type: null
if no match was found.
"""
return self._view.manager_dependency(**kwds)
@expose_api
@require_admin
def resolver_requirements(self, trans, id, **kwds):
"""
GET /api/dependencies_resolver/{index}/requirements
Find all "simple" requirements that could be resolved "exactly"
by this dependency resolver. The dependency resolver must implement
ListDependencyResolver.
:type index: int
:param index: index of the dependency resolver
:rtype: dict
:returns: a dictified description of the requirement that could
be resolved.
"""
return self._view.resolver_requirements(id)
@expose_api
@require_admin
def manager_requirements(self, trans, **kwds):
"""
GET /api/dependencies_resolver/requirements
Find all "simple" requirements that could be resolved "exactly"
by all dependency resolvers that support this operation.
:type index: int
:param index: index of the dependency resolver
:rtype: dict
:returns: a dictified description of the requirement that could
be resolved (keyed on 'requirement') and the index of
the corresponding resolver (keyed on 'index').
"""
return self._view.manager_requirements()
lib/galaxy/webapps/galaxy/buildapp.py
+7
View File
@@ -251,6 +251,13 @@ def populate_api_routes( webapp, app ):
webapp.mapper.connect( '/api/tools/{id:.+?}', action='show', controller="tools" )
webapp.mapper.resource( 'tool', 'tools', path_prefix='/api' )
webapp.mapper.connect( '/api/dependency_resolvers/dependency', action="manager_dependency", controller="tool_dependencies" )
webapp.mapper.connect( '/api/dependency_resolvers/requirements', action="manager_requirements", controller="tool_dependencies" )
webapp.mapper.connect( '/api/dependency_resolvers/{id}/dependency', action="resolver_dependency", controller="tool_dependencies", method=['GET'] )
webapp.mapper.connect( '/api/dependency_resolvers/{id}/dependency', action="install_dependency", controller="tool_dependencies", method=['POST'] )
webapp.mapper.connect( '/api/dependency_resolvers/{id}/requirements', action="resolver_requirements", controller="tool_dependencies" )
webapp.mapper.resource( 'dependency_resolver', 'dependency_resolvers', controller="tool_dependencies", path_prefix='api' )
webapp.mapper.resource_with_deleted( 'user', 'users', path_prefix='/api' )
webapp.mapper.resource( 'genome', 'genomes', path_prefix='/api' )
webapp.mapper.resource( 'visualization', 'visualizations', path_prefix='/api' )
test/unit/test_routes.py
+24
View File
@@ -84,6 +84,30 @@ def test_galaxy_routes( ):
"/api/histories/123/contents/datasets/456"
)
test_webapp.assert_maps(
"/api/dependency_resolvers",
controller="tool_dependencies",
action="index"
)
test_webapp.assert_maps(
"/api/dependency_resolvers/dependency",
controller="tool_dependencies",
action="manager_dependency"
)
test_webapp.assert_maps(
"/api/dependency_resolvers/0",
controller="tool_dependencies",
action="show"
)
test_webapp.assert_maps(
"/api/dependency_resolvers/0/dependency",
controller="tool_dependencies",
action="resolver_dependency"
)
def assert_url_is( actual, expected ):
assert actual == expected, "Expected URL [%s] but obtained [%s]" % ( expected, actual )
test/unit/tools/test_conda_resolution.py
+1
View File
@@ -30,6 +30,7 @@ def test_conda_resolution():
dependency_manager,
auto_init=True,
auto_install=True,
use_path_exec=False, # For the test ensure this is always a clean install
)
conda_context = resolver.conda_context
assert len(list(conda_util.installed_conda_targets(conda_context))) == 0