From 596d12da44908e158905d7c1bce30d9e3557b258 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Sat, 21 Jan 2017 22:02:14 +0100 Subject: [PATCH 1/3] Implement ToolRequirements class This class holds indiviual ToolRequirement objects as an ordered set. This makes it easier to compare, intersect, and get unions of multiple ToolRequirements and to filter for resolvable or package type dependencies. This will be useful when installing tool dependencies in batch. --- lib/galaxy/tools/__init__.py | 7 +- lib/galaxy/tools/deps/__init__.py | 33 ++++----- lib/galaxy/tools/deps/dependencies.py | 4 +- lib/galaxy/tools/deps/requirements.py | 69 ++++++++++++++++++- lib/galaxy/tools/deps/views.py | 16 ++--- lib/galaxy/util/oset.py | 62 +++++++++++++++++ .../galaxy/controllers/admin_toolshed.py | 4 +- .../galaxy_install/install_manager.py | 13 ++-- lib/tool_shed/util/shed_util_common.py | 2 +- test/unit/tools/test_tool_deps.py | 28 +++++++- 10 files changed, 191 insertions(+), 47 deletions(-) create mode 100644 lib/galaxy/util/oset.py diff --git a/lib/galaxy/tools/__init__.py b/lib/galaxy/tools/__init__.py index b82a833ed2b..c7fe3330b42 100755 --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -197,8 +197,8 @@ class ToolBox( BaseGalaxyToolBox ): @property def all_requirements(self): - reqs = [json.dumps(req, sort_keys=True) for _, tool in self.tools() for req in tool.tool_requirements] - return [json.loads(req) for req in set(reqs)] + reqs = set([req for _, tool in self.tools() for req in tool.tool_requirements]) + return [r.to_dict() for r in reqs] @property def tools_by_id( self ): @@ -1426,8 +1426,7 @@ class Tool( object, Dictifiable ): """ Return all requiremens of type package """ - reqs = [req for req in self.requirements if req.type == 'package'] - return reqs + return self.requirements.packages @property def tool_requirements_status(self): diff --git a/lib/galaxy/tools/deps/__init__.py b/lib/galaxy/tools/deps/__init__.py index 6e03b09d3e2..f842d4479be 100644 --- a/lib/galaxy/tools/deps/__init__.py +++ b/lib/galaxy/tools/deps/__init__.py @@ -14,7 +14,10 @@ from galaxy.util import ( plugin_config ) -from .requirements import ToolRequirement +from .requirements import ( + ToolRequirement, + ToolRequirements +) from .resolvers import NullDependency from .resolvers.conda import CondaDependencyResolver, DEFAULT_ENSURE_CHANNELS from .resolvers.galaxy_packages import GalaxyPackageDependencyResolver @@ -125,12 +128,7 @@ class DependencyManager( object ): require_exact = kwds.get('exact', False) return_null_dependencies = kwds.get('return_null', False) - resolvable_requirements = [] - for requirement in requirements: - if requirement.type in ['package', 'set_environment']: - resolvable_requirements.append(requirement) - else: - log.debug("Unresolvable requirement type [%s] found, will be ignored." % requirement.type) + resolvable_requirements = requirements.resolvable for i, resolver in enumerate(self.dependency_resolvers): if index is not None and i != index: @@ -157,16 +155,15 @@ class DependencyManager( object ): if requirement in requirement_to_dependency: continue - if requirement.type in ['package', 'set_environment']: - dependency = resolver.resolve( requirement.name, requirement.version, requirement.type, **kwds ) - if require_exact and not dependency.exact: - continue + dependency = resolver.resolve( requirement.name, requirement.version, requirement.type, **kwds ) + if require_exact and not dependency.exact: + continue - log.debug(dependency.resolver_msg) - if not isinstance(dependency, NullDependency): - requirement_to_dependency[requirement] = dependency - elif return_null_dependencies and (resolver == self.dependency_resolvers[-1] or i == index): - requirement_to_dependency[requirement] = dependency + log.debug(dependency.resolver_msg) + if not isinstance(dependency, NullDependency): + requirement_to_dependency[requirement] = dependency + elif return_null_dependencies and (resolver == self.dependency_resolvers[-1] or i == index): + requirement_to_dependency[requirement] = dependency return requirement_to_dependency @@ -175,8 +172,8 @@ class DependencyManager( object ): def find_dep( self, name, version=None, type='package', **kwds ): log.debug('Find dependency %s version %s' % (name, version)) - requirement = ToolRequirement(name=name, version=version, type=type) - dep_dict = self._requirements_to_dependencies_dict([requirement], **kwds) + requirements = ToolRequirements([ToolRequirement(name=name, version=version, type=type)]) + dep_dict = self._requirements_to_dependencies_dict(requirements, **kwds) if len(dep_dict) > 0: return dep_dict.values()[0] else: diff --git a/lib/galaxy/tools/deps/dependencies.py b/lib/galaxy/tools/deps/dependencies.py index 6dd372a76fa..35aafcfb329 100644 --- a/lib/galaxy/tools/deps/dependencies.py +++ b/lib/galaxy/tools/deps/dependencies.py @@ -1,4 +1,4 @@ -from galaxy.tools.deps.requirements import ToolRequirement +from galaxy.tools.deps.requirements import ToolRequirements from galaxy.util import bunch @@ -29,7 +29,7 @@ class DependenciesDescription(object): return None requirements_dicts = as_dict.get('requirements', []) - requirements = [ToolRequirement.from_dict(r) for r in requirements_dicts] + requirements = ToolRequirements.from_list(requirements_dicts) installed_tool_dependencies_dicts = as_dict.get('installed_tool_dependencies', []) installed_tool_dependencies = map(DependenciesDescription._toolshed_install_dependency_from_dict, installed_tool_dependencies_dicts) return DependenciesDescription( diff --git a/lib/galaxy/tools/deps/requirements.py b/lib/galaxy/tools/deps/requirements.py index 005fbfac399..bb2cfaa612f 100644 --- a/lib/galaxy/tools/deps/requirements.py +++ b/lib/galaxy/tools/deps/requirements.py @@ -1,4 +1,8 @@ -from galaxy.util import asbool, xml_text +from galaxy.util import ( + asbool, + xml_text, +) +from galaxy.util.oset import OrderedSet DEFAULT_REQUIREMENT_TYPE = "package" DEFAULT_REQUIREMENT_VERSION = None @@ -28,10 +32,69 @@ class ToolRequirement( object ): def __eq__(self, other): return self.name == other.name and self.type == other.type and self.version == other.version + def __ne__(self, other): + return not self.__eq__(other) + def __hash__(self): return hash((self.name, self.type, self.version)) +class ToolRequirements(object): + """ + Represents all requirements (packages, env vars) needed to run a tool. + """ + def __init__(self, tool_requirements=None): + if tool_requirements: + if not isinstance(tool_requirements, list): + raise ToolRequirementsException('ToolRequirements Constructor expects a list') + self.tool_requirements = OrderedSet([r if isinstance(r, ToolRequirement) else ToolRequirement.from_dict(r) for r in tool_requirements]) + else: + self.tool_requirements = OrderedSet() + + @staticmethod + def from_list(requirements): + return ToolRequirements(requirements) + + @property + def resolvable(self): + return ToolRequirements([r for r in self.tool_requirements if r.type in {'package', 'set_environment'}]) + + @property + def packages(self): + return ToolRequirements([r for r in self.tool_requirements if r.type == 'package']) + + def to_list(self): + return [r.to_dict() for r in self.tool_requirements] + + def append(self, requirement): + if not isinstance(requirement, ToolRequirement): + requirement = ToolRequirement.from_dict(requirement) + self.tool_requirements.add(requirement) + + def __eq__(self, other): + return len(self.tool_requirements & other.tool_requirements) == len(self.tool_requirements) == len(other.tool_requirements) + + def __ne__(self, other): + return not self.__eq__(other) + + def __iter__(self): + for r in self.tool_requirements: + yield r + + def __getitem__(self, ii): + return list(self.tool_requirements)[ii] + + def __len__(self): + return len(self.tool_requirements) + + def __hash__(self): + return sum([r.__hash__() for r in self.tool_requirements]) + + +class ToolRequirementsException(Exception): + pass + + DEFAULT_CONTAINER_TYPE = "docker" DEFAULT_CONTAINER_RESOLVE_DEPENDENCIES = False DEFAULT_CONTAINER_SHELL = "/bin/sh" # Galaxy assumes bash, but containers are usually thinner. @@ -76,7 +139,7 @@ class ContainerDescription( object ): def parse_requirements_from_dict( root_dict ): requirements = root_dict.get("requirements", []) containers = root_dict.get("containers", []) - return map(ToolRequirement.from_dict, requirements), map(ContainerDescription.from_dict, containers) + return ToolRequirements.from_list(requirements), map(ContainerDescription.from_dict, containers) def parse_requirements_from_xml( xml_root ): @@ -108,7 +171,7 @@ def parse_requirements_from_xml( xml_root ): if requirements_elem is not None: requirement_elems = requirements_elem.findall( 'requirement' ) - requirements = [] + requirements = ToolRequirements() for requirement_elem in requirement_elems: name = xml_text( requirement_elem ) type = requirement_elem.get( "type", DEFAULT_REQUIREMENT_TYPE ) diff --git a/lib/galaxy/tools/deps/views.py b/lib/galaxy/tools/deps/views.py index 8ef231dae73..a0b611cc769 100644 --- a/lib/galaxy/tools/deps/views.py +++ b/lib/galaxy/tools/deps/views.py @@ -45,14 +45,14 @@ class DependencyResolversView(object): def resolver_dependency(self, index, **kwds): return self._dependency(**kwds) - def show_dependencies(self, requirements, installed_tool_dependencies=None): + def show_dependencies(self, tool_requirements_d, installed_tool_dependencies=None): """ Resolves dependencies to build a requirements status in the admin panel/API """ kwds = {'install': False, 'return_null': True, 'installed_tool_dependencies': installed_tool_dependencies} - dependencies_per_tool = {tool: self._dependency_manager.requirements_to_dependencies(dependencies, **kwds) for tool, dependencies in requirements.items()} + dependencies_per_tool = {tool: self._dependency_manager.requirements_to_dependencies(requirements, **kwds) for tool, requirements in tool_requirements_d.items()} return dependencies_per_tool def install_dependencies(self, requirements): @@ -138,12 +138,12 @@ class DependencyResolversView(object): """ return [index for index, resolver in enumerate(self._dependency_resolvers) if hasattr(resolver, "install_dependency") and not resolver.disabled ] - def get_requirements_status(self, requested_requirements, installed_tool_dependencies=None): - dependencies = self.show_dependencies(requested_requirements, installed_tool_dependencies) - # dependencies is a dict keyed on tool_ids, values are lists of ToolRequirements for that tool. - # we collapse requested requirements to single set, - # then use collapsed requirements to get resolved dependencies without duplicates. - flat_tool_requirements = set([r for requirement_list in requested_requirements.values() for r in requirement_list]) + def get_requirements_status(self, tool_requirements_d, installed_tool_dependencies=None): + dependencies = self.show_dependencies(tool_requirements_d, installed_tool_dependencies) + # dependencies is a dict keyed on tool_ids, value is a ToolRequirements object for that tool. + # We use the union of resolvable ToolRequirements to get resolved dependencies without duplicates. + requirements = [r.resolvable for r in tool_requirements_d.values()] + flat_tool_requirements = set().union(*requirements) flat_dependencies = [] for requirements_odict in dependencies.values(): for requirement in requirements_odict: diff --git a/lib/galaxy/util/oset.py b/lib/galaxy/util/oset.py new file mode 100644 index 00000000000..4081498fab7 --- /dev/null +++ b/lib/galaxy/util/oset.py @@ -0,0 +1,62 @@ +""" +Ordered set implementation from https://code.activestate.com/recipes/576694/ +""" +import collections + + +class OrderedSet(collections.MutableSet): + def __init__(self, iterable=None): + self.end = end = [] + end += [None, end, end] # sentinel node for doubly linked list + self.map = {} # key --> [key, prev, next] + if iterable is not None: + self |= iterable + + def __len__(self): + return len(self.map) + + def __contains__(self, key): + return key in self.map + + def add(self, key): + if key not in self.map: + end = self.end + curr = end[1] + curr[2] = end[1] = self.map[key] = [key, curr, end] + + def discard(self, key): + if key in self.map: + key, prev, next = self.map.pop(key) + prev[2] = next + next[1] = prev + + def __iter__(self): + end = self.end + curr = end[2] + while curr is not end: + yield curr[0] + curr = curr[2] + + def __reversed__(self): + end = self.end + curr = end[1] + while curr is not end: + yield curr[0] + curr = curr[1] + + def pop(self, last=True): + if not self: + raise KeyError('set is empty') + key = self.end[1][0] if last else self.end[2][0] + self.discard(key) + return key + + def __repr__(self): + if not self: + return '%s()' % (self.__class__.__name__,) + return '%s(%r)' % (self.__class__.__name__, list(self)) + + def __eq__(self, other): + if isinstance(other, OrderedSet): + return len(self) == len(other) and list(self) == list(other) + return set(self) == set(other) diff --git a/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py b/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py index fec0f425713..e0742004dab 100644 --- a/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py +++ b/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py @@ -781,8 +781,8 @@ class AdminToolshed( AdminGalaxy ): reinstalling=False, required_repo_info_dicts=None ) view = views.DependencyResolversView(self.app) - requirements = suc.get_requirements_from_repository(repository) - requirements_status = view.get_requirements_status(requirements, repository.installed_tool_dependencies) + tool_requirements_d = suc.get_requirements_from_repository(repository) + requirements_status = view.get_requirements_status(tool_requirements_d, repository.installed_tool_dependencies) return trans.fill_template( '/admin/tool_shed_repository/manage_repository.mako', repository=repository, description=description, diff --git a/lib/tool_shed/galaxy_install/install_manager.py b/lib/tool_shed/galaxy_install/install_manager.py index eaaa0469d21..dd4c5248f62 100644 --- a/lib/tool_shed/galaxy_install/install_manager.py +++ b/lib/tool_shed/galaxy_install/install_manager.py @@ -905,14 +905,11 @@ class InstallRepositoryManager( object ): if 'tools' in metadata and install_resolver_dependencies: self.update_tool_shed_repository_status( tool_shed_repository, self.install_model.ToolShedRepository.installation_status.INSTALLING_TOOL_DEPENDENCIES ) - installed_requirements = [] - for tool_d in metadata['tools']: - tool = self.app.toolbox._tools_by_id.get(tool_d['guid'], None) - if tool and tool.requirements not in installed_requirements: - self._view.install_dependencies(tool.requirements) - installed_requirements.append(tool.requirements) - if self.app.config.use_cached_dependency_manager: - tool.build_dependency_cache() + new_tools = [self.app.toolbox._tools_by_id.get(tool_d['guid'], None) for tool_d in metadata['tools']] + new_requirements = set([tool.requirements.packages for tool in new_tools if tool]) + [self._view.install_dependencies(r) for r in new_requirements] + if self.app.config.use_cached_dependency_manager: + [self.app.toolbox.dependency_manager.build_cache(r) for r in new_requirements] if install_tool_dependencies and tool_shed_repository.tool_dependencies and 'tool_dependencies' in metadata: work_dir = tempfile.mkdtemp( prefix="tmp-toolshed-itsr" ) diff --git a/lib/tool_shed/util/shed_util_common.py b/lib/tool_shed/util/shed_util_common.py index e77dce8454e..6bec996e44d 100644 --- a/lib/tool_shed/util/shed_util_common.py +++ b/lib/tool_shed/util/shed_util_common.py @@ -225,7 +225,7 @@ def get_tool_shed_repo_requirements(app, tool_shed_url, repositories=None, repo_ def get_requirements_from_tools(tools): - return {tool['id']: [galaxy.tools.deps.requirements.ToolRequirement.from_dict(r) for r in tool['requirements']] for tool in tools} + return {tool['id']: galaxy.tools.deps.requirements.ToolRequirements.from_list(tool['requirements']) for tool in tools} def get_requirements_from_repository(repository): diff --git a/test/unit/tools/test_tool_deps.py b/test/unit/tools/test_tool_deps.py index d8ef0a06daa..491d577c0d5 100644 --- a/test/unit/tools/test_tool_deps.py +++ b/test/unit/tools/test_tool_deps.py @@ -14,6 +14,10 @@ from stat import S_IXUSR from subprocess import PIPE, Popen from galaxy.tools.deps import DependencyManager +from galaxy.tools.deps.requirements import ( + ToolRequirement, + ToolRequirements +) from galaxy.tools.deps.resolvers import NullDependency from galaxy.tools.deps.resolvers.galaxy_packages import GalaxyPackageDependency from galaxy.tools.deps.resolvers.modules import ModuleDependency, ModuleDependencyResolver @@ -112,6 +116,28 @@ def __build_ts_test_package(base_path, script_contents=''): return package_dir +REQUIREMENT_A = {'name': 'gnuplot', + 'type': 'package', + 'version': '4.6'} +REQUIREMENT_B = REQUIREMENT_A.copy() +REQUIREMENT_B['version'] = '4.7' + + +def test_tool_requirement_equality(): + a = ToolRequirement.from_dict(REQUIREMENT_A) + assert a == ToolRequirement(**REQUIREMENT_A) + b = ToolRequirement(**REQUIREMENT_B) + assert a != b + + +def test_tool_requirements(): + tool_requirements_ab = ToolRequirements([REQUIREMENT_A, REQUIREMENT_B]) + tool_requirements_b = ToolRequirements([REQUIREMENT_A]) + assert tool_requirements_ab == ToolRequirements([REQUIREMENT_B, REQUIREMENT_A]) + assert tool_requirements_ab == ToolRequirements([REQUIREMENT_B, REQUIREMENT_A, REQUIREMENT_A]) + assert tool_requirements_ab != tool_requirements_b + + def test_module_dependency_resolver(): with __test_base_path() as temp_directory: module_script = os.path.join(temp_directory, "modulecmd") @@ -193,7 +219,7 @@ def test_shell_commands_built(): with __test_base_path() as base_path: dm = DependencyManager( default_base_path=base_path ) __setup_galaxy_package_dep( base_path, TEST_REPO_NAME, TEST_VERSION, contents="export FOO=\"bar\"" ) - mock_requirements = [ Bunch(type="package", version=TEST_VERSION, name=TEST_REPO_NAME ) ] + mock_requirements = ToolRequirements([{'type': 'package', 'version': TEST_VERSION, 'name': TEST_REPO_NAME}]) commands = dm.dependency_shell_commands( mock_requirements ) __assert_foo_exported( commands ) From 5ee7e2e3e26bdbdecec0b74ed4154734d9b56ae4 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Sun, 22 Jan 2017 14:36:15 +0100 Subject: [PATCH 2/3] Log failed dep resolution just once --- lib/galaxy/tools/deps/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/galaxy/tools/deps/__init__.py b/lib/galaxy/tools/deps/__init__.py index f842d4479be..6f254d3965a 100644 --- a/lib/galaxy/tools/deps/__init__.py +++ b/lib/galaxy/tools/deps/__init__.py @@ -159,10 +159,11 @@ class DependencyManager( object ): if require_exact and not dependency.exact: continue - log.debug(dependency.resolver_msg) if not isinstance(dependency, NullDependency): + log.debug(dependency.resolver_msg) requirement_to_dependency[requirement] = dependency elif return_null_dependencies and (resolver == self.dependency_resolvers[-1] or i == index): + log.debug(dependency.resolver_msg) requirement_to_dependency[requirement] = dependency return requirement_to_dependency From 9c9b405b6f969421c2073a53dad117ffefed17c6 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Sun, 22 Jan 2017 18:04:26 +0100 Subject: [PATCH 3/3] Add test for set on list of ToolRequirements --- test/unit/tools/test_tool_deps.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/unit/tools/test_tool_deps.py b/test/unit/tools/test_tool_deps.py index 491d577c0d5..2f4bda30fcb 100644 --- a/test/unit/tools/test_tool_deps.py +++ b/test/unit/tools/test_tool_deps.py @@ -132,10 +132,12 @@ def test_tool_requirement_equality(): def test_tool_requirements(): tool_requirements_ab = ToolRequirements([REQUIREMENT_A, REQUIREMENT_B]) + tool_requirements_ab_dup = ToolRequirements([REQUIREMENT_A, REQUIREMENT_B]) tool_requirements_b = ToolRequirements([REQUIREMENT_A]) assert tool_requirements_ab == ToolRequirements([REQUIREMENT_B, REQUIREMENT_A]) assert tool_requirements_ab == ToolRequirements([REQUIREMENT_B, REQUIREMENT_A, REQUIREMENT_A]) assert tool_requirements_ab != tool_requirements_b + assert len(set([tool_requirements_ab, tool_requirements_ab_dup])) == 1 def test_module_dependency_resolver():