diff --git a/lib/galaxy/tools/__init__.py b/lib/galaxy/tools/__init__.py index b82a833ed2b..c7fe3330b42 100755 --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -197,8 +197,8 @@ class ToolBox( BaseGalaxyToolBox ): @property def all_requirements(self): - reqs = [json.dumps(req, sort_keys=True) for _, tool in self.tools() for req in tool.tool_requirements] - return [json.loads(req) for req in set(reqs)] + reqs = set([req for _, tool in self.tools() for req in tool.tool_requirements]) + return [r.to_dict() for r in reqs] @property def tools_by_id( self ): @@ -1426,8 +1426,7 @@ class Tool( object, Dictifiable ): """ Return all requiremens of type package """ - reqs = [req for req in self.requirements if req.type == 'package'] - return reqs + return self.requirements.packages @property def tool_requirements_status(self): diff --git a/lib/galaxy/tools/deps/__init__.py b/lib/galaxy/tools/deps/__init__.py index 6e03b09d3e2..6f254d3965a 100644 --- a/lib/galaxy/tools/deps/__init__.py +++ b/lib/galaxy/tools/deps/__init__.py @@ -14,7 +14,10 @@ from galaxy.util import ( plugin_config ) -from .requirements import ToolRequirement +from .requirements import ( + ToolRequirement, + ToolRequirements +) from .resolvers import NullDependency from .resolvers.conda import CondaDependencyResolver, DEFAULT_ENSURE_CHANNELS from .resolvers.galaxy_packages import GalaxyPackageDependencyResolver @@ -125,12 +128,7 @@ class DependencyManager( object ): require_exact = kwds.get('exact', False) return_null_dependencies = kwds.get('return_null', False) - resolvable_requirements = [] - for requirement in requirements: - if requirement.type in ['package', 'set_environment']: - resolvable_requirements.append(requirement) - else: - log.debug("Unresolvable requirement type [%s] found, will be ignored." % requirement.type) + resolvable_requirements = requirements.resolvable for i, resolver in enumerate(self.dependency_resolvers): if index is not None and i != index: @@ -157,16 +155,16 @@ class DependencyManager( object ): if requirement in requirement_to_dependency: continue - if requirement.type in ['package', 'set_environment']: - dependency = resolver.resolve( requirement.name, requirement.version, requirement.type, **kwds ) - if require_exact and not dependency.exact: - continue + dependency = resolver.resolve( requirement.name, requirement.version, requirement.type, **kwds ) + if require_exact and not dependency.exact: + continue + if not isinstance(dependency, NullDependency): log.debug(dependency.resolver_msg) - if not isinstance(dependency, NullDependency): - requirement_to_dependency[requirement] = dependency - elif return_null_dependencies and (resolver == self.dependency_resolvers[-1] or i == index): - requirement_to_dependency[requirement] = dependency + requirement_to_dependency[requirement] = dependency + elif return_null_dependencies and (resolver == self.dependency_resolvers[-1] or i == index): + log.debug(dependency.resolver_msg) + requirement_to_dependency[requirement] = dependency return requirement_to_dependency @@ -175,8 +173,8 @@ class DependencyManager( object ): def find_dep( self, name, version=None, type='package', **kwds ): log.debug('Find dependency %s version %s' % (name, version)) - requirement = ToolRequirement(name=name, version=version, type=type) - dep_dict = self._requirements_to_dependencies_dict([requirement], **kwds) + requirements = ToolRequirements([ToolRequirement(name=name, version=version, type=type)]) + dep_dict = self._requirements_to_dependencies_dict(requirements, **kwds) if len(dep_dict) > 0: return dep_dict.values()[0] else: diff --git a/lib/galaxy/tools/deps/dependencies.py b/lib/galaxy/tools/deps/dependencies.py index 6dd372a76fa..35aafcfb329 100644 --- a/lib/galaxy/tools/deps/dependencies.py +++ b/lib/galaxy/tools/deps/dependencies.py @@ -1,4 +1,4 @@ -from galaxy.tools.deps.requirements import ToolRequirement +from galaxy.tools.deps.requirements import ToolRequirements from galaxy.util import bunch @@ -29,7 +29,7 @@ class DependenciesDescription(object): return None requirements_dicts = as_dict.get('requirements', []) - requirements = [ToolRequirement.from_dict(r) for r in requirements_dicts] + requirements = ToolRequirements.from_list(requirements_dicts) installed_tool_dependencies_dicts = as_dict.get('installed_tool_dependencies', []) installed_tool_dependencies = map(DependenciesDescription._toolshed_install_dependency_from_dict, installed_tool_dependencies_dicts) return DependenciesDescription( diff --git a/lib/galaxy/tools/deps/requirements.py b/lib/galaxy/tools/deps/requirements.py index 005fbfac399..bb2cfaa612f 100644 --- a/lib/galaxy/tools/deps/requirements.py +++ b/lib/galaxy/tools/deps/requirements.py @@ -1,4 +1,8 @@ -from galaxy.util import asbool, xml_text +from galaxy.util import ( + asbool, + xml_text, +) +from galaxy.util.oset import OrderedSet DEFAULT_REQUIREMENT_TYPE = "package" DEFAULT_REQUIREMENT_VERSION = None @@ -28,10 +32,69 @@ class ToolRequirement( object ): def __eq__(self, other): return self.name == other.name and self.type == other.type and self.version == other.version + def __ne__(self, other): + return not self.__eq__(other) + def __hash__(self): return hash((self.name, self.type, self.version)) +class ToolRequirements(object): + """ + Represents all requirements (packages, env vars) needed to run a tool. + """ + def __init__(self, tool_requirements=None): + if tool_requirements: + if not isinstance(tool_requirements, list): + raise ToolRequirementsException('ToolRequirements Constructor expects a list') + self.tool_requirements = OrderedSet([r if isinstance(r, ToolRequirement) else ToolRequirement.from_dict(r) for r in tool_requirements]) + else: + self.tool_requirements = OrderedSet() + + @staticmethod + def from_list(requirements): + return ToolRequirements(requirements) + + @property + def resolvable(self): + return ToolRequirements([r for r in self.tool_requirements if r.type in {'package', 'set_environment'}]) + + @property + def packages(self): + return ToolRequirements([r for r in self.tool_requirements if r.type == 'package']) + + def to_list(self): + return [r.to_dict() for r in self.tool_requirements] + + def append(self, requirement): + if not isinstance(requirement, ToolRequirement): + requirement = ToolRequirement.from_dict(requirement) + self.tool_requirements.add(requirement) + + def __eq__(self, other): + return len(self.tool_requirements & other.tool_requirements) == len(self.tool_requirements) == len(other.tool_requirements) + + def __ne__(self, other): + return not self.__eq__(other) + + def __iter__(self): + for r in self.tool_requirements: + yield r + + def __getitem__(self, ii): + return list(self.tool_requirements)[ii] + + def __len__(self): + return len(self.tool_requirements) + + def __hash__(self): + return sum([r.__hash__() for r in self.tool_requirements]) + + +class ToolRequirementsException(Exception): + pass + + DEFAULT_CONTAINER_TYPE = "docker" DEFAULT_CONTAINER_RESOLVE_DEPENDENCIES = False DEFAULT_CONTAINER_SHELL = "/bin/sh" # Galaxy assumes bash, but containers are usually thinner. @@ -76,7 +139,7 @@ class ContainerDescription( object ): def parse_requirements_from_dict( root_dict ): requirements = root_dict.get("requirements", []) containers = root_dict.get("containers", []) - return map(ToolRequirement.from_dict, requirements), map(ContainerDescription.from_dict, containers) + return ToolRequirements.from_list(requirements), map(ContainerDescription.from_dict, containers) def parse_requirements_from_xml( xml_root ): @@ -108,7 +171,7 @@ def parse_requirements_from_xml( xml_root ): if requirements_elem is not None: requirement_elems = requirements_elem.findall( 'requirement' ) - requirements = [] + requirements = ToolRequirements() for requirement_elem in requirement_elems: name = xml_text( requirement_elem ) type = requirement_elem.get( "type", DEFAULT_REQUIREMENT_TYPE ) diff --git a/lib/galaxy/tools/deps/views.py b/lib/galaxy/tools/deps/views.py index 8ef231dae73..a0b611cc769 100644 --- a/lib/galaxy/tools/deps/views.py +++ b/lib/galaxy/tools/deps/views.py @@ -45,14 +45,14 @@ class DependencyResolversView(object): def resolver_dependency(self, index, **kwds): return self._dependency(**kwds) - def show_dependencies(self, requirements, installed_tool_dependencies=None): + def show_dependencies(self, tool_requirements_d, installed_tool_dependencies=None): """ Resolves dependencies to build a requirements status in the admin panel/API """ kwds = {'install': False, 'return_null': True, 'installed_tool_dependencies': installed_tool_dependencies} - dependencies_per_tool = {tool: self._dependency_manager.requirements_to_dependencies(dependencies, **kwds) for tool, dependencies in requirements.items()} + dependencies_per_tool = {tool: self._dependency_manager.requirements_to_dependencies(requirements, **kwds) for tool, requirements in tool_requirements_d.items()} return dependencies_per_tool def install_dependencies(self, requirements): @@ -138,12 +138,12 @@ class DependencyResolversView(object): """ return [index for index, resolver in enumerate(self._dependency_resolvers) if hasattr(resolver, "install_dependency") and not resolver.disabled ] - def get_requirements_status(self, requested_requirements, installed_tool_dependencies=None): - dependencies = self.show_dependencies(requested_requirements, installed_tool_dependencies) - # dependencies is a dict keyed on tool_ids, values are lists of ToolRequirements for that tool. - # we collapse requested requirements to single set, - # then use collapsed requirements to get resolved dependencies without duplicates. - flat_tool_requirements = set([r for requirement_list in requested_requirements.values() for r in requirement_list]) + def get_requirements_status(self, tool_requirements_d, installed_tool_dependencies=None): + dependencies = self.show_dependencies(tool_requirements_d, installed_tool_dependencies) + # dependencies is a dict keyed on tool_ids, value is a ToolRequirements object for that tool. + # We use the union of resolvable ToolRequirements to get resolved dependencies without duplicates. + requirements = [r.resolvable for r in tool_requirements_d.values()] + flat_tool_requirements = set().union(*requirements) flat_dependencies = [] for requirements_odict in dependencies.values(): for requirement in requirements_odict: diff --git a/lib/galaxy/util/oset.py b/lib/galaxy/util/oset.py new file mode 100644 index 00000000000..4081498fab7 --- /dev/null +++ b/lib/galaxy/util/oset.py @@ -0,0 +1,62 @@ +""" +Ordered set implementation from https://code.activestate.com/recipes/576694/ +""" +import collections + + +class OrderedSet(collections.MutableSet): + def __init__(self, iterable=None): + self.end = end = [] + end += [None, end, end] # sentinel node for doubly linked list + self.map = {} # key --> [key, prev, next] + if iterable is not None: + self |= iterable + + def __len__(self): + return len(self.map) + + def __contains__(self, key): + return key in self.map + + def add(self, key): + if key not in self.map: + end = self.end + curr = end[1] + curr[2] = end[1] = self.map[key] = [key, curr, end] + + def discard(self, key): + if key in self.map: + key, prev, next = self.map.pop(key) + prev[2] = next + next[1] = prev + + def __iter__(self): + end = self.end + curr = end[2] + while curr is not end: + yield curr[0] + curr = curr[2] + + def __reversed__(self): + end = self.end + curr = end[1] + while curr is not end: + yield curr[0] + curr = curr[1] + + def pop(self, last=True): + if not self: + raise KeyError('set is empty') + key = self.end[1][0] if last else self.end[2][0] + self.discard(key) + return key + + def __repr__(self): + if not self: + return '%s()' % (self.__class__.__name__,) + return '%s(%r)' % (self.__class__.__name__, list(self)) + + def __eq__(self, other): + if isinstance(other, OrderedSet): + return len(self) == len(other) and list(self) == list(other) + return set(self) == set(other) diff --git a/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py b/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py index fec0f425713..e0742004dab 100644 --- a/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py +++ b/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py @@ -781,8 +781,8 @@ class AdminToolshed( AdminGalaxy ): reinstalling=False, required_repo_info_dicts=None ) view = views.DependencyResolversView(self.app) - requirements = suc.get_requirements_from_repository(repository) - requirements_status = view.get_requirements_status(requirements, repository.installed_tool_dependencies) + tool_requirements_d = suc.get_requirements_from_repository(repository) + requirements_status = view.get_requirements_status(tool_requirements_d, repository.installed_tool_dependencies) return trans.fill_template( '/admin/tool_shed_repository/manage_repository.mako', repository=repository, description=description, diff --git a/lib/tool_shed/galaxy_install/install_manager.py b/lib/tool_shed/galaxy_install/install_manager.py index eaaa0469d21..dd4c5248f62 100644 --- a/lib/tool_shed/galaxy_install/install_manager.py +++ b/lib/tool_shed/galaxy_install/install_manager.py @@ -905,14 +905,11 @@ class InstallRepositoryManager( object ): if 'tools' in metadata and install_resolver_dependencies: self.update_tool_shed_repository_status( tool_shed_repository, self.install_model.ToolShedRepository.installation_status.INSTALLING_TOOL_DEPENDENCIES ) - installed_requirements = [] - for tool_d in metadata['tools']: - tool = self.app.toolbox._tools_by_id.get(tool_d['guid'], None) - if tool and tool.requirements not in installed_requirements: - self._view.install_dependencies(tool.requirements) - installed_requirements.append(tool.requirements) - if self.app.config.use_cached_dependency_manager: - tool.build_dependency_cache() + new_tools = [self.app.toolbox._tools_by_id.get(tool_d['guid'], None) for tool_d in metadata['tools']] + new_requirements = set([tool.requirements.packages for tool in new_tools if tool]) + [self._view.install_dependencies(r) for r in new_requirements] + if self.app.config.use_cached_dependency_manager: + [self.app.toolbox.dependency_manager.build_cache(r) for r in new_requirements] if install_tool_dependencies and tool_shed_repository.tool_dependencies and 'tool_dependencies' in metadata: work_dir = tempfile.mkdtemp( prefix="tmp-toolshed-itsr" ) diff --git a/lib/tool_shed/util/shed_util_common.py b/lib/tool_shed/util/shed_util_common.py index e77dce8454e..6bec996e44d 100644 --- a/lib/tool_shed/util/shed_util_common.py +++ b/lib/tool_shed/util/shed_util_common.py @@ -225,7 +225,7 @@ def get_tool_shed_repo_requirements(app, tool_shed_url, repositories=None, repo_ def get_requirements_from_tools(tools): - return {tool['id']: [galaxy.tools.deps.requirements.ToolRequirement.from_dict(r) for r in tool['requirements']] for tool in tools} + return {tool['id']: galaxy.tools.deps.requirements.ToolRequirements.from_list(tool['requirements']) for tool in tools} def get_requirements_from_repository(repository): diff --git a/test/unit/tools/test_tool_deps.py b/test/unit/tools/test_tool_deps.py index d8ef0a06daa..2f4bda30fcb 100644 --- a/test/unit/tools/test_tool_deps.py +++ b/test/unit/tools/test_tool_deps.py @@ -14,6 +14,10 @@ from stat import S_IXUSR from subprocess import PIPE, Popen from galaxy.tools.deps import DependencyManager +from galaxy.tools.deps.requirements import ( + ToolRequirement, + ToolRequirements +) from galaxy.tools.deps.resolvers import NullDependency from galaxy.tools.deps.resolvers.galaxy_packages import GalaxyPackageDependency from galaxy.tools.deps.resolvers.modules import ModuleDependency, ModuleDependencyResolver @@ -112,6 +116,30 @@ def __build_ts_test_package(base_path, script_contents=''): return package_dir +REQUIREMENT_A = {'name': 'gnuplot', + 'type': 'package', + 'version': '4.6'} +REQUIREMENT_B = REQUIREMENT_A.copy() +REQUIREMENT_B['version'] = '4.7' + + +def test_tool_requirement_equality(): + a = ToolRequirement.from_dict(REQUIREMENT_A) + assert a == ToolRequirement(**REQUIREMENT_A) + b = ToolRequirement(**REQUIREMENT_B) + assert a != b + + +def test_tool_requirements(): + tool_requirements_ab = ToolRequirements([REQUIREMENT_A, REQUIREMENT_B]) + tool_requirements_ab_dup = ToolRequirements([REQUIREMENT_A, REQUIREMENT_B]) + tool_requirements_b = ToolRequirements([REQUIREMENT_A]) + assert tool_requirements_ab == ToolRequirements([REQUIREMENT_B, REQUIREMENT_A]) + assert tool_requirements_ab == ToolRequirements([REQUIREMENT_B, REQUIREMENT_A, REQUIREMENT_A]) + assert tool_requirements_ab != tool_requirements_b + assert len(set([tool_requirements_ab, tool_requirements_ab_dup])) == 1 + + def test_module_dependency_resolver(): with __test_base_path() as temp_directory: module_script = os.path.join(temp_directory, "modulecmd") @@ -193,7 +221,7 @@ def test_shell_commands_built(): with __test_base_path() as base_path: dm = DependencyManager( default_base_path=base_path ) __setup_galaxy_package_dep( base_path, TEST_REPO_NAME, TEST_VERSION, contents="export FOO=\"bar\"" ) - mock_requirements = [ Bunch(type="package", version=TEST_VERSION, name=TEST_REPO_NAME ) ] + mock_requirements = ToolRequirements([{'type': 'package', 'version': TEST_VERSION, 'name': TEST_REPO_NAME}]) commands = dm.dependency_shell_commands( mock_requirements ) __assert_foo_exported( commands )