Merge pull request #3461 from mvdbeek/ToolRequirements

Implement ToolRequirements class

lib/galaxy/tools/__init__.py

@@ -197,8 +197,8 @@ class ToolBox( BaseGalaxyToolBox ):
 
     @property
     def all_requirements(self):
-        reqs = [json.dumps(req, sort_keys=True) for _, tool in self.tools() for req in tool.tool_requirements]
-        return [json.loads(req) for req in set(reqs)]
+        reqs = set([req for _, tool in self.tools() for req in tool.tool_requirements])
+        return [r.to_dict() for r in reqs]
 
     @property
     def tools_by_id( self ):
@@ -1426,8 +1426,7 @@ class Tool( object, Dictifiable ):
         """
         Return all requiremens of type package
         """
-        reqs = [req for req in self.requirements if req.type == 'package']
-        return reqs
+        return self.requirements.packages
 
     @property
     def tool_requirements_status(self):

lib/galaxy/tools/deps/__init__.py

@@ -14,7 +14,10 @@ from galaxy.util import (
     plugin_config
 )
 
-from .requirements import ToolRequirement
+from .requirements import (
+    ToolRequirement,
+    ToolRequirements
+)
 from .resolvers import NullDependency
 from .resolvers.conda import CondaDependencyResolver, DEFAULT_ENSURE_CHANNELS
 from .resolvers.galaxy_packages import GalaxyPackageDependencyResolver
@@ -125,12 +128,7 @@ class DependencyManager( object ):
         require_exact = kwds.get('exact', False)
         return_null_dependencies = kwds.get('return_null', False)
 
-        resolvable_requirements = []
-        for requirement in requirements:
-            if requirement.type in ['package', 'set_environment']:
-                resolvable_requirements.append(requirement)
-            else:
-                log.debug("Unresolvable requirement type [%s] found, will be ignored." % requirement.type)
+        resolvable_requirements = requirements.resolvable
 
         for i, resolver in enumerate(self.dependency_resolvers):
             if index is not None and i != index:
@@ -157,16 +155,16 @@ class DependencyManager( object ):
                 if requirement in requirement_to_dependency:
                     continue
 
-                if requirement.type in ['package', 'set_environment']:
-                    dependency = resolver.resolve( requirement.name, requirement.version, requirement.type, **kwds )
-                    if require_exact and not dependency.exact:
-                        continue
+                dependency = resolver.resolve( requirement.name, requirement.version, requirement.type, **kwds )
+                if require_exact and not dependency.exact:
+                    continue
 
+                if not isinstance(dependency, NullDependency):
                     log.debug(dependency.resolver_msg)
-                    if not isinstance(dependency, NullDependency):
-                        requirement_to_dependency[requirement] = dependency
-                    elif return_null_dependencies and (resolver == self.dependency_resolvers[-1] or i == index):
-                        requirement_to_dependency[requirement] = dependency
+                    requirement_to_dependency[requirement] = dependency
+                elif return_null_dependencies and (resolver == self.dependency_resolvers[-1] or i == index):
+                    log.debug(dependency.resolver_msg)
+                    requirement_to_dependency[requirement] = dependency
 
         return requirement_to_dependency
 
@@ -175,8 +173,8 @@ class DependencyManager( object ):
 
     def find_dep( self, name, version=None, type='package', **kwds ):
         log.debug('Find dependency %s version %s' % (name, version))
-        requirement = ToolRequirement(name=name, version=version, type=type)
-        dep_dict = self._requirements_to_dependencies_dict([requirement], **kwds)
+        requirements = ToolRequirements([ToolRequirement(name=name, version=version, type=type)])
+        dep_dict = self._requirements_to_dependencies_dict(requirements, **kwds)
         if len(dep_dict) > 0:
             return dep_dict.values()[0]
         else:

lib/galaxy/tools/deps/dependencies.py

@@ -1,4 +1,4 @@
-from galaxy.tools.deps.requirements import ToolRequirement
+from galaxy.tools.deps.requirements import ToolRequirements
 from galaxy.util import bunch
 
 
@@ -29,7 +29,7 @@ class DependenciesDescription(object):
             return None
 
         requirements_dicts = as_dict.get('requirements', [])
-        requirements = [ToolRequirement.from_dict(r) for r in requirements_dicts]
+        requirements = ToolRequirements.from_list(requirements_dicts)
         installed_tool_dependencies_dicts = as_dict.get('installed_tool_dependencies', [])
         installed_tool_dependencies = map(DependenciesDescription._toolshed_install_dependency_from_dict, installed_tool_dependencies_dicts)
         return DependenciesDescription(

lib/galaxy/tools/deps/requirements.py

@@ -1,4 +1,8 @@
-from galaxy.util import asbool, xml_text
+from galaxy.util import (
+    asbool,
+    xml_text,
+)
+from galaxy.util.oset import OrderedSet
 
 DEFAULT_REQUIREMENT_TYPE = "package"
 DEFAULT_REQUIREMENT_VERSION = None
@@ -28,10 +32,69 @@ class ToolRequirement( object ):
     def __eq__(self, other):
         return self.name == other.name and self.type == other.type and self.version == other.version
 
+    def __ne__(self, other):
+        return not self.__eq__(other)
+
     def __hash__(self):
         return hash((self.name, self.type, self.version))
 
 
+class ToolRequirements(object):
+    """
+    Represents all requirements (packages, env vars) needed to run a tool.
+    """
+    def __init__(self, tool_requirements=None):
+        if tool_requirements:
+            if not isinstance(tool_requirements, list):
+                raise ToolRequirementsException('ToolRequirements Constructor expects a list')
+            self.tool_requirements = OrderedSet([r if isinstance(r, ToolRequirement) else ToolRequirement.from_dict(r) for r in tool_requirements])
+        else:
+            self.tool_requirements = OrderedSet()
+
+    @staticmethod
+    def from_list(requirements):
+        return ToolRequirements(requirements)
+
+    @property
+    def resolvable(self):
+        return ToolRequirements([r for r in self.tool_requirements if r.type in {'package', 'set_environment'}])
+
+    @property
+    def packages(self):
+        return ToolRequirements([r for r in self.tool_requirements if r.type == 'package'])
+
+    def to_list(self):
+        return [r.to_dict() for r in self.tool_requirements]
+
+    def append(self, requirement):
+        if not isinstance(requirement, ToolRequirement):
+            requirement = ToolRequirement.from_dict(requirement)
+        self.tool_requirements.add(requirement)
+
+    def __eq__(self, other):
+        return len(self.tool_requirements & other.tool_requirements) == len(self.tool_requirements) == len(other.tool_requirements)
+
+    def __ne__(self, other):
+        return not self.__eq__(other)
+
+    def __iter__(self):
+        for r in self.tool_requirements:
+            yield r
+
+    def __getitem__(self, ii):
+        return list(self.tool_requirements)[ii]
+
+    def __len__(self):
+        return len(self.tool_requirements)
+
+    def __hash__(self):
+        return sum([r.__hash__() for r in self.tool_requirements])
+
+
+class ToolRequirementsException(Exception):
+    pass
+
+
 DEFAULT_CONTAINER_TYPE = "docker"
 DEFAULT_CONTAINER_RESOLVE_DEPENDENCIES = False
 DEFAULT_CONTAINER_SHELL = "/bin/sh"  # Galaxy assumes bash, but containers are usually thinner.
@@ -76,7 +139,7 @@ class ContainerDescription( object ):
 def parse_requirements_from_dict( root_dict ):
     requirements = root_dict.get("requirements", [])
     containers = root_dict.get("containers", [])
-    return map(ToolRequirement.from_dict, requirements), map(ContainerDescription.from_dict, containers)
+    return ToolRequirements.from_list(requirements), map(ContainerDescription.from_dict, containers)
 
 
 def parse_requirements_from_xml( xml_root ):
@@ -108,7 +171,7 @@ def parse_requirements_from_xml( xml_root ):
     if requirements_elem is not None:
         requirement_elems = requirements_elem.findall( 'requirement' )
 
-    requirements = []
+    requirements = ToolRequirements()
     for requirement_elem in requirement_elems:
         name = xml_text( requirement_elem )
         type = requirement_elem.get( "type", DEFAULT_REQUIREMENT_TYPE )

lib/galaxy/tools/deps/views.py

@@ -45,14 +45,14 @@ class DependencyResolversView(object):
     def resolver_dependency(self, index, **kwds):
         return self._dependency(**kwds)
 
-    def show_dependencies(self, requirements, installed_tool_dependencies=None):
+    def show_dependencies(self, tool_requirements_d, installed_tool_dependencies=None):
         """
         Resolves dependencies to build a requirements status in the admin panel/API
         """
         kwds = {'install': False,
                 'return_null': True,
                 'installed_tool_dependencies': installed_tool_dependencies}
-        dependencies_per_tool = {tool: self._dependency_manager.requirements_to_dependencies(dependencies, **kwds) for tool, dependencies in requirements.items()}
+        dependencies_per_tool = {tool: self._dependency_manager.requirements_to_dependencies(requirements, **kwds) for tool, requirements in tool_requirements_d.items()}
         return dependencies_per_tool
 
     def install_dependencies(self, requirements):
@@ -138,12 +138,12 @@ class DependencyResolversView(object):
         """
         return [index for index, resolver in enumerate(self._dependency_resolvers) if hasattr(resolver, "install_dependency") and not resolver.disabled ]
 
-    def get_requirements_status(self, requested_requirements, installed_tool_dependencies=None):
-        dependencies = self.show_dependencies(requested_requirements, installed_tool_dependencies)
-        # dependencies is a dict keyed on tool_ids, values are lists of ToolRequirements for that tool.
-        # we collapse requested requirements to single set,
-        # then use collapsed requirements to get resolved dependencies without duplicates.
-        flat_tool_requirements = set([r for requirement_list in requested_requirements.values() for r in requirement_list])
+    def get_requirements_status(self, tool_requirements_d, installed_tool_dependencies=None):
+        dependencies = self.show_dependencies(tool_requirements_d, installed_tool_dependencies)
+        # dependencies is a dict keyed on tool_ids, value is a ToolRequirements object for that tool.
+        # We use the union of resolvable ToolRequirements to get resolved dependencies without duplicates.
+        requirements = [r.resolvable for r in tool_requirements_d.values()]
+        flat_tool_requirements = set().union(*requirements)
         flat_dependencies = []
         for requirements_odict in dependencies.values():
             for requirement in requirements_odict:

lib/galaxy/util/oset.py

@@ -0,0 +1,62 @@
+"""
+Ordered set implementation from https://code.activestate.com/recipes/576694/
+"""
+import collections
+
+
+class OrderedSet(collections.MutableSet):
+    def __init__(self, iterable=None):
+        self.end = end = []
+        end += [None, end, end]         # sentinel node for doubly linked list
+        self.map = {}                   # key --> [key, prev, next]
+        if iterable is not None:
+            self |= iterable
+
+    def __len__(self):
+        return len(self.map)
+
+    def __contains__(self, key):
+        return key in self.map
+
+    def add(self, key):
+        if key not in self.map:
+            end = self.end
+            curr = end[1]
+            curr[2] = end[1] = self.map[key] = [key, curr, end]
+
+    def discard(self, key):
+        if key in self.map:
+            key, prev, next = self.map.pop(key)
+            prev[2] = next
+            next[1] = prev
+
+    def __iter__(self):
+        end = self.end
+        curr = end[2]
+        while curr is not end:
+            yield curr[0]
+            curr = curr[2]
+
+    def __reversed__(self):
+        end = self.end
+        curr = end[1]
+        while curr is not end:
+            yield curr[0]
+            curr = curr[1]
+
+    def pop(self, last=True):
+        if not self:
+            raise KeyError('set is empty')
+        key = self.end[1][0] if last else self.end[2][0]
+        self.discard(key)
+        return key
+
+    def __repr__(self):
+        if not self:
+            return '%s()' % (self.__class__.__name__,)
+        return '%s(%r)' % (self.__class__.__name__, list(self))
+
+    def __eq__(self, other):
+        if isinstance(other, OrderedSet):
+            return len(self) == len(other) and list(self) == list(other)
+        return set(self) == set(other)

lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py

@@ -781,8 +781,8 @@ class AdminToolshed( AdminGalaxy ):
                                                                                 reinstalling=False,
                                                                                 required_repo_info_dicts=None )
         view = views.DependencyResolversView(self.app)
-        requirements = suc.get_requirements_from_repository(repository)
-        requirements_status = view.get_requirements_status(requirements, repository.installed_tool_dependencies)
+        tool_requirements_d = suc.get_requirements_from_repository(repository)
+        requirements_status = view.get_requirements_status(tool_requirements_d, repository.installed_tool_dependencies)
         return trans.fill_template( '/admin/tool_shed_repository/manage_repository.mako',
                                     repository=repository,
                                     description=description,

lib/tool_shed/galaxy_install/install_manager.py

@@ -905,14 +905,11 @@ class InstallRepositoryManager( object ):
             if 'tools' in metadata and install_resolver_dependencies:
                 self.update_tool_shed_repository_status( tool_shed_repository,
                                                          self.install_model.ToolShedRepository.installation_status.INSTALLING_TOOL_DEPENDENCIES )
-                installed_requirements = []
-                for tool_d in metadata['tools']:
-                    tool = self.app.toolbox._tools_by_id.get(tool_d['guid'], None)
-                    if tool and tool.requirements not in installed_requirements:
-                        self._view.install_dependencies(tool.requirements)
-                        installed_requirements.append(tool.requirements)
-                        if self.app.config.use_cached_dependency_manager:
-                            tool.build_dependency_cache()
+                new_tools = [self.app.toolbox._tools_by_id.get(tool_d['guid'], None) for tool_d in metadata['tools']]
+                new_requirements = set([tool.requirements.packages for tool in new_tools if tool])
+                [self._view.install_dependencies(r) for r in new_requirements]
+                if self.app.config.use_cached_dependency_manager:
+                    [self.app.toolbox.dependency_manager.build_cache(r) for r in new_requirements]
 
             if install_tool_dependencies and tool_shed_repository.tool_dependencies and 'tool_dependencies' in metadata:
                 work_dir = tempfile.mkdtemp( prefix="tmp-toolshed-itsr" )

lib/tool_shed/util/shed_util_common.py

@@ -225,7 +225,7 @@ def get_tool_shed_repo_requirements(app, tool_shed_url, repositories=None, repo_
 
 
 def get_requirements_from_tools(tools):
-    return {tool['id']: [galaxy.tools.deps.requirements.ToolRequirement.from_dict(r) for r in tool['requirements']] for tool in tools}
+    return {tool['id']: galaxy.tools.deps.requirements.ToolRequirements.from_list(tool['requirements']) for tool in tools}
 
 
 def get_requirements_from_repository(repository):

test/unit/tools/test_tool_deps.py

@@ -14,6 +14,10 @@ from stat import S_IXUSR
 from subprocess import PIPE, Popen
 
 from galaxy.tools.deps import DependencyManager
+from galaxy.tools.deps.requirements import (
+    ToolRequirement,
+    ToolRequirements
+)
 from galaxy.tools.deps.resolvers import NullDependency
 from galaxy.tools.deps.resolvers.galaxy_packages import GalaxyPackageDependency
 from galaxy.tools.deps.resolvers.modules import ModuleDependency, ModuleDependencyResolver
@@ -112,6 +116,30 @@ def __build_ts_test_package(base_path, script_contents=''):
     return package_dir
 
 
+REQUIREMENT_A = {'name': 'gnuplot',
+                 'type': 'package',
+                 'version': '4.6'}
+REQUIREMENT_B = REQUIREMENT_A.copy()
+REQUIREMENT_B['version'] = '4.7'
+
+
+def test_tool_requirement_equality():
+    a = ToolRequirement.from_dict(REQUIREMENT_A)
+    assert a == ToolRequirement(**REQUIREMENT_A)
+    b = ToolRequirement(**REQUIREMENT_B)
+    assert a != b
+
+
+def test_tool_requirements():
+    tool_requirements_ab = ToolRequirements([REQUIREMENT_A, REQUIREMENT_B])
+    tool_requirements_ab_dup = ToolRequirements([REQUIREMENT_A, REQUIREMENT_B])
+    tool_requirements_b = ToolRequirements([REQUIREMENT_A])
+    assert tool_requirements_ab == ToolRequirements([REQUIREMENT_B, REQUIREMENT_A])
+    assert tool_requirements_ab == ToolRequirements([REQUIREMENT_B, REQUIREMENT_A, REQUIREMENT_A])
+    assert tool_requirements_ab != tool_requirements_b
+    assert len(set([tool_requirements_ab, tool_requirements_ab_dup])) == 1
+
+
 def test_module_dependency_resolver():
     with __test_base_path() as temp_directory:
         module_script = os.path.join(temp_directory, "modulecmd")
@@ -193,7 +221,7 @@ def test_shell_commands_built():
     with __test_base_path() as base_path:
         dm = DependencyManager( default_base_path=base_path )
         __setup_galaxy_package_dep( base_path, TEST_REPO_NAME, TEST_VERSION, contents="export FOO=\"bar\"" )
-        mock_requirements = [ Bunch(type="package", version=TEST_VERSION, name=TEST_REPO_NAME ) ]
+        mock_requirements = ToolRequirements([{'type': 'package', 'version': TEST_VERSION, 'name': TEST_REPO_NAME}])
         commands = dm.dependency_shell_commands( mock_requirements )
         __assert_foo_exported( commands )