Merge pull request #3222 from mvdbeek/api_cache_env

Add API to install resolver tool dependencies

lib/galaxy/datatypes/converters/fasta_to_2bit.xml

@@ -2,6 +2,7 @@
     <!-- <description>__NOT_USED_CURRENTLY_FOR_CONVERTERS__</description> -->
     <!-- Used on the metadata edit page. -->
     <requirements>
+        <requirement type="package" version="332">ucsc-fatotwobit</requirement>
         <requirement type="package">ucsc_tools</requirement>
     </requirements>
     <command>faToTwoBit '$input' '$output'</command>

lib/galaxy/tools/__init__.py

@@ -1306,7 +1306,7 @@ class Tool( object, Dictifiable ):
         visit_input_values( self.inputs, values, validate_inputs )
         return messages
 
-    def build_dependency_cache(self):
+    def build_dependency_cache(self, **kwds):
         if isinstance(self.app.toolbox.dependency_manager, CachedDependencyManager):
             self.app.toolbox.dependency_manager.build_cache(
                 requirements=self.requirements,
@@ -1314,7 +1314,8 @@ class Tool( object, Dictifiable ):
                 tool_dir=self.tool_dir,
                 job_directory=None,
                 metadata=False,
-                tool_instance=self
+                tool_instance=self,
+                **kwds
             )
 
     def build_dependency_shell_commands( self, job_directory=None, metadata=False ):

lib/galaxy/tools/deps/__init__.py

@@ -5,6 +5,7 @@ Dependency management for tools.
 import json
 import logging
 import os.path
+import shutil
 
 from collections import OrderedDict
 
@@ -175,6 +176,12 @@ class CachedDependencyManager(DependencyManager):
         resolved_dependencies = self.requirements_to_dependencies(requirements, **kwds)
         cacheable_dependencies = [dep for req, dep in resolved_dependencies.items() if dep.cacheable]
         hashed_requirements_dir = self.get_hashed_requirements_path(cacheable_dependencies)
+        if kwds.get('force_rebuild', False) and os.path.exists(hashed_requirements_dir):
+            try:
+                shutil.rmtree(hashed_requirements_dir)
+            except Exception:
+                log.warning("Could not delete cached requirements directory '%s'" % hashed_requirements_dir)
+                pass
         [dep.build_cache(hashed_requirements_dir) for dep in cacheable_dependencies]
 
     def dependency_shell_commands( self, requirements, **kwds ):

lib/galaxy/tools/deps/conda_util.py

@@ -214,6 +214,17 @@ class CondaContext(installable.InstallableContext):
         install_base_args.extend(args)
         return self.exec_command("install", install_base_args)
 
+    def exec_clean(self, args=[]):
+        """
+        Clean up after conda installation.
+        """
+        clean_base_args = [
+            "--tarballs",
+            "-y"
+        ]
+        clean_base_args.extend(args)
+        return self.exec_command("clean", clean_base_args)
+
     def export_list(self, name, path):
         return self.exec_command("list", [
             "--name", name,
@@ -488,6 +499,7 @@ def build_isolated_environment(
 
         return (path or tempdir_name, exit_code)
     finally:
+        conda_context.exec_clean()
         shutil.rmtree(tempdir)
 
 

lib/galaxy/tools/deps/resolvers/conda.py

@@ -98,6 +98,9 @@ class CondaDependencyResolver(DependencyResolver, ListableDependencyResolver, In
         self.auto_install = auto_install
         self.copy_dependencies = copy_dependencies
 
+    def clean(self, **kwds):
+        return self.conda_context.exec_clean()
+
     def resolve(self, name, version, type, **kwds):
         # Check for conda just not being there, this way we can enable
         # conda by default and just do nothing in not configured.

lib/galaxy/tools/deps/views.py

@@ -127,3 +127,15 @@ class DependencyResolversView(object):
 
     def get_requirements_status(self, requested_requirements, installed_tool_dependencies=None):
         return [self.manager_dependency(installed_tool_dependencies=installed_tool_dependencies, **req) for req in requested_requirements]
+
+    def clean(self, index=None, **kwds):
+        if index:
+            resolver = self._dependency_resolver(index)
+            if not hasattr(resolver, "clean"):
+                raise NotImplemented()
+            else:
+                resolver.clean()
+                return "OK"
+        else:
+            [resolver.clean(**kwds) for resolver in self._dependency_resolvers if hasattr(resolver, 'clean')]
+            return "OK"

lib/galaxy/webapps/galaxy/api/tool_dependencies.py

@@ -162,3 +162,22 @@ class ToolDependenciesAPIController( BaseAPIController ):
                     the corresponding resolver (keyed on 'index').
         """
         return self._view.manager_requirements()
+
+    @expose_api
+    @require_admin
+    def clean(self, trans, id=None, **kwds):
+        """
+        POST /api/dependencies_resolver/{index}/clean
+
+        Cleans up intermediate files created by resolvers during the dependency
+        installation.
+
+        :type   index:    int
+        :param  index:    index of the dependency resolver
+
+        :rtype:     dict
+        :returns:   a dictified description of the requirement that could
+                    be resolved (keyed on 'requirement') and the index of
+                    the corresponding resolver (keyed on 'index').
+        """
+        return self._view.clean(id, **kwds)

lib/galaxy/webapps/galaxy/api/tools.py

@@ -129,6 +129,40 @@ class ToolsController( BaseAPIController, UsesVisualizationMixin ):
         tool = self._get_tool(id)
         return tool.tool_requirements_status
 
+    @expose_api
+    @web.require_admin
+    def install_dependencies(self, trans, id, **kwds):
+        """
+        POST /api/tools/{tool_id}/install_dependencies
+        Attempts to install requirements via the dependency resolver
+
+        parameters:
+            build_dependency_cache:  If true, attempts to cache dependencies for this tool
+            force_rebuild:           If true and chache dir exists, attempts to delete cache dir
+        """
+        tool = self._get_tool(id)
+        [tool._view.install_dependency(id=None, **req.to_dict()) for req in tool.requirements]
+        if kwds.get('build_dependency_cache'):
+            tool.build_dependency_cache(**kwds)
+        # TODO: rework resolver install system to log and report what has been done.
+        # _view.install_dependency should return a dict with stdout, stderr and success status
+        return tool.tool_requirements_status
+
+    @expose_api
+    @web.require_admin
+    def build_dependency_cache(self, trans, id, **kwds):
+        """
+        POST /api/tools/{tool_id}/build_dependency_cache
+        Attempts to cache installed dependencies.
+
+        parameters:
+            force_rebuild:           If true and chache dir exists, attempts to delete cache dir
+        """
+        tool = self._get_tool(id)
+        tool.build_dependency_cache(**kwds)
+        # TODO: Should also have a more meaningful return.
+        return tool.tool_requirements_status
+
     @expose_api
     @web.require_admin
     def diagnostics( self, trans, id, **kwd ):

lib/galaxy/webapps/galaxy/buildapp.py

@@ -262,12 +262,16 @@ def populate_api_routes( webapp, app ):
     webapp.mapper.connect( '/api/tools/{id:.+?}/citations', action='citations', controller="tools" )
     webapp.mapper.connect( '/api/tools/{id:.+?}/download', action='download', controller="tools" )
     webapp.mapper.connect( '/api/tools/{id:.+?}/requirements', action='requirements', controller="tools")
+    webapp.mapper.connect( '/api/tools/{id:.+?}/install_dependencies', action='install_dependencies', controller="tools", conditions=dict( method=[ "POST" ] ))
+    webapp.mapper.connect( '/api/tools/{id:.+?}/build_dependency_cache', action='build_dependency_cache', controller="tools", conditions=dict( method=[ "POST" ] ))
     webapp.mapper.connect( '/api/tools/{id:.+?}', action='show', controller="tools" )
     webapp.mapper.resource( 'tool', 'tools', path_prefix='/api' )
 
+    webapp.mapper.connect( '/api/dependency_resolvers/clean', action="clean", controller="tool_dependencies", conditions=dict( method=[ "POST" ]) )
     webapp.mapper.connect( '/api/dependency_resolvers/dependency', action="manager_dependency", controller="tool_dependencies", conditions=dict( method=[ "GET" ] ) )
     webapp.mapper.connect( '/api/dependency_resolvers/dependency', action="install_dependency", controller="tool_dependencies", conditions=dict( method=[ "POST" ] ) )
     webapp.mapper.connect( '/api/dependency_resolvers/requirements', action="manager_requirements", controller="tool_dependencies" )
+    webapp.mapper.connect( '/api/dependency_resolvers/{id}/clean', action="clean", controller="tool_dependencies", conditions=dict( method=[ "POST" ]) )
     webapp.mapper.connect( '/api/dependency_resolvers/{id}/dependency', action="resolver_dependency", controller="tool_dependencies", conditions=dict( method=[ "GET" ] ) )
     webapp.mapper.connect( '/api/dependency_resolvers/{id}/dependency', action="install_dependency", controller="tool_dependencies", conditions=dict( method=[ "POST" ] ) )
     webapp.mapper.connect( '/api/dependency_resolvers/{id}/requirements', action="resolver_requirements", controller="tool_dependencies" )

test/integration/test_resolvers.py

@@ -17,6 +17,7 @@ class CondaResolutionIntegrationTestCase(integration_util.IntegrationTestCase, A
     @classmethod
     def handle_galaxy_config_kwds(cls, config):
         cls.conda_tmp_prefix = mkdtemp()
+        config["use_cached_dep_manager"] = True
         config["conda_auto_init"] = True
         config["conda_prefix"] = os.path.join(cls.conda_tmp_prefix, 'conda')
 
@@ -82,3 +83,22 @@ class CondaResolutionIntegrationTestCase(integration_util.IntegrationTestCase, A
         self._assert_status_code_is( create_response, 200 )
         response = create_response.json()
         assert response['dependency_type'] == 'conda' and not response['exact']
+
+    def test_conda_install_through_tools_api( self ):
+        tool_id = 'mulled_example_multi_1'
+        endpoint = "tools/%s/install_dependencies" % tool_id
+        data = {'id': tool_id}
+        create_response = self._post(endpoint, data=data, admin=True)
+        self._assert_status_code_is( create_response, 200 )
+        response = create_response.json()
+        assert any([True for d in response if d['dependency_type'] == 'conda'])
+        endpoint = "tools/%s/build_dependency_cache" % tool_id
+        create_response = self._post(endpoint, data=data, admin=True)
+        self._assert_status_code_is( create_response, 200 )
+
+    def test_conda_clean( self ):
+        endpoint = 'dependency_resolvers/clean'
+        create_response = self._post(endpoint, data={}, admin=True)
+        self._assert_status_code_is(create_response, 200)
+        response = create_response.json()
+        assert response == "OK"