From 40a95a300326bddeb0e78add288a47f4f23c087c Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Sun, 27 Nov 2016 16:38:34 +0100 Subject: [PATCH 1/7] Add API to install resolver tool dependencies and to build a tool dependency cache (if activated in galaxy.ini). An example to install dependencies for the twobit converter: ``` import bioblend.galaxy url = 'http://localhost:8080/' api_key = 'admin_api_key' tool_id = 'CONVERTER_fasta_to_2bit' endpoint = "api/tools/%s/install_dependencies" % tool_id gi = bioblend.galaxy.GalaxyInstance(url, api_key) gi.make_post_request("/".join((url, endpoint)), payload={'id': tool_id}) ``` If `use_cached_dependency_manager` is activated in the galaxy.ini, a cached environment can be built like this: ``` endpoint = "api/tools/%s/build_dependency_cache" % tool_id gi.make_post_request("/".join((url, endpoint)), payload={'id': tool_id}) ``` --- .../datatypes/converters/fasta_to_2bit.xml | 1 + lib/galaxy/webapps/galaxy/api/tools.py | 27 +++++++++++++++++++ lib/galaxy/webapps/galaxy/buildapp.py | 2 ++ 3 files changed, 30 insertions(+) diff --git a/lib/galaxy/datatypes/converters/fasta_to_2bit.xml b/lib/galaxy/datatypes/converters/fasta_to_2bit.xml index a8e3bb3204f..f0e004661fb 100644 --- a/lib/galaxy/datatypes/converters/fasta_to_2bit.xml +++ b/lib/galaxy/datatypes/converters/fasta_to_2bit.xml @@ -2,6 +2,7 @@ + ucsc-twobittofa ucsc_tools faToTwoBit '$input' '$output' diff --git a/lib/galaxy/webapps/galaxy/api/tools.py b/lib/galaxy/webapps/galaxy/api/tools.py index 8dc7372738c..f91081d6b82 100644 --- a/lib/galaxy/webapps/galaxy/api/tools.py +++ b/lib/galaxy/webapps/galaxy/api/tools.py @@ -129,6 +129,33 @@ class ToolsController( BaseAPIController, UsesVisualizationMixin ): tool = self._get_tool(id) return tool.tool_requirements_status + @expose_api + @web.require_admin + def install_dependencies(self, trans, id, **kwds): + """ + POST /api/tools/{tool_id}/install_dependencies + Attempts to install requirements via the dependency resolver + """ + tool = self._get_tool(id) + [tool._view.install_dependency(id=None, **req.to_dict()) for req in tool.requirements] + if kwds.get('build_dependency_cache'): + tool.build_dependency_cache() + # TODO: rework resolver install system to log and report what has been done. + # _view.install_dependency should return a dict with stdout, stderr and success status + return tool.tool_requirements_status + + @expose_api + @web.require_admin + def build_dependency_cache(self, trans, id, **kwds): + """ + POST /api/tools/{tool_id}/build_dependency_cache + Attempts to cache installed dependencies. + """ + tool = self._get_tool(id) + tool.build_dependency_cache() + # TODO: Should also have a more meaningful return. + return tool.tool_requirements_status + @expose_api @web.require_admin def diagnostics( self, trans, id, **kwd ): diff --git a/lib/galaxy/webapps/galaxy/buildapp.py b/lib/galaxy/webapps/galaxy/buildapp.py index 53d0c948383..75eee3a7d4b 100644 --- a/lib/galaxy/webapps/galaxy/buildapp.py +++ b/lib/galaxy/webapps/galaxy/buildapp.py @@ -262,6 +262,8 @@ def populate_api_routes( webapp, app ): webapp.mapper.connect( '/api/tools/{id:.+?}/citations', action='citations', controller="tools" ) webapp.mapper.connect( '/api/tools/{id:.+?}/download', action='download', controller="tools" ) webapp.mapper.connect( '/api/tools/{id:.+?}/requirements', action='requirements', controller="tools") + webapp.mapper.connect( '/api/tools/{id:.+?}/install_dependencies', action='install_dependencies', controller="tools", conditions=dict( method=[ "POST" ] )) + webapp.mapper.connect( '/api/tools/{id:.+?}/build_dependency_cache', action='build_dependency_cache', controller="tools", conditions=dict( method=[ "POST" ] )) webapp.mapper.connect( '/api/tools/{id:.+?}', action='show', controller="tools" ) webapp.mapper.resource( 'tool', 'tools', path_prefix='/api' ) From e905b8baeb9081400ad775bcc20c2956808a11fd Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Sun, 27 Nov 2016 17:49:59 +0100 Subject: [PATCH 2/7] Add integration test-case for install tool dependency API endpoint --- test/integration/test_resolvers.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/test/integration/test_resolvers.py b/test/integration/test_resolvers.py index f1ee7d9b8eb..cf8993f0e8a 100644 --- a/test/integration/test_resolvers.py +++ b/test/integration/test_resolvers.py @@ -17,6 +17,7 @@ class CondaResolutionIntegrationTestCase(integration_util.IntegrationTestCase, A @classmethod def handle_galaxy_config_kwds(cls, config): cls.conda_tmp_prefix = mkdtemp() + config["use_cached_dep_manager"] = True config["conda_auto_init"] = True config["conda_prefix"] = os.path.join(cls.conda_tmp_prefix, 'conda') @@ -82,3 +83,15 @@ class CondaResolutionIntegrationTestCase(integration_util.IntegrationTestCase, A self._assert_status_code_is( create_response, 200 ) response = create_response.json() assert response['dependency_type'] == 'conda' and not response['exact'] + + def test_conda_install_through_tools_api( self ): + tool_id = 'mulled_example_multi_1' + endpoint = "tools/%s/install_dependencies" % tool_id + data = {'id': tool_id} + create_response = self._post(endpoint, data=data, admin=True) + self._assert_status_code_is( create_response, 200 ) + response = create_response.json() + assert any([True for d in response if d['dependency_type'] == 'conda']) + endpoint = "tools/%s/build_dependency_cache" % tool_id + create_response = self._post(endpoint, data=data, admin=True) + self._assert_status_code_is( create_response, 200 ) From 3e48f3792bc10c081a8079cfa3d773531d6dad15 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Sun, 27 Nov 2016 18:15:22 +0100 Subject: [PATCH 3/7] Install the correct converter dependency --- lib/galaxy/datatypes/converters/fasta_to_2bit.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/datatypes/converters/fasta_to_2bit.xml b/lib/galaxy/datatypes/converters/fasta_to_2bit.xml index f0e004661fb..3ccd7ab7db8 100644 --- a/lib/galaxy/datatypes/converters/fasta_to_2bit.xml +++ b/lib/galaxy/datatypes/converters/fasta_to_2bit.xml @@ -2,7 +2,7 @@ - ucsc-twobittofa + ucsc-fatotwobit ucsc_tools faToTwoBit '$input' '$output' From 8ff7ffa063c491b886bbeed1e1eb2e69fb6f3e68 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Sun, 27 Nov 2016 18:39:13 +0100 Subject: [PATCH 4/7] Add `force_rebuild` param to build_dependency_cache This will remove a pre-existing cached dependency directory. --- lib/galaxy/tools/__init__.py | 5 +++-- lib/galaxy/tools/deps/__init__.py | 7 +++++++ lib/galaxy/webapps/galaxy/api/tools.py | 11 +++++++++-- 3 files changed, 19 insertions(+), 4 deletions(-) diff --git a/lib/galaxy/tools/__init__.py b/lib/galaxy/tools/__init__.py index 34a34b37f31..ba8ee6ce1a2 100755 --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -1306,7 +1306,7 @@ class Tool( object, Dictifiable ): visit_input_values( self.inputs, values, validate_inputs ) return messages - def build_dependency_cache(self): + def build_dependency_cache(self, **kwds): if isinstance(self.app.toolbox.dependency_manager, CachedDependencyManager): self.app.toolbox.dependency_manager.build_cache( requirements=self.requirements, @@ -1314,7 +1314,8 @@ class Tool( object, Dictifiable ): tool_dir=self.tool_dir, job_directory=None, metadata=False, - tool_instance=self + tool_instance=self, + **kwds ) def build_dependency_shell_commands( self, job_directory=None, metadata=False ): diff --git a/lib/galaxy/tools/deps/__init__.py b/lib/galaxy/tools/deps/__init__.py index 4f0a6d2a7d4..640726f73d9 100644 --- a/lib/galaxy/tools/deps/__init__.py +++ b/lib/galaxy/tools/deps/__init__.py @@ -5,6 +5,7 @@ Dependency management for tools. import json import logging import os.path +import shutil from collections import OrderedDict @@ -175,6 +176,12 @@ class CachedDependencyManager(DependencyManager): resolved_dependencies = self.requirements_to_dependencies(requirements, **kwds) cacheable_dependencies = [dep for req, dep in resolved_dependencies.items() if dep.cacheable] hashed_requirements_dir = self.get_hashed_requirements_path(cacheable_dependencies) + if kwds.get('force_rebuild', False) and os.path.exists(hashed_requirements_dir): + try: + shutil.rmtree(hashed_requirements_dir) + except Exception: + log.warning("Could not delete cached requirements directory '%s'" % hashed_requirements_dir) + pass [dep.build_cache(hashed_requirements_dir) for dep in cacheable_dependencies] def dependency_shell_commands( self, requirements, **kwds ): diff --git a/lib/galaxy/webapps/galaxy/api/tools.py b/lib/galaxy/webapps/galaxy/api/tools.py index f91081d6b82..b76dd9a3f40 100644 --- a/lib/galaxy/webapps/galaxy/api/tools.py +++ b/lib/galaxy/webapps/galaxy/api/tools.py @@ -135,11 +135,15 @@ class ToolsController( BaseAPIController, UsesVisualizationMixin ): """ POST /api/tools/{tool_id}/install_dependencies Attempts to install requirements via the dependency resolver + + parameters: + build_dependency_cache: If true, attempts to cache dependencies for this tool + force_rebuild: If true and chache dir exists, attempts to delete cache dir """ tool = self._get_tool(id) [tool._view.install_dependency(id=None, **req.to_dict()) for req in tool.requirements] if kwds.get('build_dependency_cache'): - tool.build_dependency_cache() + tool.build_dependency_cache(**kwds) # TODO: rework resolver install system to log and report what has been done. # _view.install_dependency should return a dict with stdout, stderr and success status return tool.tool_requirements_status @@ -150,9 +154,12 @@ class ToolsController( BaseAPIController, UsesVisualizationMixin ): """ POST /api/tools/{tool_id}/build_dependency_cache Attempts to cache installed dependencies. + + parameters: + force_rebuild: If true and chache dir exists, attempts to delete cache dir """ tool = self._get_tool(id) - tool.build_dependency_cache() + tool.build_dependency_cache(**kwds) # TODO: Should also have a more meaningful return. return tool.tool_requirements_status From 3ebbb37c4dc09648ee08877eeb87a1db90a75d35 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Sun, 27 Nov 2016 23:16:56 +0100 Subject: [PATCH 5/7] Add dependency_resolvers/clean endpoint --- lib/galaxy/tools/deps/resolvers/conda.py | 4 ++++ lib/galaxy/tools/deps/views.py | 14 ++++++++++++++ .../webapps/galaxy/api/tool_dependencies.py | 19 +++++++++++++++++++ lib/galaxy/webapps/galaxy/buildapp.py | 1 + test/integration/test_resolvers.py | 7 +++++++ 5 files changed, 45 insertions(+) diff --git a/lib/galaxy/tools/deps/resolvers/conda.py b/lib/galaxy/tools/deps/resolvers/conda.py index 5751212626e..beb06871028 100644 --- a/lib/galaxy/tools/deps/resolvers/conda.py +++ b/lib/galaxy/tools/deps/resolvers/conda.py @@ -11,6 +11,7 @@ import galaxy.tools.deps.installable from ..conda_util import ( build_isolated_environment, cleanup_failed_install, + exec_clean, CondaContext, CondaTarget, install_conda, @@ -98,6 +99,9 @@ class CondaDependencyResolver(DependencyResolver, ListableDependencyResolver, In self.auto_install = auto_install self.copy_dependencies = copy_dependencies + def clean(self, **kwds): + return exec_clean() + def resolve(self, name, version, type, **kwds): # Check for conda just not being there, this way we can enable # conda by default and just do nothing in not configured. diff --git a/lib/galaxy/tools/deps/views.py b/lib/galaxy/tools/deps/views.py index be4be6458b3..0735d4af4c8 100644 --- a/lib/galaxy/tools/deps/views.py +++ b/lib/galaxy/tools/deps/views.py @@ -125,5 +125,19 @@ class DependencyResolversView(object): """ return [index for index, resolver in enumerate(self._dependency_resolvers) if hasattr(resolver, "install_dependency") and not resolver.disabled ] + def get_requirements_status(self, requested_requirements, installed_tool_dependencies=None): return [self.manager_dependency(installed_tool_dependencies=installed_tool_dependencies, **req) for req in requested_requirements] + + def clean(self, index=None, **kwds): + if index: + resolver = self._dependency_resolver(index) + if not hasattr(resolver, "clean"): + raise NotImplemented() + else: + resolver.clean() + return "OK" + else: + [resolver.clean(**kwds) for resolver in self._dependency_resolvers if hasattr(resolver, 'clean')] + return "OK" + diff --git a/lib/galaxy/webapps/galaxy/api/tool_dependencies.py b/lib/galaxy/webapps/galaxy/api/tool_dependencies.py index f91c93ff5a5..cd961472355 100644 --- a/lib/galaxy/webapps/galaxy/api/tool_dependencies.py +++ b/lib/galaxy/webapps/galaxy/api/tool_dependencies.py @@ -162,3 +162,22 @@ class ToolDependenciesAPIController( BaseAPIController ): the corresponding resolver (keyed on 'index'). """ return self._view.manager_requirements() + + @expose_api + @require_admin + def clean(self, trans, id=None, **kwds): + """ + POST /api/dependencies_resolver/clean + + Cleans up intermediate files created by resolvers during the dependency + installation. + + :type index: int + :param index: index of the dependency resolver + + :rtype: dict + :returns: a dictified description of the requirement that could + be resolved (keyed on 'requirement') and the index of + the corresponding resolver (keyed on 'index'). + """ + return self._view.clean(id, **kwds) diff --git a/lib/galaxy/webapps/galaxy/buildapp.py b/lib/galaxy/webapps/galaxy/buildapp.py index 75eee3a7d4b..52b9fb3882a 100644 --- a/lib/galaxy/webapps/galaxy/buildapp.py +++ b/lib/galaxy/webapps/galaxy/buildapp.py @@ -270,6 +270,7 @@ def populate_api_routes( webapp, app ): webapp.mapper.connect( '/api/dependency_resolvers/dependency', action="manager_dependency", controller="tool_dependencies", conditions=dict( method=[ "GET" ] ) ) webapp.mapper.connect( '/api/dependency_resolvers/dependency', action="install_dependency", controller="tool_dependencies", conditions=dict( method=[ "POST" ] ) ) webapp.mapper.connect( '/api/dependency_resolvers/requirements', action="manager_requirements", controller="tool_dependencies" ) + webapp.mapper.connect('/api/dependency_resolvers/clean', action="clean", controller="tool_dependencies", conditions=dict(method=["POST"])) webapp.mapper.connect( '/api/dependency_resolvers/{id}/dependency', action="resolver_dependency", controller="tool_dependencies", conditions=dict( method=[ "GET" ] ) ) webapp.mapper.connect( '/api/dependency_resolvers/{id}/dependency', action="install_dependency", controller="tool_dependencies", conditions=dict( method=[ "POST" ] ) ) webapp.mapper.connect( '/api/dependency_resolvers/{id}/requirements', action="resolver_requirements", controller="tool_dependencies" ) diff --git a/test/integration/test_resolvers.py b/test/integration/test_resolvers.py index cf8993f0e8a..d2f1208432f 100644 --- a/test/integration/test_resolvers.py +++ b/test/integration/test_resolvers.py @@ -95,3 +95,10 @@ class CondaResolutionIntegrationTestCase(integration_util.IntegrationTestCase, A endpoint = "tools/%s/build_dependency_cache" % tool_id create_response = self._post(endpoint, data=data, admin=True) self._assert_status_code_is( create_response, 200 ) + + def test_conda_clean( self ): + endpoint = 'dependencies_resolvers/clean' + create_response = self._post(endpoint, data={}, admin=True) + self._assert_status_code_is(create_response, 200) + response = create_response.json() + assert response == "OK" \ No newline at end of file From 23c6b1c6ff6b810fed69c05eec74166cd2d8fc5f Mon Sep 17 00:00:00 2001 From: Bjoern Gruening Date: Fri, 16 Sep 2016 18:46:37 +0200 Subject: [PATCH 6/7] add a conda clean function to remove downloaded tarballs after installation --- lib/galaxy/tools/deps/conda_util.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/lib/galaxy/tools/deps/conda_util.py b/lib/galaxy/tools/deps/conda_util.py index 42ddc44f77d..4f12716f927 100644 --- a/lib/galaxy/tools/deps/conda_util.py +++ b/lib/galaxy/tools/deps/conda_util.py @@ -214,6 +214,16 @@ class CondaContext(installable.InstallableContext): install_base_args.extend(args) return self.exec_command("install", install_base_args) + def exec_clean(self, args=[]): + """ + Clean up after conda installation. + """ + clean_base_args = [ + "--tarballs" + ] + clean_base_args.extend(args) + return self.exec_command("clean", clean_base_args) + def export_list(self, name, path): return self.exec_command("list", [ "--name", name, @@ -488,6 +498,7 @@ def build_isolated_environment( return (path or tempdir_name, exit_code) finally: + conda_context.exec_clean() shutil.rmtree(tempdir) From 00ca0a600a083bf05e2c8f90610851972e7e94ea Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Mon, 28 Nov 2016 00:07:40 +0100 Subject: [PATCH 7/7] Connect `clean` endpoint properly and add `-y` to exec_clean --- lib/galaxy/tools/deps/conda_util.py | 3 ++- lib/galaxy/tools/deps/resolvers/conda.py | 3 +-- lib/galaxy/tools/deps/views.py | 2 -- lib/galaxy/webapps/galaxy/api/tool_dependencies.py | 2 +- lib/galaxy/webapps/galaxy/buildapp.py | 3 ++- test/integration/test_resolvers.py | 4 ++-- 6 files changed, 8 insertions(+), 9 deletions(-) diff --git a/lib/galaxy/tools/deps/conda_util.py b/lib/galaxy/tools/deps/conda_util.py index 4f12716f927..47f1b090006 100644 --- a/lib/galaxy/tools/deps/conda_util.py +++ b/lib/galaxy/tools/deps/conda_util.py @@ -219,7 +219,8 @@ class CondaContext(installable.InstallableContext): Clean up after conda installation. """ clean_base_args = [ - "--tarballs" + "--tarballs", + "-y" ] clean_base_args.extend(args) return self.exec_command("clean", clean_base_args) diff --git a/lib/galaxy/tools/deps/resolvers/conda.py b/lib/galaxy/tools/deps/resolvers/conda.py index beb06871028..a21a37ebd4a 100644 --- a/lib/galaxy/tools/deps/resolvers/conda.py +++ b/lib/galaxy/tools/deps/resolvers/conda.py @@ -11,7 +11,6 @@ import galaxy.tools.deps.installable from ..conda_util import ( build_isolated_environment, cleanup_failed_install, - exec_clean, CondaContext, CondaTarget, install_conda, @@ -100,7 +99,7 @@ class CondaDependencyResolver(DependencyResolver, ListableDependencyResolver, In self.copy_dependencies = copy_dependencies def clean(self, **kwds): - return exec_clean() + return self.conda_context.exec_clean() def resolve(self, name, version, type, **kwds): # Check for conda just not being there, this way we can enable diff --git a/lib/galaxy/tools/deps/views.py b/lib/galaxy/tools/deps/views.py index 0735d4af4c8..8f23335d7df 100644 --- a/lib/galaxy/tools/deps/views.py +++ b/lib/galaxy/tools/deps/views.py @@ -125,7 +125,6 @@ class DependencyResolversView(object): """ return [index for index, resolver in enumerate(self._dependency_resolvers) if hasattr(resolver, "install_dependency") and not resolver.disabled ] - def get_requirements_status(self, requested_requirements, installed_tool_dependencies=None): return [self.manager_dependency(installed_tool_dependencies=installed_tool_dependencies, **req) for req in requested_requirements] @@ -140,4 +139,3 @@ class DependencyResolversView(object): else: [resolver.clean(**kwds) for resolver in self._dependency_resolvers if hasattr(resolver, 'clean')] return "OK" - diff --git a/lib/galaxy/webapps/galaxy/api/tool_dependencies.py b/lib/galaxy/webapps/galaxy/api/tool_dependencies.py index cd961472355..ea5821dfea3 100644 --- a/lib/galaxy/webapps/galaxy/api/tool_dependencies.py +++ b/lib/galaxy/webapps/galaxy/api/tool_dependencies.py @@ -167,7 +167,7 @@ class ToolDependenciesAPIController( BaseAPIController ): @require_admin def clean(self, trans, id=None, **kwds): """ - POST /api/dependencies_resolver/clean + POST /api/dependencies_resolver/{index}/clean Cleans up intermediate files created by resolvers during the dependency installation. diff --git a/lib/galaxy/webapps/galaxy/buildapp.py b/lib/galaxy/webapps/galaxy/buildapp.py index 52b9fb3882a..8bee0a15ddb 100644 --- a/lib/galaxy/webapps/galaxy/buildapp.py +++ b/lib/galaxy/webapps/galaxy/buildapp.py @@ -267,10 +267,11 @@ def populate_api_routes( webapp, app ): webapp.mapper.connect( '/api/tools/{id:.+?}', action='show', controller="tools" ) webapp.mapper.resource( 'tool', 'tools', path_prefix='/api' ) + webapp.mapper.connect( '/api/dependency_resolvers/clean', action="clean", controller="tool_dependencies", conditions=dict( method=[ "POST" ]) ) webapp.mapper.connect( '/api/dependency_resolvers/dependency', action="manager_dependency", controller="tool_dependencies", conditions=dict( method=[ "GET" ] ) ) webapp.mapper.connect( '/api/dependency_resolvers/dependency', action="install_dependency", controller="tool_dependencies", conditions=dict( method=[ "POST" ] ) ) webapp.mapper.connect( '/api/dependency_resolvers/requirements', action="manager_requirements", controller="tool_dependencies" ) - webapp.mapper.connect('/api/dependency_resolvers/clean', action="clean", controller="tool_dependencies", conditions=dict(method=["POST"])) + webapp.mapper.connect( '/api/dependency_resolvers/{id}/clean', action="clean", controller="tool_dependencies", conditions=dict( method=[ "POST" ]) ) webapp.mapper.connect( '/api/dependency_resolvers/{id}/dependency', action="resolver_dependency", controller="tool_dependencies", conditions=dict( method=[ "GET" ] ) ) webapp.mapper.connect( '/api/dependency_resolvers/{id}/dependency', action="install_dependency", controller="tool_dependencies", conditions=dict( method=[ "POST" ] ) ) webapp.mapper.connect( '/api/dependency_resolvers/{id}/requirements', action="resolver_requirements", controller="tool_dependencies" ) diff --git a/test/integration/test_resolvers.py b/test/integration/test_resolvers.py index d2f1208432f..5b134ac54ee 100644 --- a/test/integration/test_resolvers.py +++ b/test/integration/test_resolvers.py @@ -97,8 +97,8 @@ class CondaResolutionIntegrationTestCase(integration_util.IntegrationTestCase, A self._assert_status_code_is( create_response, 200 ) def test_conda_clean( self ): - endpoint = 'dependencies_resolvers/clean' + endpoint = 'dependency_resolvers/clean' create_response = self._post(endpoint, data={}, admin=True) self._assert_status_code_is(create_response, 200) response = create_response.json() - assert response == "OK" \ No newline at end of file + assert response == "OK"