From 58a2cde89fdc5e52c4b6b10842d47f1e1f24bfca Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Tue, 27 Jun 2017 15:02:43 +0200 Subject: [PATCH 1/5] Refactor deactivate_or_uninstall_repository Move code that deals with repository uninstallation into InstalledRepositoryManager, so that this can be reused by the API. --- .../galaxy/controllers/admin_toolshed.py | 71 +------------------ .../installed_repository_manager.py | 70 ++++++++++++++++++ 2 files changed, 73 insertions(+), 68 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py b/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py index b02553419a5..3f966bc4ad6 100644 --- a/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py +++ b/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py @@ -1,7 +1,6 @@ import json import logging import os -import shutil from admin import AdminGalaxy from six import string_types @@ -13,8 +12,8 @@ from galaxy.tools.deps import views from galaxy.web.form_builder import CheckboxField from tool_shed.galaxy_install import dependency_display from tool_shed.galaxy_install import install_manager -from tool_shed.galaxy_install.datatypes import custom_datatype_manager from tool_shed.galaxy_install.grids import admin_toolshed_grids +from tool_shed.galaxy_install.installed_repository_manager import InstalledRepositoryManager from tool_shed.galaxy_install.metadata.installed_repository_metadata_manager import InstalledRepositoryMetadataManager from tool_shed.galaxy_install.repair_repository_manager import RepairRepositoryManager from tool_shed.galaxy_install.repository_dependencies import repository_dependency_manager @@ -249,74 +248,10 @@ class AdminToolshed( AdminGalaxy ): tool_shed_repositories = repository_util.get_installed_tool_shed_repository( trans.app, kwd[ 'id' ] ) if not isinstance( tool_shed_repositories, list ): tool_shed_repositories = [tool_shed_repositories] + irm = InstalledRepositoryManager(app=trans.app) for tool_shed_repository in tool_shed_repositories: - shed_tool_conf, tool_path, relative_install_dir = \ - suc.get_tool_panel_config_tool_path_install_dir( trans.app, tool_shed_repository ) - if relative_install_dir: - if tool_path: - relative_install_dir = os.path.join( tool_path, relative_install_dir ) - repository_install_dir = os.path.abspath( relative_install_dir ) - else: - repository_install_dir = None - errors = '' if kwd.get( 'deactivate_or_uninstall_repository_button', False ): - if tool_shed_repository.includes_tools_for_display_in_tool_panel: - # Handle tool panel alterations. - tpm = tool_panel_manager.ToolPanelManager( trans.app ) - tpm.remove_repository_contents( tool_shed_repository, - shed_tool_conf, - uninstall=remove_from_disk_checked ) - if tool_shed_repository.includes_data_managers: - dmh = data_manager.DataManagerHandler( trans.app ) - dmh.remove_from_data_manager( tool_shed_repository ) - if tool_shed_repository.includes_datatypes: - # Deactivate proprietary datatypes. - cdl = custom_datatype_manager.CustomDatatypeLoader( trans.app ) - installed_repository_dict = cdl.load_installed_datatypes( tool_shed_repository, - repository_install_dir, - deactivate=True ) - if installed_repository_dict: - converter_path = installed_repository_dict.get( 'converter_path' ) - if converter_path is not None: - cdl.load_installed_datatype_converters( installed_repository_dict, deactivate=True ) - display_path = installed_repository_dict.get( 'display_path' ) - if display_path is not None: - cdl.load_installed_display_applications( installed_repository_dict, deactivate=True ) - if remove_from_disk_checked: - try: - # Remove the repository from disk. - shutil.rmtree( repository_install_dir ) - log.debug( "Removed repository installation directory: %s" % str( repository_install_dir ) ) - removed = True - except Exception as e: - log.debug( "Error removing repository installation directory %s: %s" % ( str( repository_install_dir ), str( e ) ) ) - if isinstance( e, OSError ) and not os.path.exists( repository_install_dir ): - removed = True - log.debug( "Repository directory does not exist on disk, marking as uninstalled." ) - else: - removed = False - if removed: - tool_shed_repository.uninstalled = True - # Remove all installed tool dependencies and tool dependencies stuck in the INSTALLING state, but don't touch any - # repository dependencies. - tool_dependencies_to_uninstall = tool_shed_repository.tool_dependencies_installed_or_in_error - tool_dependencies_to_uninstall.extend( tool_shed_repository.tool_dependencies_being_installed ) - for tool_dependency in tool_dependencies_to_uninstall: - uninstalled, error_message = tool_dependency_util.remove_tool_dependency( trans.app, tool_dependency ) - if error_message: - errors = '%s %s' % ( errors, error_message ) - tool_shed_repository.deleted = True - if remove_from_disk_checked: - tool_shed_repository.status = trans.install_model.ToolShedRepository.installation_status.UNINSTALLED - tool_shed_repository.error_message = None - if trans.app.config.manage_dependency_relationships: - # Remove the uninstalled repository and any tool dependencies from the in-memory dictionaries in the - # installed_repository_manager. - trans.app.installed_repository_manager.handle_repository_uninstall( tool_shed_repository ) - else: - tool_shed_repository.status = trans.install_model.ToolShedRepository.installation_status.DEACTIVATED - trans.install_model.context.current.add( tool_shed_repository ) - trans.install_model.context.current.flush() + errors = irm.uninstall_repository(repository=tool_shed_repository, remove_from_disk=remove_from_disk_checked) if remove_from_disk_checked: message += 'The repository named %s has been uninstalled. ' % escape( tool_shed_repository.name ) if errors: diff --git a/lib/tool_shed/galaxy_install/installed_repository_manager.py b/lib/tool_shed/galaxy_install/installed_repository_manager.py index c8c67e5790f..739deaac507 100644 --- a/lib/tool_shed/galaxy_install/installed_repository_manager.py +++ b/lib/tool_shed/galaxy_install/installed_repository_manager.py @@ -4,6 +4,7 @@ Class encapsulating the management of repositories installed into Galaxy from th import copy import logging import os +import shutil from sqlalchemy import and_, false, true @@ -749,6 +750,75 @@ class InstalledRepositoryManager( object ): if installed_repository_dict[ 'display_path' ]: cdl.load_installed_display_applications( installed_repository_dict, deactivate=deactivate ) + def uninstall_repository(self, repository, remove_from_disk=True): + errors = '' + shed_tool_conf, tool_path, relative_install_dir = \ + suc.get_tool_panel_config_tool_path_install_dir( app=self.app, repository=repository ) + if relative_install_dir: + if tool_path: + relative_install_dir = os.path.join( tool_path, relative_install_dir ) + repository_install_dir = os.path.abspath( relative_install_dir ) + else: + repository_install_dir = None + if repository.includes_tools_for_display_in_tool_panel: + # Handle tool panel alterations. + tpm = tool_panel_manager.ToolPanelManager(app=self.app) + tpm.remove_repository_contents(repository, + shed_tool_conf, + uninstall=remove_from_disk) + if repository.includes_data_managers: + dmh = data_manager.DataManagerHandler(app=self.app) + dmh.remove_from_data_manager(repository) + if repository.includes_datatypes: + # Deactivate proprietary datatypes. + cdl = custom_datatype_manager.CustomDatatypeLoader(app=self.app) + installed_repository_dict = cdl.load_installed_datatypes( repository, + repository_install_dir, + deactivate=True ) + if installed_repository_dict: + converter_path = installed_repository_dict.get( 'converter_path' ) + if converter_path is not None: + cdl.load_installed_datatype_converters( installed_repository_dict, deactivate=True ) + display_path = installed_repository_dict.get( 'display_path' ) + if display_path is not None: + cdl.load_installed_display_applications( installed_repository_dict, deactivate=True ) + if remove_from_disk: + try: + # Remove the repository from disk. + shutil.rmtree( repository_install_dir ) + log.debug( "Removed repository installation directory: %s" % str( repository_install_dir ) ) + removed = True + except Exception as e: + log.debug( "Error removing repository installation directory %s: %s" % ( str( repository_install_dir ), str( e ) ) ) + if isinstance( e, OSError ) and not os.path.exists( repository_install_dir ): + removed = True + log.debug( "Repository directory does not exist on disk, marking as uninstalled." ) + else: + removed = False + if removed: + repository.uninstalled = True + # Remove all installed tool dependencies and tool dependencies stuck in the INSTALLING state, but don't touch any + # repository dependencies. + tool_dependencies_to_uninstall = repository.tool_dependencies_installed_or_in_error + tool_dependencies_to_uninstall.extend( repository.tool_dependencies_being_installed ) + for tool_dependency in tool_dependencies_to_uninstall: + uninstalled, error_message = tool_dependency_util.remove_tool_dependency( self.app, tool_dependency ) + if error_message: + errors = '%s %s' % ( errors, error_message ) + repository.deleted = True + if remove_from_disk: + repository.status = self.app.install_model.ToolShedRepository.installation_status.UNINSTALLED + repository.error_message = None + if self.app.config.manage_dependency_relationships: + # Remove the uninstalled repository and any tool dependencies from the in-memory dictionaries in the + # installed_repository_manager. + self.handle_repository_uninstall( repository ) + else: + repository.status = self.app.install_model.ToolShedRepository.installation_status.DEACTIVATED + self.app.install_model.context.current.add( repository ) + self.app.install_model.context.current.flush() + return errors + def purge_repository( self, repository ): """Purge a repository with status New (a white ghost) from the database.""" sa_session = self.app.model.context.current From b29f23500d0b8ccb0ef94521cbb42aaae430c755 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Tue, 27 Jun 2017 16:08:42 +0200 Subject: [PATCH 2/5] Simplify uninstall message creation --- lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py b/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py index 3f966bc4ad6..1301c224973 100644 --- a/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py +++ b/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py @@ -252,16 +252,13 @@ class AdminToolshed( AdminGalaxy ): for tool_shed_repository in tool_shed_repositories: if kwd.get( 'deactivate_or_uninstall_repository_button', False ): errors = irm.uninstall_repository(repository=tool_shed_repository, remove_from_disk=remove_from_disk_checked) + action = 'removed' if remove_from_disk_checked else 'uninstalled' + status = max( status, statuses.index( 'done' ) ) + message += 'The repository named %s has been %s. ' % (escape( tool_shed_repository.name ), action) if remove_from_disk_checked: - message += 'The repository named %s has been uninstalled. ' % escape( tool_shed_repository.name ) if errors: message += 'Attempting to uninstall tool dependencies resulted in errors: %s' % errors status = max( status, statuses.index( 'error' ) ) - else: - status = max( status, statuses.index( 'done' ) ) - else: - message = 'The repository named %s has been deactivated. ' % escape( tool_shed_repository.name ) - status = max( status, statuses.index( 'done' ) ) status = statuses[ status ] if kwd.get( 'deactivate_or_uninstall_repository_button', False ): return trans.response.send_redirect( web.url_for( controller='admin_toolshed', From 67c66861c50f02475e497aa0c4feec7dfa16c2c2 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Tue, 27 Jun 2017 23:05:22 +0200 Subject: [PATCH 3/5] Add `uninstall_repository` to ToolShedRepositoriesController And connect routes. It is now possible to POST to api/tool_shed_repositories to trigger a Repository installation and to DELETE to api/tool_shed_repositories to uninstall a repository. In addition one can also DELETE to api/tool_shed_repositories/, which will uninstall the repository with repository_id. --- .../galaxy/api/tool_shed_repositories.py | 41 +++++++++++++++++++ lib/galaxy/webapps/galaxy/buildapp.py | 18 ++++++++ 2 files changed, 59 insertions(+) diff --git a/lib/galaxy/webapps/galaxy/api/tool_shed_repositories.py b/lib/galaxy/webapps/galaxy/api/tool_shed_repositories.py index 9bb66b3e31a..005961ba0b3 100644 --- a/lib/galaxy/webapps/galaxy/api/tool_shed_repositories.py +++ b/lib/galaxy/webapps/galaxy/api/tool_shed_repositories.py @@ -15,6 +15,7 @@ from galaxy.web import _future_expose_api as expose_api from galaxy.web.base.controller import BaseAPIController from tool_shed.galaxy_install.install_manager import InstallRepositoryManager +from tool_shed.galaxy_install.installed_repository_manager import InstalledRepositoryManager from tool_shed.galaxy_install.metadata.installed_repository_metadata_manager import InstalledRepositoryMetadataManager from tool_shed.galaxy_install.repair_repository_manager import RepairRepositoryManager from tool_shed.util import common_util @@ -616,6 +617,46 @@ class ToolShedRepositoriesController( BaseAPIController ): all_installed_tool_shed_repositories.extend( installed_tool_shed_repositories ) return all_installed_tool_shed_repositories + @expose_api + def uninstall_repository(self, trans, id=None, **kwd): + """ + DELETE /api/tool_shed_repositories/id + DELETE /api/tool_shed_repositories/ + + :param id: encoded repository id. Either id or name, owner, changeset_revision and tool_shed_url need to be supplied + :param kwd: 'remove_from_disk' : Remove repository from disk or deactivate repository. + Defaults to `True` (= remove repository from disk). + 'name' : Repository name + 'owner' : Repository owner + 'changeset_revision': Changeset revision to uninstall + 'tool_shed_url' : Tool Shed URL + """ + if id: + try: + repository = repository_util.get_tool_shed_repository_by_id(self.app, id) + except ValueError: + raise HTTPBadRequest(detail="No repository with id '%s' found" % id) + else: + tsr_arguments = ['name', 'owner', 'changeset_revision', 'tool_shed_url'] + try: + tsr_arguments = {key: kwd[key] for key in tsr_arguments} + except KeyError as e: + raise HTTPBadRequest(detail="Missing required parameter '%s'" % e.args[0]) + repository = repository_util.get_installed_repository(app=self.app, + tool_shed=tsr_arguments['tool_shed_url'], + name=tsr_arguments['name'], + owner=tsr_arguments['owner'], + changeset_revision=tsr_arguments['changeset_revision']) + if not repository: + raise HTTPBadRequest(detail="Repository not found") + irm = InstalledRepositoryManager(app=self.app) + errors = irm.uninstall_repository(repository=repository, remove_from_disk=kwd.get('remove_from_disk', True)) + if not errors: + action = 'removed' if kwd.get('remove_from_disk', True) else 'deactivated' + return {'message': 'The repository named %s has been %s.' % (repository.name, action)} + else: + raise Exception('Attempting to uninstall tool dependencies for repository named %s resulted in errors: %s' % (repository.name, errors)) + @expose_api def repair_repository_revision( self, trans, payload, **kwd ): """ diff --git a/lib/galaxy/webapps/galaxy/buildapp.py b/lib/galaxy/webapps/galaxy/buildapp.py index fef45799a64..328e30d2d79 100644 --- a/lib/galaxy/webapps/galaxy/buildapp.py +++ b/lib/galaxy/webapps/galaxy/buildapp.py @@ -826,12 +826,30 @@ def populate_api_routes( webapp, app ): action='status', conditions=dict( method=[ "GET" ] ) ) + webapp.mapper.connect( 'install_repository', + '/api/tool_shed_repositories', + controller='tool_shed_repositories', + action='install_repository_revision', + conditions=dict( method=[ 'POST' ] ) ) + webapp.mapper.connect( 'install_repository', '/api/tool_shed_repositories/install', controller='tool_shed_repositories', action='install', conditions=dict( method=[ 'POST' ] ) ) + webapp.mapper.connect( 'tool_shed_repository', + '/api/tool_shed_repositories', + controller='tool_shed_repositories', + action='uninstall_repository', + conditions=dict( method=[ "DELETE" ])) + + webapp.mapper.connect( 'tool_shed_repository', + '/api/tool_shed_repositories/{id}', + controller='tool_shed_repositories', + action='uninstall_repository', + conditions=dict( method=[ "DELETE" ])) + # Galaxy API for tool shed features. webapp.mapper.resource( 'tool_shed_repository', 'tool_shed_repositories', From 3492f0e18fe389496623d9a09b57c40fd113b7c8 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Sun, 2 Jul 2017 15:42:38 +0200 Subject: [PATCH 4/5] Fix the uninstall/deactivate message that is send to the mako template --- lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py b/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py index 1301c224973..5f412a786c2 100644 --- a/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py +++ b/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py @@ -252,7 +252,7 @@ class AdminToolshed( AdminGalaxy ): for tool_shed_repository in tool_shed_repositories: if kwd.get( 'deactivate_or_uninstall_repository_button', False ): errors = irm.uninstall_repository(repository=tool_shed_repository, remove_from_disk=remove_from_disk_checked) - action = 'removed' if remove_from_disk_checked else 'uninstalled' + action = 'uninstalled' if remove_from_disk_checked else 'deactivated' status = max( status, statuses.index( 'done' ) ) message += 'The repository named %s has been %s. ' % (escape( tool_shed_repository.name ), action) if remove_from_disk_checked: From 6a8f15bacb147e5189be983ad829a492d6dc0657 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Tue, 4 Jul 2017 14:40:50 +0200 Subject: [PATCH 5/5] Add basic unittest for InstalledRepositoryManager that exercises the most basic functionality, which is the deactivate and to uninstall repositories. Also moves shared `_init_dynamic_tool_conf` method to BaseToolBoxTestCase --- .../installed_repository_manager.py | 2 +- .../test_installed_repository_manager.py | 36 +++++++++++++++++++ .../unit/shed_unit/test_tool_panel_manager.py | 4 --- test/unit/tools/test_toolbox.py | 11 ++++-- 4 files changed, 46 insertions(+), 7 deletions(-) create mode 100644 test/unit/shed_unit/test_installed_repository_manager.py diff --git a/lib/tool_shed/galaxy_install/installed_repository_manager.py b/lib/tool_shed/galaxy_install/installed_repository_manager.py index 739deaac507..739e7716e79 100644 --- a/lib/tool_shed/galaxy_install/installed_repository_manager.py +++ b/lib/tool_shed/galaxy_install/installed_repository_manager.py @@ -79,11 +79,11 @@ class InstalledRepositoryManager( object ): def activate_repository( self, repository ): """Activate an installed tool shed repository that has been marked as deactivated.""" - repository_clone_url = common_util.generate_clone_url_for_installed_repository( self.app, repository ) shed_tool_conf, tool_path, relative_install_dir = suc.get_tool_panel_config_tool_path_install_dir( self.app, repository ) repository.deleted = False repository.status = self.install_model.ToolShedRepository.installation_status.INSTALLED if repository.includes_tools_for_display_in_tool_panel: + repository_clone_url = common_util.generate_clone_url_for_installed_repository( self.app, repository ) tpm = tool_panel_manager.ToolPanelManager( self.app ) irmm = InstalledRepositoryMetadataManager( app=self.app, tpm=tpm, diff --git a/test/unit/shed_unit/test_installed_repository_manager.py b/test/unit/shed_unit/test_installed_repository_manager.py new file mode 100644 index 00000000000..3fc33637363 --- /dev/null +++ b/test/unit/shed_unit/test_installed_repository_manager.py @@ -0,0 +1,36 @@ +from tool_shed.galaxy_install.installed_repository_manager import InstalledRepositoryManager +from tools.test_toolbox import BaseToolBoxTestCase + + +class InstalledRepositoryManagerTestCase(BaseToolBoxTestCase): + + def test_uninstall_repository(self): + repository = self._setup_repository() + assert repository.uninstalled is False + irm = InstalledRepositoryManager(self.app) + irm.uninstall_repository(repository=repository, remove_from_disk=True) + assert repository.uninstalled is True + + def test_deactivate_repository(self): + self._deactivate_repository() + + def test_activate_repository(self): + irm, repository = self._deactivate_repository() + irm.activate_repository(repository) + assert repository.status == self.app.install_model.ToolShedRepository.installation_status.INSTALLED + + def _deactivate_repository(self): + repository = self._setup_repository() + assert repository.uninstalled is False + irm = InstalledRepositoryManager(self.app) + irm.uninstall_repository(repository=repository, remove_from_disk=False) + assert repository.uninstalled is False + assert repository.status == self.app.install_model.ToolShedRepository.installation_status.DEACTIVATED + return irm, repository + + def _setup_repository(self): + self._init_dynamic_tool_conf() + self.app.config.tool_configs = self.config_files + self.app.config.manage_dependency_relationships = False + self.app.toolbox = self.toolbox + return self._repo_install(changeset='1', config_filename=self.config_files[0]) diff --git a/test/unit/shed_unit/test_tool_panel_manager.py b/test/unit/shed_unit/test_tool_panel_manager.py index b33d77d0b8f..b8f0c0d7a3b 100644 --- a/test/unit/shed_unit/test_tool_panel_manager.py +++ b/test/unit/shed_unit/test_tool_panel_manager.py @@ -188,10 +188,6 @@ class ToolPanelManagerTestCase( BaseToolBoxTestCase ): message = message_template % ( filename, open( filename, "r" ).read() ) raise AssertionError( message ) - def _init_dynamic_tool_conf( self ): - # Add a dynamic tool conf (such as a ToolShed managed one) to list of configs. - self._add_config( """""" % self.test_directory ) - def _init_ts_tool( self, guid=DEFAULT_GUID, **kwds ): tool = self._init_tool( **kwds ) tool.guid = guid diff --git a/test/unit/tools/test_toolbox.py b/test/unit/tools/test_toolbox.py index 6a64976ff5e..71d782df67f 100644 --- a/test/unit/tools/test_toolbox.py +++ b/test/unit/tools/test_toolbox.py @@ -68,8 +68,11 @@ class BaseToolBoxTestCase( unittest.TestCase, tools_support.UsesApp, tools_supp self.__toolbox = None self.config_files = [] - def _repo_install( self, changeset ): - repository = tool_shed_install.ToolShedRepository() + def _repo_install( self, changeset, config_filename=None ): + metadata = {} + if config_filename: + metadata['shed_config_filename'] = config_filename + repository = tool_shed_install.ToolShedRepository(metadata=metadata) repository.tool_shed = "github.com" repository.owner = "galaxyproject" repository.name = "example" @@ -137,6 +140,10 @@ class BaseToolBoxTestCase( unittest.TestCase, tools_support.UsesApp, tools_supp json.dump(content, f) self.config_files.append( path ) + def _init_dynamic_tool_conf( self ): + # Add a dynamic tool conf (such as a ToolShed managed one) to list of configs. + self._add_config( """""" % self.test_directory ) + def _tool_conf_path( self, name="tool_conf.xml" ): path = os.path.join( self.test_directory, name ) return path