From f3b51702e7b810369cbecf9e8c008f9eca41ee8d Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Sun, 31 May 2020 13:26:56 +0200 Subject: [PATCH 1/9] Fix proprietary datatypes disappearing on restart --- lib/galaxy/app.py | 9 +++++---- lib/galaxy/config/__init__.py | 18 ++++++++++-------- .../installed_repository_manager.py | 2 +- 3 files changed, 16 insertions(+), 13 deletions(-) diff --git a/lib/galaxy/app.py b/lib/galaxy/app.py index ea24d1709b9..25d1473f06a 100644 --- a/lib/galaxy/app.py +++ b/lib/galaxy/app.py @@ -91,10 +91,6 @@ class UniverseApplication(config.ConfiguresGalaxyMixin): check_migrate_tools = self.config.check_migrate_tools self._configure_models(check_migrate_databases=self.config.check_migrate_databases, check_migrate_tools=check_migrate_tools, config_file=config_file) - self.installed_repository_manager = InstalledRepositoryManager(self) - self._configure_datatypes_registry(self.installed_repository_manager) - galaxy.model.set_datatypes_registry(self.datatypes_registry) - # Security helper self._configure_security() # Tag handler @@ -134,6 +130,11 @@ class UniverseApplication(config.ConfiguresGalaxyMixin): self.tool_shed_repository_cache = ToolShedRepositoryCache(self) # Watch various config files for immediate reload self.watchers = ConfigWatchers(self) + self._configure_tool_config_files() + self.installed_repository_manager = InstalledRepositoryManager(self) + self._configure_datatypes_registry(self.installed_repository_manager) + galaxy.model.set_datatypes_registry(self.datatypes_registry) + self._configure_toolbox() # Load Data Manager diff --git a/lib/galaxy/config/__init__.py b/lib/galaxy/config/__init__.py index 8b890aef0cf..1d982b0187b 100644 --- a/lib/galaxy/config/__init__.py +++ b/lib/galaxy/config/__init__.py @@ -999,6 +999,16 @@ class ConfiguresGalaxyMixin(object): else: log.warning('Waiting for toolbox reload timed out after 60 seconds') + def _configure_tool_config_files(self): + if self.config.shed_tool_config_file not in self.config.tool_configs: + self.config.tool_configs.append(self.config.shed_tool_config_file) + # The value of migrated_tools_config is the file reserved for containing only those tools that have been + # eliminated from the distribution and moved to the tool shed. If migration checking is disabled, only add it if + # it exists (since this may be an existing deployment where migrations were previously run). + if ((self.config.check_migrate_tools or os.path.exists(self.config.migrated_tools_config)) + and self.config.migrated_tools_config not in self.config.tool_configs): + self.config.tool_configs.append(self.config.migrated_tools_config) + def _configure_toolbox(self): from galaxy import tools from galaxy.managers.citations import CitationsManager @@ -1011,14 +1021,6 @@ class ConfiguresGalaxyMixin(object): from galaxy.managers.tools import DynamicToolManager self.dynamic_tools_manager = DynamicToolManager(self) self._toolbox_lock = threading.RLock() - if self.config.shed_tool_config_file not in self.config.tool_configs: - self.config.tool_configs.append(self.config.shed_tool_config_file) - # The value of migrated_tools_config is the file reserved for containing only those tools that have been - # eliminated from the distribution and moved to the tool shed. If migration checking is disabled, only add it if - # it exists (since this may be an existing deployment where migrations were previously run). - if ((self.config.check_migrate_tools or os.path.exists(self.config.migrated_tools_config)) - and self.config.migrated_tools_config not in self.config.tool_configs): - self.config.tool_configs.append(self.config.migrated_tools_config) self.toolbox = tools.ToolBox(self.config.tool_configs, self.config.tool_path, self) galaxy_root_dir = os.path.abspath(self.config.root) file_path = os.path.abspath(getattr(self.config, "file_path")) diff --git a/lib/galaxy/tool_shed/galaxy_install/installed_repository_manager.py b/lib/galaxy/tool_shed/galaxy_install/installed_repository_manager.py index 3ccd62febb7..e5d813cd7b5 100644 --- a/lib/galaxy/tool_shed/galaxy_install/installed_repository_manager.py +++ b/lib/galaxy/tool_shed/galaxy_install/installed_repository_manager.py @@ -577,7 +577,7 @@ class InstalledRepositoryManager(object): def get_repository_install_dir(self, tool_shed_repository): for tree in self.tool_trees: if tree is None: - return None + continue root = tree.getroot() tool_path = root.get('tool_path', None) if tool_path: From c2669f3454da996bb3b5d07e5cbc24aff5320a53 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Sun, 31 May 2020 13:57:03 +0200 Subject: [PATCH 2/9] Simplify get_repository_install_dir --- .../installed_repository_manager.py | 32 +++++++++---------- 1 file changed, 15 insertions(+), 17 deletions(-) diff --git a/lib/galaxy/tool_shed/galaxy_install/installed_repository_manager.py b/lib/galaxy/tool_shed/galaxy_install/installed_repository_manager.py index e5d813cd7b5..1b065d9f76b 100644 --- a/lib/galaxy/tool_shed/galaxy_install/installed_repository_manager.py +++ b/lib/galaxy/tool_shed/galaxy_install/installed_repository_manager.py @@ -41,12 +41,15 @@ class InstalledRepositoryManager(object): self.context = self.install_model.context self.tool_configs = self.app.config.tool_configs - self.tool_trees = [] + self.tool_paths = [] for tool_config in self.tool_configs: tree, error_message = parse_xml(tool_config) if error_message: log.error(error_message) - self.tool_trees.append(tree) + else: + tool_path = tree.getroot().get('tool_path') + if tool_path: + self.tool_paths.append(tool_path) self.installed_repository_dicts = [] # Keep an in-memory dictionary whose keys are tuples defining tool_shed_repository objects (whose status is 'Installed') @@ -575,21 +578,16 @@ class InstalledRepositoryManager(object): str(repository.installed_changeset_revision)) def get_repository_install_dir(self, tool_shed_repository): - for tree in self.tool_trees: - if tree is None: - continue - root = tree.getroot() - tool_path = root.get('tool_path', None) - if tool_path: - ts = common_util.remove_port_from_tool_shed_url(str(tool_shed_repository.tool_shed)) - relative_path = os.path.join(tool_path, - ts, - 'repos', - str(tool_shed_repository.owner), - str(tool_shed_repository.name), - str(tool_shed_repository.installed_changeset_revision)) - if os.path.exists(relative_path): - return relative_path + for tool_path in self.tool_paths: + ts = common_util.remove_port_from_tool_shed_url(str(tool_shed_repository.tool_shed)) + relative_path = os.path.join(tool_path, + ts, + 'repos', + str(tool_shed_repository.owner), + str(tool_shed_repository.name), + str(tool_shed_repository.installed_changeset_revision)) + if os.path.exists(relative_path): + return relative_path return None def get_runtime_dependent_tool_dependency_tuples(self, tool_dependency, status=None): From b04efe7533add7c3198ce1059471a66650cebf29 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Sun, 31 May 2020 14:08:13 +0200 Subject: [PATCH 3/9] Make sure tool paths are updated when shed_tool_config file is created --- .../installed_repository_manager.py | 28 +++++++++++++------ 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/lib/galaxy/tool_shed/galaxy_install/installed_repository_manager.py b/lib/galaxy/tool_shed/galaxy_install/installed_repository_manager.py index 1b065d9f76b..0c0cf967319 100644 --- a/lib/galaxy/tool_shed/galaxy_install/installed_repository_manager.py +++ b/lib/galaxy/tool_shed/galaxy_install/installed_repository_manager.py @@ -41,15 +41,7 @@ class InstalledRepositoryManager(object): self.context = self.install_model.context self.tool_configs = self.app.config.tool_configs - self.tool_paths = [] - for tool_config in self.tool_configs: - tree, error_message = parse_xml(tool_config) - if error_message: - log.error(error_message) - else: - tool_path = tree.getroot().get('tool_path') - if tool_path: - self.tool_paths.append(tool_path) + self._tool_paths = [] self.installed_repository_dicts = [] # Keep an in-memory dictionary whose keys are tuples defining tool_shed_repository objects (whose status is 'Installed') @@ -83,6 +75,24 @@ class InstalledRepositoryManager(object): # at runtime. The value defines the entire tool dependency tree. self.installed_runtime_dependent_tool_dependencies_of_installed_tool_dependencies = {} + @property + def tool_paths(self): + """Return all possible tool_path attributes of all tool config files.""" + if len(self._tool_paths) != len(self.tool_configs): + # This could be happen at startup or after the creation of a new shed_tool_conf.xml file + # before the installation of the first repository + tool_paths = [] + for tool_config in self.tool_configs: + tree, error_message = parse_xml(tool_config) + if error_message: + log.error(error_message) + else: + tool_path = tree.getroot().get('tool_path') + if tool_path: + tool_paths.append(tool_path) + self._tool_paths = tool_paths + return self._tool_paths + def activate_repository(self, repository): """Activate an installed tool shed repository that has been marked as deactivated.""" shed_tool_conf, tool_path, relative_install_dir = suc.get_tool_panel_config_tool_path_install_dir(self.app, repository) From 481079d02d8311d07c72f70b714ff6c32bfe6c0b Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Sun, 31 May 2020 16:19:52 +0200 Subject: [PATCH 4/9] Add test for datatype installation and restart --- test/integration/test_shed_tool_tests.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/test/integration/test_shed_tool_tests.py b/test/integration/test_shed_tool_tests.py index f6d629d6527..44ecb4d7a0f 100644 --- a/test/integration/test_shed_tool_tests.py +++ b/test/integration/test_shed_tool_tests.py @@ -17,3 +17,22 @@ class ToolShedToolTestIntegrationTestCase(integration_util.IntegrationTestCase, def test_tool_test(self): self.install_repository("devteam", "fastqc", "ff9530579d1f") self._run_tool_test("toolshed.g2.bx.psu.edu/repos/devteam/fastqc/fastqc/0.71") + + +class ToolShedDatatypeTestIntegrationTestCase(integration_util.IntegrationTestCase, UsesShed): + + """Test datatype installation""" + + framework_tool_and_types = True + + @skip_if_toolshed_down + def test_datatype_installation(self): + datatypes = self._get("datatypes").json() + assert "cond" not in datatypes + self.install_repository("sblanck", "smagexp_datatypes", "f174dc3d2641") + datatypes = self._get("datatypes").json() + assert "cond" in datatypes + # Make sure datatype survives restart + self.restart() + datatypes = self._get("datatypes").json() + assert "cond" in datatypes From 0e81239dfcafc97c1cbdc3dde40b25050864f30f Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Sun, 31 May 2020 16:25:34 +0200 Subject: [PATCH 5/9] Use timeout when determining if url is reachable --- lib/galaxy_test/base/populators.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy_test/base/populators.py b/lib/galaxy_test/base/populators.py index a04810d59ae..2c26ab98348 100644 --- a/lib/galaxy_test/base/populators.py +++ b/lib/galaxy_test/base/populators.py @@ -111,7 +111,7 @@ def skip_if_site_down(url): def site_down(): try: - response = requests.get(url) + response = requests.get(url, timeout=10) return response.status_code != 200 except Exception: return False From 228aac88238c5fb61c8dd755cfd96906c7bc1382 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Sun, 31 May 2020 19:12:31 +0200 Subject: [PATCH 6/9] Need to call configure_shed for persistent shed_tool_conf --- test/integration/test_shed_tool_tests.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/test/integration/test_shed_tool_tests.py b/test/integration/test_shed_tool_tests.py index 44ecb4d7a0f..f56fb1ed32a 100644 --- a/test/integration/test_shed_tool_tests.py +++ b/test/integration/test_shed_tool_tests.py @@ -1,3 +1,5 @@ +import os + from galaxy_test.base.populators import skip_if_toolshed_down from galaxy_test.driver import integration_util from .uses_shed import UsesShed @@ -25,6 +27,13 @@ class ToolShedDatatypeTestIntegrationTestCase(integration_util.IntegrationTestCa framework_tool_and_types = True + @classmethod + def handle_galaxy_config_kwds(cls, config): + cls.configure_shed(config) + + def handle_reconfigure_galaxy_config_kwds(self, config): + config["tool_shed_config_file"] = os.path.join(self.shed_tools_dir, "shed_tool_conf.xml") + @skip_if_toolshed_down def test_datatype_installation(self): datatypes = self._get("datatypes").json() @@ -33,6 +42,6 @@ class ToolShedDatatypeTestIntegrationTestCase(integration_util.IntegrationTestCa datatypes = self._get("datatypes").json() assert "cond" in datatypes # Make sure datatype survives restart - self.restart() + self.restart(handle_reconfig=self.handle_reconfigure_galaxy_config_kwds) datatypes = self._get("datatypes").json() assert "cond" in datatypes From 71d0c85ddd9541f193f82b4e1094116289e1107c Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Thu, 7 May 2020 11:24:57 +0200 Subject: [PATCH 7/9] Allow anonymous downloading of public workflows This allows downloading public workflows without API key. We already allow this in the workflow controller, and permissions are checked in the `get_stored_accessible_workflow` method in the WorkflowsManager class. --- lib/galaxy/webapps/galaxy/api/workflows.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/galaxy/webapps/galaxy/api/workflows.py b/lib/galaxy/webapps/galaxy/api/workflows.py index e45d536e9da..214a5fe1bb9 100644 --- a/lib/galaxy/webapps/galaxy/api/workflows.py +++ b/lib/galaxy/webapps/galaxy/api/workflows.py @@ -33,6 +33,7 @@ from galaxy.web import ( expose_api, expose_api_anonymous_and_sessionless, expose_api_raw, + expose_api_raw_anonymous_and_sessionless, format_return_as_json, ) from galaxy.webapps.base.controller import ( @@ -441,7 +442,7 @@ class WorkflowsAPIController(BaseAPIController, UsesStoredWorkflowMixin, UsesAnn invocation_response.update(rval) return invocation_response - @expose_api_raw + @expose_api_raw_anonymous_and_sessionless def workflow_dict(self, trans, workflow_id, **kwd): """ GET /api/workflows/{encoded_workflow_id}/download From d8a94035b6d2560d7e36e34e31e73ecc402c7dd7 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Mon, 1 Jun 2020 16:58:29 +0200 Subject: [PATCH 8/9] Add tests for downloading published workflows as anononymous user and making sure private workflows can't be downloaded by other or anonymous userrs. --- lib/galaxy_test/api/test_workflows.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/lib/galaxy_test/api/test_workflows.py b/lib/galaxy_test/api/test_workflows.py index a40ab737f8a..0f883587ac1 100644 --- a/lib/galaxy_test/api/test_workflows.py +++ b/lib/galaxy_test/api/test_workflows.py @@ -5,6 +5,7 @@ import time from json import dumps from uuid import uuid4 +import pytest from requests import delete, get, put from galaxy.exceptions import error_codes @@ -281,6 +282,22 @@ class WorkflowsApiTestCase(BaseWorkflowsApiTestCase): workflows_url = self._api_url("workflows/%s" % workflow_id) assert get(workflows_url).status_code == 403 + def test_cannot_download_private_workflow(self): + workflow_id = self.workflow_populator.simple_workflow("test_not_importportable") + with self._different_user(): + with pytest.raises(AssertionError) as excinfo: + self._download_workflow(workflow_id) + assert '403' in str(excinfo.value) + workflows_url = self._api_url("workflows/%s/download" % workflow_id) + assert get(workflows_url).status_code == 403 + + def test_anon_can_download_public_workflow(self): + workflow_id = self.workflow_populator.simple_workflow("test_not_importportable", publish=True) + workflows_url = self._api_url("workflows/%s/download" % workflow_id) + response = get(workflows_url) + response.raise_for_status() + assert response.json()['a_galaxy_workflow'] == 'true' + def test_delete(self): workflow_id = self.workflow_populator.simple_workflow("test_delete") workflow_name = "test_delete" From a100d86f43a2a90ab39c80e8bdf9b6e46537b404 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Mon, 1 Jun 2020 20:44:24 +0200 Subject: [PATCH 9/9] Use less typo-ish workflow names --- lib/galaxy_test/api/test_workflows.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/galaxy_test/api/test_workflows.py b/lib/galaxy_test/api/test_workflows.py index 0f883587ac1..1d306d5a285 100644 --- a/lib/galaxy_test/api/test_workflows.py +++ b/lib/galaxy_test/api/test_workflows.py @@ -273,7 +273,7 @@ class WorkflowsApiTestCase(BaseWorkflowsApiTestCase): self._assert_status_code_is(show_response, 400) def test_cannot_show_private_workflow(self): - workflow_id = self.workflow_populator.simple_workflow("test_not_importportable") + workflow_id = self.workflow_populator.simple_workflow("test_not_importable") with self._different_user(): show_response = self._get("workflows/%s" % workflow_id) self._assert_status_code_is(show_response, 403) @@ -283,7 +283,7 @@ class WorkflowsApiTestCase(BaseWorkflowsApiTestCase): assert get(workflows_url).status_code == 403 def test_cannot_download_private_workflow(self): - workflow_id = self.workflow_populator.simple_workflow("test_not_importportable") + workflow_id = self.workflow_populator.simple_workflow("test_not_downloadable") with self._different_user(): with pytest.raises(AssertionError) as excinfo: self._download_workflow(workflow_id) @@ -292,7 +292,7 @@ class WorkflowsApiTestCase(BaseWorkflowsApiTestCase): assert get(workflows_url).status_code == 403 def test_anon_can_download_public_workflow(self): - workflow_id = self.workflow_populator.simple_workflow("test_not_importportable", publish=True) + workflow_id = self.workflow_populator.simple_workflow("test_downloadable", publish=True) workflows_url = self._api_url("workflows/%s/download" % workflow_id) response = get(workflows_url) response.raise_for_status()