Merge branch 'release_20.01' into release_20.05

lib/galaxy/app.py

@@ -96,10 +96,6 @@ class UniverseApplication(config.ConfiguresGalaxyMixin):
         check_migrate_tools = self.config.check_migrate_tools
         self._configure_models(check_migrate_databases=self.config.check_migrate_databases, check_migrate_tools=check_migrate_tools, config_file=config_file)
 
-        self.installed_repository_manager = InstalledRepositoryManager(self)
-        self._configure_datatypes_registry(self.installed_repository_manager)
-        galaxy.model.set_datatypes_registry(self.datatypes_registry)
-
         # Security helper
         self._configure_security()
         # Tag handler
@@ -139,6 +135,11 @@ class UniverseApplication(config.ConfiguresGalaxyMixin):
         self.tool_shed_repository_cache = ToolShedRepositoryCache(self)
         # Watch various config files for immediate reload
         self.watchers = ConfigWatchers(self)
+        self._configure_tool_config_files()
+        self.installed_repository_manager = InstalledRepositoryManager(self)
+        self._configure_datatypes_registry(self.installed_repository_manager)
+        galaxy.model.set_datatypes_registry(self.datatypes_registry)
+
         self._configure_toolbox()
 
         # Load Data Manager

lib/galaxy/config/__init__.py

@@ -983,6 +983,16 @@ class ConfiguresGalaxyMixin(object):
         else:
             log.warning('Waiting for toolbox reload timed out after 60 seconds')
 
+    def _configure_tool_config_files(self):
+        if self.config.shed_tool_config_file not in self.config.tool_configs:
+            self.config.tool_configs.append(self.config.shed_tool_config_file)
+        # The value of migrated_tools_config is the file reserved for containing only those tools that have been
+        # eliminated from the distribution and moved to the tool shed. If migration checking is disabled, only add it if
+        # it exists (since this may be an existing deployment where migrations were previously run).
+        if ((self.config.check_migrate_tools or os.path.exists(self.config.migrated_tools_config))
+                and self.config.migrated_tools_config not in self.config.tool_configs):
+            self.config.tool_configs.append(self.config.migrated_tools_config)
+
     def _configure_toolbox(self):
         from galaxy import tools
         from galaxy.managers.citations import CitationsManager
@@ -995,14 +1005,6 @@ class ConfiguresGalaxyMixin(object):
         from galaxy.managers.tools import DynamicToolManager
         self.dynamic_tools_manager = DynamicToolManager(self)
         self._toolbox_lock = threading.RLock()
-        if self.config.shed_tool_config_file not in self.config.tool_configs:
-            self.config.tool_configs.append(self.config.shed_tool_config_file)
-        # The value of migrated_tools_config is the file reserved for containing only those tools that have been
-        # eliminated from the distribution and moved to the tool shed. If migration checking is disabled, only add it if
-        # it exists (since this may be an existing deployment where migrations were previously run).
-        if ((self.config.check_migrate_tools or os.path.exists(self.config.migrated_tools_config))
-                and self.config.migrated_tools_config not in self.config.tool_configs):
-            self.config.tool_configs.append(self.config.migrated_tools_config)
         self.toolbox = tools.ToolBox(self.config.tool_configs, self.config.tool_path, self)
         galaxy_root_dir = os.path.abspath(self.config.root)
         file_path = os.path.abspath(getattr(self.config, "file_path"))

lib/galaxy/tool_shed/galaxy_install/installed_repository_manager.py

@@ -41,12 +41,7 @@ class InstalledRepositoryManager(object):
         self.context = self.install_model.context
         self.tool_configs = self.app.config.tool_configs
 
-        self.tool_trees = []
-        for tool_config in self.tool_configs:
-            tree, error_message = parse_xml(tool_config)
-            if error_message:
-                log.error(error_message)
-            self.tool_trees.append(tree)
+        self._tool_paths = []
 
         self.installed_repository_dicts = []
         # Keep an in-memory dictionary whose keys are tuples defining tool_shed_repository objects (whose status is 'Installed')
@@ -80,6 +75,24 @@ class InstalledRepositoryManager(object):
         # at runtime.  The value defines the entire tool dependency tree.
         self.installed_runtime_dependent_tool_dependencies_of_installed_tool_dependencies = {}
 
+    @property
+    def tool_paths(self):
+        """Return all possible tool_path attributes of all tool config files."""
+        if len(self._tool_paths) != len(self.tool_configs):
+            # This could be happen at startup or after the creation of a new shed_tool_conf.xml file
+            # before the installation of the first repository
+            tool_paths = []
+            for tool_config in self.tool_configs:
+                tree, error_message = parse_xml(tool_config)
+                if error_message:
+                    log.error(error_message)
+                else:
+                    tool_path = tree.getroot().get('tool_path')
+                    if tool_path:
+                        tool_paths.append(tool_path)
+            self._tool_paths = tool_paths
+        return self._tool_paths
+
     def activate_repository(self, repository):
         """Activate an installed tool shed repository that has been marked as deactivated."""
         shed_tool_conf, tool_path, relative_install_dir = suc.get_tool_panel_config_tool_path_install_dir(self.app, repository)
@@ -575,21 +588,16 @@ class InstalledRepositoryManager(object):
                 str(repository.installed_changeset_revision))
 
     def get_repository_install_dir(self, tool_shed_repository):
-        for tree in self.tool_trees:
-            if tree is None:
-                return None
-            root = tree.getroot()
-            tool_path = root.get('tool_path', None)
-            if tool_path:
-                ts = common_util.remove_port_from_tool_shed_url(str(tool_shed_repository.tool_shed))
-                relative_path = os.path.join(tool_path,
-                                             ts,
-                                             'repos',
-                                             str(tool_shed_repository.owner),
-                                             str(tool_shed_repository.name),
-                                             str(tool_shed_repository.installed_changeset_revision))
-                if os.path.exists(relative_path):
-                    return relative_path
+        for tool_path in self.tool_paths:
+            ts = common_util.remove_port_from_tool_shed_url(str(tool_shed_repository.tool_shed))
+            relative_path = os.path.join(tool_path,
+                                         ts,
+                                         'repos',
+                                         str(tool_shed_repository.owner),
+                                         str(tool_shed_repository.name),
+                                         str(tool_shed_repository.installed_changeset_revision))
+            if os.path.exists(relative_path):
+                return relative_path
         return None
 
     def get_runtime_dependent_tool_dependency_tuples(self, tool_dependency, status=None):

lib/galaxy/webapps/galaxy/api/workflows.py

@@ -34,6 +34,7 @@ from galaxy.web import (
     expose_api,
     expose_api_anonymous_and_sessionless,
     expose_api_raw,
+    expose_api_raw_anonymous_and_sessionless,
     format_return_as_json,
 )
 from galaxy.webapps.base.controller import (
@@ -443,7 +444,7 @@ class WorkflowsAPIController(BaseAPIController, UsesStoredWorkflowMixin, UsesAnn
         invocation_response.update(rval)
         return invocation_response
 
-    @expose_api_raw
+    @expose_api_raw_anonymous_and_sessionless
     def workflow_dict(self, trans, workflow_id, **kwd):
         """
         GET /api/workflows/{encoded_workflow_id}/download

lib/galaxy_test/api/test_workflows.py

@@ -5,6 +5,7 @@ import time
 from json import dumps
 from uuid import uuid4
 
+import pytest
 from requests import delete, get, put
 
 from galaxy.exceptions import error_codes
@@ -231,7 +232,7 @@ class WorkflowsApiTestCase(BaseWorkflowsApiTestCase):
         self._assert_status_code_is(show_response, 400)
 
     def test_cannot_show_private_workflow(self):
-        workflow_id = self.workflow_populator.simple_workflow("test_not_importportable")
+        workflow_id = self.workflow_populator.simple_workflow("test_not_importable")
         with self._different_user():
             show_response = self._get("workflows/%s" % workflow_id)
             self._assert_status_code_is(show_response, 403)
@@ -240,6 +241,22 @@ class WorkflowsApiTestCase(BaseWorkflowsApiTestCase):
             workflows_url = self._api_url("workflows/%s" % workflow_id)
             assert get(workflows_url).status_code == 403
 
+    def test_cannot_download_private_workflow(self):
+        workflow_id = self.workflow_populator.simple_workflow("test_not_downloadable")
+        with self._different_user():
+            with pytest.raises(AssertionError) as excinfo:
+                self._download_workflow(workflow_id)
+            assert '403' in str(excinfo.value)
+        workflows_url = self._api_url("workflows/%s/download" % workflow_id)
+        assert get(workflows_url).status_code == 403
+
+    def test_anon_can_download_public_workflow(self):
+        workflow_id = self.workflow_populator.simple_workflow("test_downloadable", publish=True)
+        workflows_url = self._api_url("workflows/%s/download" % workflow_id)
+        response = get(workflows_url)
+        response.raise_for_status()
+        assert response.json()['a_galaxy_workflow'] == 'true'
+
     def test_delete(self):
         workflow_id = self.workflow_populator.simple_workflow("test_delete")
         workflow_name = "test_delete"

lib/galaxy_test/base/populators.py

@@ -111,7 +111,7 @@ def skip_if_site_down(url):
 
     def site_down():
         try:
-            response = requests.get(url)
+            response = requests.get(url, timeout=10)
             return response.status_code != 200
         except Exception:
             return False

test/integration/test_shed_tool_tests.py

@@ -1,3 +1,5 @@
+import os
+
 from galaxy_test.base.populators import skip_if_toolshed_down
 from galaxy_test.driver import integration_util
 from .uses_shed import UsesShed
@@ -17,3 +19,29 @@ class ToolShedToolTestIntegrationTestCase(integration_util.IntegrationTestCase,
     def test_tool_test(self):
         self.install_repository("devteam", "fastqc", "ff9530579d1f")
         self._run_tool_test("toolshed.g2.bx.psu.edu/repos/devteam/fastqc/fastqc/0.71")
+
+
+class ToolShedDatatypeTestIntegrationTestCase(integration_util.IntegrationTestCase, UsesShed):
+
+    """Test datatype installation"""
+
+    framework_tool_and_types = True
+
+    @classmethod
+    def handle_galaxy_config_kwds(cls, config):
+        cls.configure_shed(config)
+
+    def handle_reconfigure_galaxy_config_kwds(self, config):
+        config["tool_shed_config_file"] = os.path.join(self.shed_tools_dir, "shed_tool_conf.xml")
+
+    @skip_if_toolshed_down
+    def test_datatype_installation(self):
+        datatypes = self._get("datatypes").json()
+        assert "cond" not in datatypes
+        self.install_repository("sblanck", "smagexp_datatypes", "f174dc3d2641")
+        datatypes = self._get("datatypes").json()
+        assert "cond" in datatypes
+        # Make sure datatype survives restart
+        self.restart(handle_reconfig=self.handle_reconfigure_galaxy_config_kwds)
+        datatypes = self._get("datatypes").json()
+        assert "cond" in datatypes