Merge pull request #14642 from jmchilton/tool_shed_registry_refactor

Refactor tool shed registry into utility code.

lib/galaxy/app.py

@@ -91,7 +91,6 @@ from galaxy.security.vault import (
     Vault,
     VaultFactory,
 )
-from galaxy.tool_shed import tool_shed_registry
 from galaxy.tool_shed.galaxy_install.installed_repository_manager import InstalledRepositoryManager
 from galaxy.tool_shed.galaxy_install.update_repository_manager import UpdateRepositoryManager
 from galaxy.tool_util.deps import containers
@@ -120,6 +119,7 @@ from galaxy.util import (
 )
 from galaxy.util.dbkeys import GenomeBuilds
 from galaxy.util.task import IntervalTask
+from galaxy.util.tool_shed import tool_shed_registry
 from galaxy.visualization.data_providers.registry import DataProviderRegistry
 from galaxy.visualization.genomes import Genomes
 from galaxy.visualization.plugins.registry import VisualizationsRegistry

lib/galaxy/structured_app.py

@@ -30,6 +30,7 @@ from galaxy.security.vault import Vault
 from galaxy.tool_util.deps.views import DependencyResolversView
 from galaxy.tool_util.verify import test_data
 from galaxy.util.dbkeys import GenomeBuilds
+from galaxy.util.tool_shed.tool_shed_registry import Registry as ToolShedRegistry
 from galaxy.web_stack import ApplicationStack
 from galaxy.webhooks import WebhooksRegistry
 from galaxy.workflow.trs_proxy import TrsProxy
@@ -134,6 +135,7 @@ class StructuredApp(MinimalManagerApp):
     data_provider_registry: Any  # 'galaxy.visualization.data_providers.registry.DataProviderRegistry'
     tool_data_tables: "ToolDataTableManager"
     tool_cache: Any  # 'galaxy.tools.cache.ToolCache'
+    tool_shed_registry: ToolShedRegistry
     tool_shed_repository_cache: Optional[Any]  # 'galaxy.tools.cache.ToolShedRepositoryCache'
     watchers: "ConfigWatchers"
     workflow_scheduling_manager: Any  # 'galaxy.workflow.scheduling_manager.WorkflowSchedulingManager'

lib/galaxy/util/tool_shed/tool_shed_registry.py

@@ -1,15 +1,22 @@
 import logging
-from typing import NamedTuple
+from typing import (
+    Dict,
+    NamedTuple,
+    Optional,
+)
 
 from galaxy.util import parse_xml_string
-from galaxy.util.tool_shed.common_util import remove_protocol_from_tool_shed_url
+from galaxy.util.path import StrPath
+from galaxy.util.tool_shed import common_util
 from galaxy.util.tool_shed.xml_util import parse_xml
 
 log = logging.getLogger(__name__)
 
-DEFAULT_TOOL_SHEDS_CONF_XML = """<?xml version="1.0"?>
+DEFAULT_TOOL_SHED_URL = "https://toolshed.g2.bx.psu.edu/"
+DEFAULT_TOOL_SHED_NAME = "Galaxy Main Tool Shed"
+DEFAULT_TOOL_SHEDS_CONF_XML = f"""<?xml version="1.0"?>
 <tool_sheds>
-    <tool_shed name="Galaxy Main Tool Shed" url="https://toolshed.g2.bx.psu.edu/"/>
+    <tool_shed name="{DEFAULT_TOOL_SHED_NAME}" url="{DEFAULT_TOOL_SHED_URL}"/>
 </tool_sheds>
 """
 
@@ -20,7 +27,10 @@ class AUTH_TUPLE(NamedTuple):
 
 
 class Registry:
-    def __init__(self, config=None):
+    tool_sheds: Dict[str, str]
+    tool_sheds_auth: Dict[str, Optional[AUTH_TUPLE]]
+
+    def __init__(self, config: Optional[StrPath] = None):
         self.tool_sheds = {}
         self.tool_sheds_auth = {}
         if config:
@@ -49,7 +59,7 @@ class Registry:
             except Exception as e:
                 log.warning(f'Error loading reference to tool shed "{name}", problem: {str(e)}')
 
-    def url_auth(self, url):
+    def url_auth(self, url: str) -> Optional[AUTH_TUPLE]:
         """
         If the tool shed is using external auth, the client to the tool shed must authenticate to that
         as well.  This provides access to the six.moves.urllib.request.HTTPPasswordMgrWithdefaultRealm() object for the
@@ -58,9 +68,9 @@ class Registry:
         Following more what galaxy.demo_sequencer.controllers.common does might be more appropriate at
         some stage...
         """
-        url_sans_protocol = remove_protocol_from_tool_shed_url(url)
+        url_sans_protocol = common_util.remove_protocol_from_tool_shed_url(url)
         for shed_name, shed_url in self.tool_sheds.items():
-            shed_url_sans_protocol = remove_protocol_from_tool_shed_url(shed_url)
+            shed_url_sans_protocol = common_util.remove_protocol_from_tool_shed_url(shed_url)
             if url_sans_protocol.startswith(shed_url_sans_protocol):
                 return self.tool_sheds_auth[shed_name]
         log.debug(f"Invalid url '{str(url)}' received by tool shed registry's url_auth method.")

lib/galaxy/util/tool_shed/xml_util.py

@@ -12,6 +12,7 @@ from galaxy.util import (
     unicodify,
     xml_to_string,
 )
+from galaxy.util.path import StrPath
 
 log = logging.getLogger(__name__)
 
@@ -25,7 +26,7 @@ def create_and_write_tmp_file(elem):
     return tmp_filename
 
 
-def parse_xml(file_name, check_exists=True) -> Tuple[Optional[etree.ElementTree], str]:
+def parse_xml(file_name: StrPath, check_exists=True) -> Tuple[Optional[etree.ElementTree], str]:
     """Returns a parsed xml tree with comments intact."""
     error_message = ""
     if check_exists and not os.path.exists(file_name):

lib/galaxy/webapps/galaxy/services/workflows.py

@@ -15,7 +15,7 @@ from galaxy.managers.workflows import (
     WorkflowsManager,
 )
 from galaxy.schema.schema import WorkflowIndexQueryPayload
-from galaxy.tool_shed.tool_shed_registry import Registry
+from galaxy.util.tool_shed.tool_shed_registry import Registry
 from galaxy.webapps.galaxy.services.base import ServiceBase
 from galaxy.webapps.galaxy.services.sharable import ShareableService
 

mypy.ini

@@ -296,8 +296,6 @@ check_untyped_defs = False
 check_untyped_defs = False
 [mypy-galaxy.tool_util.verify.interactor]
 check_untyped_defs = False
-[mypy-galaxy.tool_shed.tool_shed_registry]
-check_untyped_defs = False
 [mypy-galaxy.objectstore.s3]
 check_untyped_defs = False
 [mypy-galaxy.objectstore.pithos]