Merge pull request #13977 from jmchilton/ts_api_tests

Implement initial tool shed API tests.

lib/galaxy/config/__init__.py

@@ -457,6 +457,11 @@ class BaseAppConfiguration(HasDynamicProperties):
             return value
 
         for key, value in kwargs.items():
+            if key in self.schema._deprecated_aliases:
+                new_key = self.schema._deprecated_aliases[key]
+                log.warning(f"Option {key} has been deprecated in favor of {new_key}")
+                key = new_key
+
             if key in self.schema.app_schema:
                 value = convert_datatype(key, value)
                 if value and self.deprecated_dirs:

lib/galaxy/config/config_manage.py

@@ -147,7 +147,8 @@ OPTION_ACTIONS = {
     "serve_xss_vulnerable_mimetypes": _ProductionUnsafe(True),
     "use_printdebug": _ProductionUnsafe(True),
     "id_secret": _ProductionUnsafe("USING THE DEFAULT IS NOT SECURE!"),
-    "master_api_key": _ProductionUnsafe("changethis"),
+    "master_api_key": _RenameAction("bootstrap_admin_api_key"),
+    "bootstrap_admin_api_key": _ProductionUnsafe("changethis"),
     "external_service_type_config_file": _DeprecatedAndDroppedAction(),
     "external_service_type_path": _DeprecatedAndDroppedAction(),
     "enable_sequencer_communication": _DeprecatedAndDroppedAction(),
@@ -358,7 +359,7 @@ def _validate(args, app_desc):
         ordered_dump(raw_config, config_p)
 
     def _clean(p, k, v):
-        return k not in ["reloadable", "path_resolves_to", "per_host"]
+        return k not in ["reloadable", "path_resolves_to", "per_host", "deprecated_alias"]
 
     clean_schema = remap(app_desc.schema.raw_schema, _clean)
     with tempfile.NamedTemporaryFile("w", suffix=".yml") as fp:

lib/galaxy/config/schema.py

@@ -52,8 +52,11 @@ class AppSchema(Schema):
         self._reloadable_options = set()  # config options we can reload at runtime
         self._paths_to_resolve = {}  # {config option: referenced config option}
         self._per_host_options = set()  # config options that can be set using a per_host config parameter
+        self._deprecated_aliases = {}
         for key, data in app_schema.items():
             self._defaults[key] = data.get("default")
+            if data.get("deprecated_alias"):
+                self._deprecated_aliases[data.get("deprecated_alias")] = key
             if data.get("reloadable"):
                 self._reloadable_options.add(key)
             if data.get("per_host"):

lib/galaxy/config/schemas/config_schema.yml

@@ -2761,9 +2761,10 @@ mapping:
           Optional list of email addresses of API users who can make calls on behalf of
           other users.
 
-      master_api_key:
+      bootstrap_admin_api_key:
         type: str
         required: false
+        deprecated_alias: master_api_key
         desc: |
           API key that allows performing some admin actions without actually
           having a real admin user in the database and config.

lib/galaxy/config/schemas/tool_shed_config_schema.yml

@@ -611,6 +611,16 @@ mapping:
           (e.g. LDAP) instead or in addition to local authentication (.sample is used
           if default does not exist).
 
+      bootstrap_admin_api_key:
+        type: str
+        required: false
+        desc: |
+          API key that allows performing some admin actions without actually
+          having a real admin user in the database and config.
+          Only set this if you need to bootstrap Galaxy, in particular to create
+          a real admin user account via API.
+          You should probably not set this on a production server.
+
       shed_tool_data_table_config:
         type: str
         default: shed_tool_data_table_conf.xml

lib/galaxy/managers/users.py

@@ -261,7 +261,7 @@ class UserManager(base.ModelManager, deletable.PurgableManagerMixin):
         """
         Find a user by API key.
         """
-        if self.check_master_api_key(api_key=api_key):
+        if self.check_bootstrap_admin_api_key(api_key=api_key):
             return schema.BootstrapAdminUser()
         sa_session = sa_session or self.app.model.session
         try:
@@ -276,14 +276,14 @@ class UserManager(base.ModelManager, deletable.PurgableManagerMixin):
             raise exceptions.AuthenticationFailed("Provided API key has expired.")
         return provided_key.user
 
-    def check_master_api_key(self, api_key):
-        master_api_key = getattr(self.app.config, "master_api_key", None)
-        if not master_api_key:
+    def check_bootstrap_admin_api_key(self, api_key):
+        bootstrap_admin_api_key = getattr(self.app.config, "bootstrap_admin_api_key", None)
+        if not bootstrap_admin_api_key:
             return False
         # Hash keys to make them the same size, so we can do safe comparison.
-        master_hash = hashlib.sha256(util.smart_str(master_api_key)).hexdigest()
+        bootstrap_hash = hashlib.sha256(util.smart_str(bootstrap_admin_api_key)).hexdigest()
         provided_hash = hashlib.sha256(util.smart_str(api_key)).hexdigest()
-        return util.safe_str_cmp(master_hash, provided_hash)
+        return util.safe_str_cmp(bootstrap_hash, provided_hash)
 
     # ---- admin
     def is_admin(self, user: Optional[model.User], trans=None) -> bool:

lib/galaxy/tool_util/verify/interactor.py

@@ -122,7 +122,7 @@ class GalaxyInteractorApi:
         )
         if kwds.get("user_api_key_is_admin_key", False):
             self.master_api_key = self.api_key
-        self.keep_outputs_dir = kwds["keep_outputs_dir"]
+        self.keep_outputs_dir = kwds.get("keep_outputs_dir", None)
         self.download_attempts = kwds.get("download_attempts", 1)
         self.download_sleep = kwds.get("download_sleep", 1)
         # Local test data directories.
@@ -142,18 +142,15 @@ class GalaxyInteractorApi:
     def supports_test_data_download(self):
         return self.target_galaxy_version >= Version("19.01")
 
-    def __get_user_key(self, user_key, admin_key, test_user=None):
+    def __get_user_key(self, user_key: Optional[str], admin_key: Optional[str], test_user: Optional[str] = None) -> str:
         if not test_user:
             test_user = "test@bx.psu.edu"
         if user_key:
             return user_key
-        test_user = self.ensure_user_with_email(test_user)
-        return self._post(f"users/{test_user['id']}/api_key", key=admin_key).json()
-
-    # def get_tools(self):
-    #    response = self._get("tools?in_panel=false")
-    #    assert response.status_code == 200, "Non 200 response from tool index API. [%s]" % response.content
-    #    return response.json()
+        test_user_response = self.ensure_user_with_email(test_user)
+        if not admin_key:
+            raise Exception("Must specify either a user key or admin key to interact with the Galaxy API")
+        return self._post(f"users/{test_user_response['id']}/api_key", key=admin_key).json()
 
     def get_tests_summary(self):
         response = self._get("tools/tests_summary")

lib/galaxy/util/resources.py

@@ -1,22 +1,35 @@
 """Provide a consistent interface into and utilities for importlib file resources.
 """
 try:
+    from importlib.abc import Traversable  # type: ignore[attr-defined]
     from importlib.resources import files  # type: ignore[attr-defined]
 except ImportError:
     # Python < 3.9
     from importlib_resources import files  # type: ignore[no-redef]
+    from importlib_resources.abc import Traversable  # type: ignore[no-redef]
 
 
-def resource_string(package_or_requirement, resource_name):
+def resource_path(package_or_requirement, resource_name):
     """
     Return specified resource as a string.
 
     Replacement function for pkg_resources.resource_string, but returns unicode string instead of bytestring.
     """
-    return files(package_or_requirement).joinpath(resource_name).read_text()
+    return files(package_or_requirement).joinpath(resource_name)
+
+
+def resource_string(package_or_requirement, resource_name) -> str:
+    """
+    Return specified resource as a string.
+
+    Replacement function for pkg_resources.resource_string, but returns unicode string instead of bytestring.
+    """
+    return resource_path(package_or_requirement, resource_name).read_text()
 
 
 __all__ = (
     "files",
     "resource_string",
+    "resource_path",
+    "Traversable",
 )

lib/galaxy_test/api/test_authenticate.py

@@ -1,8 +1,8 @@
-import base64
 from urllib.parse import urljoin
 
 from requests import get
 
+from galaxy_test.base.api_util import baseauth_headers
 from ._framework import ApiTestCase
 
 TEST_USER_EMAIL = "auth_user_test@bx.psu.edu"
@@ -13,11 +13,7 @@ class AuthenticationApiTestCase(ApiTestCase):
     def test_auth(self):
         self._setup_user(TEST_USER_EMAIL, TEST_USER_PASSWORD)
         baseauth_url = self._api_url("authenticate/baseauth", use_key=False)
-        unencoded_credentials = f"{TEST_USER_EMAIL}:{TEST_USER_PASSWORD}"
-        authorization = base64.b64encode(unencoded_credentials.encode("utf-8"))
-        headers = {
-            "Authorization": authorization,
-        }
+        headers = baseauth_headers(TEST_USER_EMAIL, TEST_USER_PASSWORD)
         auth_response = get(baseauth_url, headers=headers)
         self._assert_status_code_is(auth_response, 200)
         auth_dict = auth_response.json()

lib/galaxy_test/api/test_roles.py

@@ -67,23 +67,23 @@ class RolesApiTestCase(ApiTestCase):
         assert "description" in response.json()["err_msg"]
 
         # Test missing name
-        payload = {
+        payload_missing_name = {
             "name": None,
             "description": description,
             "user_ids": [self.dataset_populator.user_id()],
         }
-        response = self._post("roles", payload, admin=True, json=True)
+        response = self._post("roles", payload_missing_name, admin=True, json=True)
         assert_status_code_is(response, 400)
         assert_error_code_is(response, error_codes.error_codes_by_name["USER_REQUEST_MISSING_PARAMETER"].code)
         assert "name" in response.json()["err_msg"]
 
         # Test invalid type for name
-        payload = {
+        payload_invalid_type = {
             "name": ["a test", "name"],
             "description": description,
             "user_ids": [self.dataset_populator.user_id()],
         }
-        response = self._post("roles", payload, admin=True, json=True)
+        response = self._post("roles", payload_invalid_type, admin=True, json=True)
         assert_status_code_is(response, 400)
         assert_error_code_is(response, error_codes.error_codes_by_name["USER_REQUEST_INVALID_PARAMETER"].code)
         assert "name" in response.json()["err_msg"]

lib/galaxy_test/base/api.py

@@ -1,5 +1,6 @@
 import os
 from contextlib import contextmanager
+from typing import Optional
 from urllib.parse import (
     urlencode,
     urljoin,
@@ -81,6 +82,7 @@ class UsesCeleryTasks:
 
 class UsesApiTestCaseMixin:
     url: str
+    _galaxy_interactor: Optional["ApiTestInteractor"] = None
 
     def tearDown(self):
         if os.environ.get("GALAXY_TEST_EXTERNAL") is None:
@@ -104,9 +106,14 @@ class UsesApiTestCaseMixin:
     def _setup_interactor(self):
         self.user_api_key = get_user_api_key()
         self.master_api_key = get_admin_api_key()
-        self.galaxy_interactor = self._get_interactor()
+        self._galaxy_interactor = self._get_interactor()
 
-    def _get_interactor(self, api_key=None):
+    @property
+    def galaxy_interactor(self) -> "ApiTestInteractor":
+        assert self._galaxy_interactor is not None
+        return self._galaxy_interactor
+
+    def _get_interactor(self, api_key=None) -> "ApiTestInteractor":
         return ApiTestInteractor(self, api_key=api_key)
 
     def _setup_user(self, email, password=None, is_admin=True):

lib/galaxy_test/base/api_util.py

@@ -1,5 +1,11 @@
+import base64
 import os
-from typing import Optional
+import random
+import string
+from typing import (
+    Dict,
+    Optional,
+)
 
 DEFAULT_GALAXY_MASTER_API_KEY = "TEST123"
 DEFAULT_GALAXY_USER_API_KEY = None
@@ -33,3 +39,20 @@ def get_user_api_key() -> Optional[str]:
     be used to create a new user and API key for tests.
     """
     return os.environ.get("GALAXY_TEST_USER_API_KEY", DEFAULT_GALAXY_USER_API_KEY)
+
+
+def baseauth_headers(username: str, password: str) -> Dict[str, str]:
+    unencoded_credentials = f"{username}:{password}"
+    authorization = base64.b64encode(unencoded_credentials.encode("utf-8")).decode("utf-8")
+    headers = {
+        "Authorization": authorization,
+    }
+    return headers
+
+
+def random_name(prefix: Optional[str] = None, suffix: Optional[str] = None, len: int = 10) -> str:
+    return "{}{}{}".format(
+        prefix or "",
+        "".join(random.choice(string.ascii_lowercase + string.digits) for _ in range(len)),
+        suffix or "",
+    )

lib/galaxy_test/base/populators.py

@@ -41,8 +41,6 @@ import base64
 import contextlib
 import json
 import os
-import random
-import string
 import tarfile
 import tempfile
 import time
@@ -100,6 +98,7 @@ from galaxy.util import (
 from galaxy.util.resources import resource_string
 from . import api_asserts
 from .api import ApiTestInteractor
+from .api_util import random_name
 
 FILE_URL = "https://raw.githubusercontent.com/galaxyproject/galaxy/dev/test-data/4.bed"
 FILE_MD5 = "37b59762b59fff860460522d271bc111"
@@ -1293,13 +1292,8 @@ class BaseDatasetPopulator(BasePopulator):
 
         return imported_history_id
 
-    def get_random_name(self, prefix=None, suffix=None, len=10):
-        # stolen from navigates_galaxy.py
-        return "{}{}{}".format(
-            prefix or "",
-            "".join(random.choice(string.ascii_lowercase + string.digits) for _ in range(len)),
-            suffix or "",
-        )
+    def get_random_name(self, prefix: Optional[str] = None, suffix: Optional[str] = None, len: int = 10) -> str:
+        return random_name(prefix=prefix, suffix=suffix, len=len)
 
     def wait_for_dataset(
         self, history_id: str, dataset_id: str, assert_ok: bool = False, timeout: timeout_type = DEFAULT_TIMEOUT

lib/galaxy_test/base/uses_shed_api.py

@@ -6,7 +6,9 @@ from galaxy_test.base.api_asserts import assert_status_code_is
 
 
 class UsesShedApi:
-    galaxy_interactor: GalaxyInteractorApi
+    @property
+    def galaxy_interactor(self) -> GalaxyInteractorApi:
+        ...
 
     def delete_repo_request(self, payload):
         return self.galaxy_interactor._delete("tool_shed_repositories", data=payload, admin=True)

lib/tool_shed/test/base/api.py

@@ -0,0 +1,117 @@
+import os
+import re
+from typing import (
+    Any,
+    Dict,
+    Optional,
+)
+from urllib.parse import urljoin
+
+import requests
+
+from galaxy.tool_util.verify.interactor import GalaxyInteractorApi
+from galaxy_test.base import api_asserts
+from galaxy_test.base.api_util import (
+    baseauth_headers,
+    get_admin_api_key as get_galaxy_admin_api_key,
+    get_user_api_key as get_galaxy_user_key,
+    TEST_USER,
+)
+from galaxy_test.base.uses_shed_api import UsesShedApi
+from galaxy_test.driver.testcase import DrivenFunctionalTestCase
+from .api_util import (
+    get_admin_api_key,
+    get_user_api_key,
+    ShedApiInteractor,
+)
+from .populators import ToolShedPopulator
+
+
+class ShedBaseTestCase(DrivenFunctionalTestCase):
+    def setUp(self):
+        host = os.environ.get("TOOL_SHED_TEST_HOST")
+        assert host
+        self.host = host
+        self.port = os.environ.get("TOOL_SHED_TEST_PORT")
+        self.url = f"http://{self.host}:{self.port}"
+        self.galaxy_host = os.environ.get("GALAXY_TEST_HOST")
+        self.galaxy_port = os.environ.get("GALAXY_TEST_PORT")
+        self.galaxy_url = f"http://{self.galaxy_host}:{self.galaxy_port}"
+
+
+def ensure_user_with_email(admin_api_interactor: ShedApiInteractor, email: str, password: Optional[str]):
+    all_users_response = admin_api_interactor.get("users")
+    try:
+        all_users_response.raise_for_status()
+    except requests.exceptions.HTTPError as e:
+        raise Exception(
+            f"Failed to verify user with email [{email}] exists - perhaps you're targetting the wrong Galaxy server or using an incorrect admin API key. HTTP error: {e}"
+        )
+    username = email_to_username(email)
+    all_users = all_users_response.json()
+    try:
+        test_user = [user for user in all_users if user["username"] == username][0]
+    except IndexError:
+        password = password or "testpass"
+        data = dict(
+            remote_user_email=email,
+            email=email,
+            password=password,
+            username=username,
+        )
+        test_user = admin_api_interactor.post("users", json=data).json()
+    return test_user
+
+
+def email_to_username(email: str) -> str:
+    """Pattern used for test user generation - does not use the API."""
+    return re.sub(r"[^a-z-\d]", "--", email.lower())
+
+
+class ShedGalaxyInteractorApi(GalaxyInteractorApi):
+    def __init__(self, galaxy_url: str):
+        interactor_kwds: Dict[str, Any] = {}
+        interactor_kwds["galaxy_url"] = galaxy_url
+        interactor_kwds["master_api_key"] = get_galaxy_admin_api_key()
+        interactor_kwds["api_key"] = get_galaxy_user_key()
+        super().__init__(**interactor_kwds)
+
+
+class ShedApiTestCase(ShedBaseTestCase, UsesShedApi):
+    _populator: Optional[ToolShedPopulator] = None
+    _galaxy_interactor: Optional[GalaxyInteractorApi] = None
+
+    @property
+    def populator(self) -> ToolShedPopulator:
+        if self._populator is None:
+            self._populator = ToolShedPopulator(self.admin_api_interactor, self.api_interactor)
+        return self._populator
+
+    @property
+    def galaxy_interactor(self) -> GalaxyInteractorApi:
+        if self._galaxy_interactor is None:
+            self._galaxy_interactor = ShedGalaxyInteractorApi(self.galaxy_url)
+        return self._galaxy_interactor
+
+    @property
+    def admin_api_interactor(self) -> ShedApiInteractor:
+        return ShedApiInteractor(self.url, get_admin_api_key())
+
+    @property
+    def api_interactor(self) -> ShedApiInteractor:
+        user_api_key = get_user_api_key()
+        if user_api_key is None:
+            email = TEST_USER
+            password = "testpassword"
+            ensure_user_with_email(self.admin_api_interactor, email, password)
+            user_api_key = self._api_key(email, password)
+        return ShedApiInteractor(self.url, user_api_key)
+
+    def _api_key(self, email: str, password: str) -> str:
+        headers = baseauth_headers(email, password)
+        url = urljoin(self.url, "api/authenticate/baseauth")
+        auth_response = requests.get(url, headers=headers)
+        api_asserts.assert_status_code_is(auth_response, 200)
+        auth_dict = auth_response.json()
+        api_asserts.assert_has_keys(auth_dict, "api_key")
+        return auth_dict["api_key"]

lib/tool_shed/test/base/api_util.py

@@ -0,0 +1,67 @@
+import os
+from functools import wraps
+from typing import (
+    Callable,
+    Optional,
+)
+from urllib.parse import urljoin
+
+import requests
+
+DEFAULT_TOOL_SHED_BOOTSTRAP_ADMIN_API_KEY = "TEST1234"
+DEFAULT_TOOL_SHED_USER_API_KEY = None
+
+
+def get_admin_api_key() -> str:
+    """Test admin API key to use for functional tests.
+
+    This key should be configured as a admin API key and should be able
+    to create additional users and keys.
+    """
+    for key in ["TOOL_SHED_CONFIG_BOOTSTRAP_ADMIN_API_KEY", "TOOL_SHED_CONFIG_OVERRIDE_BOOTSTRAP_ADMIN_API_KEY"]:
+        value = os.environ.get(key, None)
+        if value:
+            return value
+    return DEFAULT_TOOL_SHED_BOOTSTRAP_ADMIN_API_KEY
+
+
+def get_user_api_key() -> Optional[str]:
+    """Test user API key to use for functional tests.
+
+    If set, this should drive API based testing - if not set an admin API key will
+    be used to create a new user and API key for tests.
+    """
+    return os.environ.get("TOOL_SHED_TEST_USER_API_KEY", DEFAULT_TOOL_SHED_USER_API_KEY)
+
+
+def decorate_method(method: Callable):
+    @wraps(method)
+    def wrapper(self: "ShedApiInteractor", route: str, **kwd) -> requests.Response:
+        url = urljoin(self.url, f"api/{route}")
+        kwd = self._append_headers(kwd)
+        return method(url, **kwd)
+
+    return wrapper
+
+
+class ShedApiInteractor:
+    url: str
+    api_key: str
+
+    def __init__(self, url: str, api_key: str):
+        self.url = url
+        self.api_key = api_key
+
+    def _append_headers(self, kwd):
+        if "admin" in kwd:
+            key = get_admin_api_key()
+        else:
+            key = self.api_key
+
+        headers = kwd.get("headers", {})
+        headers["x-api-key"] = key
+        kwd["headers"] = headers
+        return kwd
+
+    get = decorate_method(requests.get)
+    post = decorate_method(requests.post)

lib/tool_shed/test/base/populators.py

@@ -0,0 +1,116 @@
+from typing import Union
+
+import requests
+
+from galaxy.util.resources import (
+    resource_path,
+    Traversable,
+)
+from galaxy_test.base import api_asserts
+from galaxy_test.base.api_util import random_name
+from tool_shed_client.schema import (
+    Category,
+    OrderedInstallableRevisions,
+    Repository,
+    RepositoryUpdate,
+)
+from .api_util import ShedApiInteractor
+
+HasRepositoryId = Union[str, Repository]
+
+DEFAULT_PREFIX = "repofortest"
+COLUMN_MAKER_PATH = resource_path(__package__, "../test_data/column_maker/column_maker.tar")
+COLUMN_MAKER_1_1_1_PATH = resource_path(__package__, "../test_data/column_maker/column_maker.tar")
+DEFAULT_COMMIT_MESSAGE = "a test commit message"
+
+
+class ToolShedPopulator:
+    """Utilities for easy fixture creation of tool shed related things."""
+
+    _admin_api_interactor: ShedApiInteractor
+    _api_interactor: ShedApiInteractor
+
+    def __init__(self, admin_api_interactor: ShedApiInteractor, api_interactor: ShedApiInteractor):
+        self._admin_api_interactor = admin_api_interactor
+        self._api_interactor = api_interactor
+
+    def setup_column_maker_repo(self, prefix=DEFAULT_PREFIX) -> Repository:
+        category_id = self.new_category(prefix=prefix).id
+        repository = self.new_repository(category_id, prefix=prefix)
+        repository_id = repository.id
+        assert repository_id
+
+        response = self.upload_revision(
+            repository_id,
+            COLUMN_MAKER_PATH,
+        )
+        assert response.is_ok
+        return repository
+
+    def update_column_maker_repo(self, repository: HasRepositoryId) -> requests.Response:
+        response = self.upload_revision(
+            repository,
+            COLUMN_MAKER_1_1_1_PATH,
+        )
+        return response
+
+    def upload_revision_raw(
+        self, repository: HasRepositoryId, path: Traversable, commit_message: str = DEFAULT_COMMIT_MESSAGE
+    ) -> requests.Response:
+        body = {
+            "commit_message": commit_message,
+        }
+        files = {"file": path.open("rb")}
+        repository_id = self._repository_id(repository)
+        response = self._api_interactor.post(f"repositories/{repository_id}/changeset_revision", json=body, files=files)
+        return response
+
+    def upload_revision(
+        self, repository: HasRepositoryId, path: Traversable, commit_message: str = DEFAULT_COMMIT_MESSAGE
+    ):
+        response = self.upload_revision_raw(repository, path, commit_message)
+        api_asserts.assert_status_code_is_ok(response)
+        return RepositoryUpdate(__root__=response.json())
+
+    def new_repository(self, category_id, prefix=DEFAULT_PREFIX) -> Repository:
+        name = random_name(prefix=prefix)
+        synopsis = random_name(prefix=prefix)
+        description = None
+        remote_repository_url = None
+        homepage_url = None
+        category_ids = category_id
+        type = "unrestricted"
+        body = {
+            "name": name,
+            "synopsis": synopsis,
+            "description": description,
+            "remote_repository_url": remote_repository_url,
+            "homepage_url": homepage_url,
+            "category_ids[]": category_ids,
+            "type": type,
+        }
+        response = self._api_interactor.post("repositories", json=body)
+        api_asserts.assert_status_code_is_ok(response)
+        return Repository(**response.json())
+
+    def new_category(self, prefix=DEFAULT_PREFIX) -> Category:
+        name = random_name(prefix=prefix)
+        body = {"name": name, "description": "testcreaterepo"}
+        response = self._admin_api_interactor.post("categories", json=body)
+        response.raise_for_status()
+        return Category(**response.json())
+
+    def get_ordered_installable_revisions(self, owner: str, name: str) -> OrderedInstallableRevisions:
+        params = {
+            "owner": owner,
+            "name": name,
+        }
+        revisions_response = self._api_interactor.get("repositories/get_ordered_installable_revisions", params=params)
+        api_asserts.assert_status_code_is_ok(revisions_response)
+        return OrderedInstallableRevisions(__root__=revisions_response.json())
+
+    def _repository_id(self, has_id: HasRepositoryId) -> str:
+        if isinstance(has_id, Repository):
+            return has_id.id
+        else:
+            return str(has_id)

lib/tool_shed/test/base/twilltestcase.py

@@ -29,7 +29,6 @@ from galaxy.util import (
     unicodify,
 )
 from galaxy_test.base.api_util import get_admin_api_key
-from galaxy_test.driver.testcase import DrivenFunctionalTestCase
 from tool_shed.util import (
     hg_util,
     hgweb_config,
@@ -39,6 +38,7 @@ from . import (
     common,
     test_db_util,
 )
+from .api import ShedBaseTestCase
 
 # Set a 10 minute timeout for repository installation.
 repository_installation_timeout = 600
@@ -47,10 +47,11 @@ log = logging.getLogger(__name__)
 tc.options["equiv_refresh_interval"] = 0
 
 
-class ShedTwillTestCase(DrivenFunctionalTestCase):
+class ShedTwillTestCase(ShedBaseTestCase):
     """Class of FunctionalTestCase geared toward HTML interactions using the Twill library."""
 
     def setUp(self):
+        super().setUp()
         # Security helper
         self.security = idencoding.IdEncodingHelper(id_secret="changethisinproductiontoo")
         self.history_id = None
@@ -58,12 +59,6 @@ class ShedTwillTestCase(DrivenFunctionalTestCase):
         self.hgweb_config_manager = hgweb_config.hgweb_config_manager
         self.hgweb_config_manager.hgweb_config_dir = self.hgweb_config_dir
         self.tool_shed_test_tmp_dir = os.environ.get("TOOL_SHED_TEST_TMP_DIR", None)
-        self.host = os.environ.get("TOOL_SHED_TEST_HOST")
-        self.port = os.environ.get("TOOL_SHED_TEST_PORT")
-        self.url = f"http://{self.host}:{self.port}"
-        self.galaxy_host = os.environ.get("GALAXY_TEST_HOST")
-        self.galaxy_port = os.environ.get("GALAXY_TEST_PORT")
-        self.galaxy_url = f"http://{self.galaxy_host}:{self.galaxy_port}"
         self.shed_tool_data_table_conf = os.environ.get("TOOL_SHED_TEST_TOOL_DATA_TABLE_CONF")
         self.file_dir = os.environ.get("TOOL_SHED_TEST_FILE_DIR", None)
         self.tool_data_path = os.environ.get("GALAXY_TEST_TOOL_DATA_PATH")

lib/tool_shed/test/functional/test_galaxy_install.py

@@ -0,0 +1,17 @@
+from ..base.api import ShedApiTestCase
+
+
+class ShedGalaxyInstallApiTestCase(ShedApiTestCase):
+    def test_install_simple_tool(self):
+        populator = self.populator
+        repository = populator.setup_column_maker_repo(prefix="repoformetadata")
+        owner = repository.owner
+        name = repository.name
+        installable_revisions = populator.get_ordered_installable_revisions(owner, name)
+        latest_install_revision = installable_revisions.__root__[-1]
+        self.install_repository(owner, name, latest_install_revision, tool_shed_url=self.url)
+        response = self.galaxy_interactor._get("tools?in_panel=False")
+        response.raise_for_status()
+        expected_tool = f"{self.host}:{self.port}/repos/{owner}/{name}/Add_a_column1/1.1.0"
+        tool_ids = [t["id"] for t in response.json()]
+        assert expected_tool in tool_ids, f"Didn't find {expected_tool} in {tool_ids}"

lib/tool_shed/test/functional/test_shed_categories.py

@@ -0,0 +1,16 @@
+from galaxy_test.base.api_util import random_name
+from ..base.api import ShedApiTestCase
+
+
+class ShedCategoriesApiTestCase(ShedApiTestCase):
+    def test_create_requires_name(self):
+        body = {}
+        response = self.admin_api_interactor.post("categories", json=body)
+        assert response.status_code == 400
+
+    def test_create_okay(self):
+        name = random_name(prefix="createokay")
+        body = {"name": name, "description": "testcreateokaydescript"}
+        response = self.admin_api_interactor.post("categories", json=body)
+        assert response.status_code == 200
+        assert response.json()["name"] == name

lib/tool_shed/test/functional/test_shed_repositories.py

@@ -0,0 +1,78 @@
+from galaxy.util.resources import resource_path
+from galaxy_test.base import api_asserts
+from ..base.api import ShedApiTestCase
+
+COLUMN_MAKER_PATH = resource_path(__package__, "../test_data/column_maker/column_maker.tar")
+
+
+# Things seemingly not used by Galaxy, Planemo, or Ephemeris...
+#   (perhaps we can delete instead of test?)...
+# - reset_metadata_on_repository
+# - repository_ids_for_setting_metadata
+# - reset_metadata_on_repositories
+# - remove_repository_registry_entry
+# - get_repository_revision_install_info
+# - get_installable_revisions
+
+# Non repositories API seemingly unused and seemingly better rewritten if wanted.
+# - The whole Groups API.
+# - The whole Repository Revisions API.
+class ShedRepositoriesApiTestCase(ShedApiTestCase):
+    def test_create(self):
+        populator = self.populator
+        category_id = populator.new_category(prefix="testcreate").id
+
+        response = self.api_interactor.get(f"categories/{category_id}/repositories")
+        api_asserts.assert_status_code_is_ok(response)
+        repos = response.json()["repositories"]
+        assert len(repos) == 0
+
+        populator.new_repository(category_id)
+        response = self.api_interactor.get(f"categories/{category_id}/repositories")
+        api_asserts.assert_status_code_is_ok(response)
+        repos = response.json()["repositories"]
+        assert len(repos) == 1
+
+    def test_update_repository(self):
+        populator = self.populator
+        prefix = "testupdate"
+        category_id = populator.new_category(prefix=prefix).id
+        repository = populator.new_repository(category_id, prefix=prefix)
+        repository_id = repository.id
+        repository_update = populator.upload_revision(
+            repository_id,
+            COLUMN_MAKER_PATH,
+        )
+        assert repository_update.is_ok
+
+    # used by getRepository in TS client.
+    def test_metadata_simple(self):
+        populator = self.populator
+        repository_id = populator.setup_column_maker_repo(prefix="repoformetadata").id
+        metadata_response = self.api_interactor.get(f"repositories/{repository_id}/metadata")
+        api_asserts.assert_status_code_is_ok(metadata_response)
+        metadata_for_revisions = metadata_response.json()
+        assert len(metadata_for_revisions) == 1
+        only_key = list(metadata_for_revisions.keys())[0]
+        assert only_key.startswith("0:")
+        only_revision = list(metadata_for_revisions.values())[0]
+        api_asserts.assert_has_keys(only_revision, "repository", "repository_dependencies", "numeric_revision")
+
+    def test_index_simple(self):
+        populator = self.populator
+        repository_id = populator.setup_column_maker_repo(prefix="repoforindex").id
+        show_response = self.api_interactor.get(f"repositories/{repository_id}")
+        index_response = self.api_interactor.get("repositories")
+        api_asserts.assert_status_code_is_ok(show_response)
+        api_asserts.assert_status_code_is_ok(index_response)
+        repository_ids = [r["id"] for r in index_response.json()]
+        assert repository_id in repository_ids
+
+    def test_get_ordered_installable_revisions(self):
+        # Used in ephemeris...
+        populator = self.populator
+        repository = populator.setup_column_maker_repo(prefix="repoforindex")
+        assert repository.owner
+        assert repository.name
+        revisions = populator.get_ordered_installable_revisions(repository.owner, repository.name)
+        assert len(revisions.__root__) == 1

lib/tool_shed/test/functional/test_shed_users.py

@@ -0,0 +1,72 @@
+from urllib.parse import urljoin
+
+from requests import post
+
+from galaxy_test.base import api_asserts
+from ..base.api import (
+    email_to_username,
+    ensure_user_with_email,
+    ShedApiTestCase,
+)
+from ..base.api_util import get_admin_api_key
+
+
+class ShedUsersApiTestCase(ShedApiTestCase):
+    def test_create_requires_admin(self):
+        url = urljoin(self.url, "api/users")
+        response = post(url)
+        assert response.status_code == 403
+
+    def test_create_user(self):
+        url = urljoin(self.url, "api/users")
+        headers = {
+            "x-api-key": get_admin_api_key(),
+        }
+        email = "testcreateuser@bx.psu.edu"
+        password = "mycoolpassword123"
+        body = {
+            "email": email,
+            "password": password,
+            "username": email_to_username(email),
+        }
+        response = post(url, json=body, headers=headers)
+        api_asserts.assert_status_code_is_ok(response)
+        self._verify_username_password(email, password)
+
+    def test_create_user_interactor(self):
+        email = "testcreateuserinteractor@bx.psu.edu"
+        password = "mycoolpassword123"
+        body = {
+            "email": email,
+            "password": password,
+            "username": email_to_username(email),
+        }
+        response = self.admin_api_interactor.post("users", json=body)
+        api_asserts.assert_status_code_is_ok(response)
+        self._verify_username_password(email, password)
+
+    def test_ensure_user_with_email(self):
+        email = "testcreateuserensure@bx.psu.edu"
+        password = "mycoolpassword123"
+        ensure_user_with_email(self.admin_api_interactor, email, password)
+        self._verify_username_password(email, password)
+
+    def test_simple_index_and_user(self):
+        email = "testindexshow@bx.psu.edu"
+        password = "mycoolpassword123"
+        ensure_user_with_email(self.admin_api_interactor, email, password)
+        user_response = self.admin_api_interactor.get("users")
+        api_asserts.assert_status_code_is_ok(user_response)
+        users = user_response.json()
+        assert isinstance(users, list)
+        username = email_to_username(email)
+        filtered_users = [u for u in users if u["username"] == username]
+        assert len(filtered_users) == 1
+        user_id = filtered_users[0]["id"]
+        show_response = self.admin_api_interactor.get(f"users/{user_id}")
+        api_asserts.assert_status_code_is_ok(show_response)
+        assert show_response.json()["username"] == username
+        assert show_response.json()["id"] == user_id
+
+    def _verify_username_password(self, email, password):
+        self._api_key(email, password)

lib/tool_shed/test/functional_tests.py

@@ -16,6 +16,7 @@ sys.path[0:1] = [os.path.join(galaxy_root, "lib")]
 # This is for the tool shed application.
 from galaxy.webapps.galaxy.buildapp import app_factory as galaxy_app_factory
 from galaxy_test.driver import driver_util
+from tool_shed.test.base.api_util import get_admin_api_key
 from tool_shed.webapp import buildapp as toolshedbuildapp
 
 log = driver_util.build_logger()
@@ -88,6 +89,7 @@ class ToolShedTestDriver(driver_util.TestDriver):
         toolshed_database_conf = driver_util.database_conf(shed_db_path, prefix="TOOL_SHED")
         kwargs = dict(
             admin_users="test@bx.psu.edu",
+            bootstrap_admin_api_key=get_admin_api_key(),
             allow_user_creation=True,
             allow_user_deletion=True,
             datatype_converters_config_file="datatype_converters_conf.xml.sample",

lib/tool_shed/webapp/api/__init__.py

@@ -0,0 +1,7 @@
+from galaxy.structured_app import BasicSharedApp
+from galaxy.webapps.base.controller import BaseAPIController
+
+
+class BaseShedAPIController(BaseAPIController):
+    def __init__(self, app: BasicSharedApp):
+        super().__init__(app)

lib/tool_shed/webapp/api/authenticate.py

@@ -17,14 +17,18 @@ Returns
 """
 import logging
 
-from galaxy.web import expose_api_raw_anonymous_and_sessionless
-from galaxy.webapps.galaxy.api.authenticate import AuthenticationController
+from galaxy.web import expose_api_anonymous_and_sessionless
+from galaxy.webapps.galaxy.api import depends
+from galaxy.webapps.galaxy.services.authenticate import AuthenticationService
+from . import BaseShedAPIController
 
 log = logging.getLogger(__name__)
 
 
-class ToolShedAuthenticationController(AuthenticationController):
-    @expose_api_raw_anonymous_and_sessionless
+class ToolShedAuthenticationController(BaseShedAPIController):
+    authentication_service = depends(AuthenticationService)
+
+    @expose_api_anonymous_and_sessionless
     def get_tool_shed_api_key(self, trans, **kwd):
         """
         GET /api/authenticate/baseauth
@@ -36,4 +40,4 @@ class ToolShedAuthenticationController(AuthenticationController):
 
         :raises: ObjectNotFound, HTTPBadRequest
         """
-        return self.get_api_key(trans, **kwd)
+        return self.authentication_service.get_api_key(trans.environ)

lib/tool_shed/webapp/api/categories.py

@@ -20,9 +20,6 @@ log = logging.getLogger(__name__)
 class CategoriesController(BaseAPIController):
     """RESTful controller for interactions with categories in the Tool Shed."""
 
-    def __get_repository_count(self, trans, category_name):
-        return self.app.repository_registry.viewable_repositories_and_suites_by_category.get(category_name, 0)
-
     def __get_value_mapper(self, trans):
         value_mapper = {"id": trans.security.encode_id}
         return value_mapper

lib/tool_shed/webapp/api/users.py

@@ -2,6 +2,7 @@ import logging
 
 import tool_shed.util.shed_util_common as suc
 from galaxy import (
+    exceptions,
     util,
     web,
 )
@@ -18,7 +19,7 @@ log = logging.getLogger(__name__)
 class UsersController(BaseAPIController):
     """RESTful controller for interactions with users in the Tool Shed."""
 
-    @web.legacy_expose_api
+    @web.expose_api
     @web.require_admin
     def create(self, trans, payload, **kwd):
         """
@@ -32,22 +33,19 @@ class UsersController(BaseAPIController):
                 :param password (required): the password of the user
                 :param username (required): the public username of the user
         """
-        user_dict = dict(message="", status="ok")
         # Get the information about the user to be created from the payload.
         email = payload.get("email", "")
         password = payload.get("password", "")
         username = payload.get("username", "")
         message = self.__validate(trans, email=email, password=password, confirm=password, username=username)
         if message:
-            message = f"email: {email}, username: {username} - {message}"
-            user_dict["message"] = message
-            user_dict["status"] = "error"
-        else:
-            # Create the user.
-            user = self.__create_user(trans, email, username, password)
-            user_dict = user.to_dict(view="element", value_mapper=self.__get_value_mapper(trans))
-            user_dict["message"] = f"User '{str(user.username)}' has been created."
-            user_dict["url"] = web.url_for(controller="users", action="show", id=trans.security.encode_id(user.id))
+            raise exceptions.RequestParameterInvalidException(message)
+
+        # Create the user.
+        user = self.__create_user(trans, email, username, password)
+        user_dict = user.to_dict(view="element", value_mapper=self.__get_value_mapper(trans))
+        user_dict["message"] = f"User '{str(user.username)}' has been created."
+        user_dict["url"] = web.url_for(controller="users", action="show", id=trans.security.encode_id(user.id))
         return user_dict
 
     def __create_user(self, trans, email, username, password):
@@ -67,7 +65,7 @@ class UsersController(BaseAPIController):
         value_mapper = {"id": trans.security.encode_id}
         return value_mapper
 
-    @web.legacy_expose_api_anonymous
+    @web.expose_api_anonymous
     def index(self, trans, deleted=False, **kwd):
         """
         GET /api/users
@@ -86,7 +84,7 @@ class UsersController(BaseAPIController):
             user_dicts.append(user_dict)
         return user_dicts
 
-    @web.legacy_expose_api_anonymous
+    @web.expose_api_anonymous
     def show(self, trans, id, **kwd):
         """
         GET /api/users/{encoded_user_id}

lib/tool_shed/webapp/buildapp.py

@@ -22,7 +22,7 @@ log = logging.getLogger(__name__)
 
 
 class CommunityWebApplication(galaxy.webapps.base.webapp.WebApplication):
-    pass
+    injection_aware: bool = True
 
 
 def add_ui_controllers(webapp, app):

lib/tool_shed_client/schema/__init__.py

@@ -0,0 +1,37 @@
+from typing import (
+    List,
+    Union,
+)
+
+from pydantic import BaseModel
+
+
+class Repository(BaseModel):
+    id: str
+    name: str
+    owner: str
+
+
+class Category(BaseModel):
+    id: str
+    name: str
+
+
+class ValidRepostiroyUpdateMessage(BaseModel):
+    message: str
+
+
+class FailedRepositoryUpdateMessage(BaseModel):
+    err_msg: str
+
+
+class OrderedInstallableRevisions(BaseModel):
+    __root__: List[str]
+
+
+class RepositoryUpdate(BaseModel):
+    __root__: Union[ValidRepostiroyUpdateMessage, FailedRepositoryUpdateMessage]
+
+    @property
+    def is_ok(self):
+        return isinstance(self.__root__, ValidRepostiroyUpdateMessage)

packages/app/tool_shed_client

@@ -0,0 +1 @@
+../../lib/tool_shed_client/

test/integration/test_config_defaults.py

@@ -197,7 +197,7 @@ DO_NOT_TEST = [
     "markdown_export_css",  # default not used?
     "markdown_export_css_pages",  # default not used?
     "markdown_export_css_invocation_reports",  # default not used?
-    "master_api_key",  # broken: default value assigned outside of config/
+    "bootstrap_admin_api_key",  # broken: default value assigned outside of config/
     "migrated_tools_config",  # needs more work (should work)
     "monitor_thread_join_timeout",  # broken: default overridden
     "new_file_path",  # value overridden for testing

test/unit/config/config_manage/embedded/config/galaxy.yml

@@ -1,5 +1,5 @@
 galaxy:
-  master_api_key: changethis
+  bootstrap_admin_api_key: changethis
   config_dir: 'false'
   helpsite_url: 'https://help.galaxyproject.org/'
   job_config:

test/unit/config/config_manage/simple/config/galaxy.yml

@@ -1,4 +1,4 @@
 galaxy:
-  master_api_key: changethis
+  bootstrap_admin_api_key: changethis
   config_dir: 'false'
   helpsite_url: 'https://help.galaxyproject.org/'