Tool Shed API Modernization ahead of FastAPI

- Use the controller base that allow DI.
- Move a bunch functionality we want to port to FastAPI into manage layer.
- Some typing enhancements.

lib/tool_shed/managers/categories.py

@@ -0,0 +1,66 @@
+from typing import (
+    Any,
+    Callable,
+    Dict,
+    List,
+)
+
+import tool_shed.util.shed_util_common as suc
+from galaxy import (
+    exceptions,
+    web,
+)
+from galaxy.model.base import transaction
+from tool_shed.context import ProvidesUserContext
+from tool_shed.structured_app import ToolShedApp
+from tool_shed.webapp.model import Category
+from tool_shed_client.schema import CreateCategoryRequest
+
+
+class CategoryManager:
+    def __init__(self, app: ToolShedApp):
+        self.app = app
+
+    def create(self, trans: ProvidesUserContext, category_request: CreateCategoryRequest) -> Category:
+        name = category_request.name
+        description = category_request.description or name
+        if name:
+            if suc.get_category_by_name(self.app, name):
+                raise exceptions.Conflict("A category with that name already exists.")
+            else:
+                # Create the category
+                category = self.app.model.Category(name=name, description=description)
+                trans.sa_session.add(category)
+                with transaction(trans.sa_session):
+                    trans.sa_session.commit()
+            return category
+        else:
+            raise exceptions.RequestParameterMissingException('Missing required parameter "name".')
+
+    def index_db(self, trans: ProvidesUserContext, deleted: bool) -> List[Category]:
+        category_db_objects: List[Category] = []
+        if deleted and not trans.user_is_admin:
+            raise exceptions.AdminRequiredException("Only administrators can query deleted categories.")
+        for category in (
+            trans.sa_session.query(Category).filter(Category.table.c.deleted == deleted).order_by(Category.table.c.name)
+        ):
+            category_db_objects.append(category)
+        return category_db_objects
+
+    def index(self, trans: ProvidesUserContext, deleted: bool) -> List[Dict[str, Any]]:
+        category_dicts: List[Dict[str, Any]] = []
+        for category in self.index_db(trans, deleted):
+            category_dict = category.to_dict(view="collection", value_mapper=get_value_mapper(trans.app))
+            category_dict["url"] = web.url_for(
+                controller="categories", action="show", id=trans.security.encode_id(category.id)
+            )
+            category_dict[
+                "repositories"
+            ] = self.app.repository_registry.viewable_repositories_and_suites_by_category.get(category.name, 0)
+            category_dicts.append(category_dict)
+        return category_dicts
+
+
+def get_value_mapper(app: ToolShedApp) -> Dict[str, Callable]:
+    value_mapper = {"id": app.security.encode_id}
+    return value_mapper

lib/tool_shed/managers/repositories.py

@@ -1,26 +1,530 @@
 """
 Manager and Serializer for TS repositories.
 """
+import json
 import logging
+from collections import namedtuple
+from time import strftime
+from typing import (
+    Any,
+    Callable,
+    cast,
+    Dict,
+    List,
+    Optional,
+    Union,
+)
 
+from pydantic import BaseModel
+from sqlalchemy import (
+    and_,
+    false,
+)
+
+from galaxy import web
 from galaxy.exceptions import (
+    ConfigDoesNotAllowException,
     InternalServerError,
     MalformedContents,
+    ObjectNotFound,
+    RequestParameterInvalidException,
 )
 from galaxy.tool_shed.util import dependency_display
+from galaxy.util import listify
+from tool_shed.context import ProvidesUserContext
 from tool_shed.metadata import repository_metadata_manager
 from tool_shed.repository_types import util as rt_util
 from tool_shed.structured_app import ToolShedApp
-from tool_shed.util import hg_util
+from tool_shed.util import (
+    encoding_util,
+    hg_util,
+)
+from tool_shed.util.metadata_util import (
+    get_all_dependencies,
+    get_current_repository_metadata_for_changeset_revision,
+    get_metadata_revisions,
+    get_next_downloadable_changeset_revision,
+    get_repository_metadata_by_changeset_revision,
+)
+from tool_shed.util.readme_util import build_readme_files_dict
 from tool_shed.util.repository_content_util import upload_tar
+from tool_shed.util.repository_util import (
+    create_repository as low_level_create_repository,
+    get_repo_info_dict,
+    get_repositories_by_category,
+    get_repository_by_name_and_owner,
+    get_repository_in_tool_shed,
+    validate_repository_name,
+)
+from tool_shed.util.shed_util_common import (
+    get_category,
+    count_repositories_in_category,
+)
+from tool_shed.util.tool_util import generate_message_for_invalid_tools
 from tool_shed.webapp.model import (
     Repository,
     User,
 )
+from tool_shed.webapp.search.repo_search import RepoSearch
+from tool_shed_client.schema import (
+    CreateRepositoryRequest,
+    ExtraRepoInfo,
+    LegacyInstallInfoTuple,
+    Repository as SchemaRepository,
+    RepositoryMetadataInstallInfoDict,
+    ResetMetadataOnRepositoryResponse,
+)
+from .categories import get_value_mapper as category_value_mapper
 
 log = logging.getLogger(__name__)
 
 
+def search(trans: ProvidesUserContext, q: str, page: int = 1, page_size: int = 10):
+    """
+    Perform the search over TS repositories.
+    Note that search works over the Whoosh index which you have
+    to pre-create with scripts/tool_shed/build_ts_whoosh_index.sh manually.
+    Also TS config option toolshed_search_on has to be True and
+    whoosh_index_dir has to be specified.
+    """
+    app = trans.app
+    conf = app.config
+    if not conf.toolshed_search_on:
+        raise ConfigDoesNotAllowException("Searching the TS through the API is turned off for this instance.")
+    if not conf.whoosh_index_dir:
+        raise ConfigDoesNotAllowException(
+            "There is no directory for the search index specified. Please contact the administrator."
+        )
+
+    search_term = q.strip()
+    if len(search_term) < 1:
+        raise RequestParameterInvalidException("The search term has to be at least one character long.")
+
+    repo_search = RepoSearch()
+
+    Boosts = namedtuple(
+        "Boosts",
+        [
+            "repo_name_boost",
+            "repo_description_boost",
+            "repo_long_description_boost",
+            "repo_homepage_url_boost",
+            "repo_remote_repository_url_boost",
+            "categories_boost",
+            "repo_owner_username_boost",
+        ],
+    )
+    boosts = Boosts(
+        float(conf.get("repo_name_boost", 0.9)),
+        float(conf.get("repo_description_boost", 0.6)),
+        float(conf.get("repo_long_description_boost", 0.5)),
+        float(conf.get("repo_homepage_url_boost", 0.3)),
+        float(conf.get("repo_remote_repository_url_boost", 0.2)),
+        float(conf.get("categories_boost", 0.5)),
+        float(conf.get("repo_owner_username_boost", 0.3)),
+    )
+
+    results = repo_search.search(trans, search_term, page, page_size, boosts)
+    results["hostname"] = web.url_for("/", qualified=True)
+    return results
+
+
+class UpdatesRequest(BaseModel):
+    name: Optional[str] = None
+    owner: Optional[str] = None
+    changeset_revision: str
+    hexlify: bool = True
+
+
+def check_updates(app: ToolShedApp, request: UpdatesRequest) -> Union[str, Dict[str, Any]]:
+    name = request.name
+    owner = request.owner
+    changeset_revision = request.changeset_revision
+    hexlify_this = request.hexlify
+    repository = get_repository_by_name_and_owner(
+        app, name, owner, eagerload_columns=[Repository.downloadable_revisions]
+    )
+    if repository and repository.downloadable_revisions:
+        repository_metadata = get_repository_metadata_by_changeset_revision(
+            app, app.security.encode_id(repository.id), changeset_revision
+        )
+        tool_shed_status_dict = {}
+        # Handle repository deprecation.
+        tool_shed_status_dict["repository_deprecated"] = str(repository.deprecated)
+        tip_revision = repository.downloadable_revisions[0]
+        # Handle latest installable revision.
+        if changeset_revision == tip_revision:
+            tool_shed_status_dict["latest_installable_revision"] = "True"
+        else:
+            next_installable_revision = get_next_downloadable_changeset_revision(app, repository, changeset_revision)
+            if repository_metadata is None:
+                if next_installable_revision and next_installable_revision != changeset_revision:
+                    tool_shed_status_dict["latest_installable_revision"] = "True"
+                else:
+                    tool_shed_status_dict["latest_installable_revision"] = "False"
+            else:
+                if next_installable_revision and next_installable_revision != changeset_revision:
+                    tool_shed_status_dict["latest_installable_revision"] = "False"
+                else:
+                    tool_shed_status_dict["latest_installable_revision"] = "True"
+        # Handle revision updates.
+        if changeset_revision == tip_revision:
+            tool_shed_status_dict["revision_update"] = "False"
+        else:
+            if repository_metadata is None:
+                tool_shed_status_dict["revision_update"] = "True"
+            else:
+                tool_shed_status_dict["revision_update"] = "False"
+        # Handle revision upgrades.
+        metadata_revisions = [revision[1] for revision in get_metadata_revisions(app, repository)]
+        num_metadata_revisions = len(metadata_revisions)
+        for index, metadata_revision in enumerate(metadata_revisions):
+            if index == num_metadata_revisions:
+                tool_shed_status_dict["revision_upgrade"] = "False"
+                break
+            if metadata_revision == changeset_revision:
+                if num_metadata_revisions - index > 1:
+                    tool_shed_status_dict["revision_upgrade"] = "True"
+                else:
+                    tool_shed_status_dict["revision_upgrade"] = "False"
+                break
+        return (
+            encoding_util.tool_shed_encode(tool_shed_status_dict) if hexlify_this else json.dumps(tool_shed_status_dict)
+        )
+    return encoding_util.tool_shed_encode({}) if hexlify_this else json.dumps({})
+
+
+def guid_to_repository(app: ToolShedApp, tool_id: str) -> "Repository":
+    # tool_id = remove_protocol_and_user_from_clone_url(tool_id)
+    shed, _, owner, name, rest = tool_id.split("/", 5)
+    clause_list = [
+        and_(
+            app.model.Repository.table.c.deprecated == false(),
+            app.model.Repository.table.c.deleted == false(),
+            app.model.Repository.table.c.name == name,
+            app.model.User.table.c.username == owner,
+            app.model.Repository.table.c.user_id == app.model.User.table.c.id,
+        )
+    ]
+    repository = app.model.context.query(app.model.Repository).filter(*clause_list).first()
+    return repository
+
+
+def index_tool_ids(app: ToolShedApp, tool_ids: List[str]) -> Dict[str, Any]:
+    repository_found = []
+    all_metadata = dict()
+    for tool_id in tool_ids:
+        repository = guid_to_repository(app, tool_id)
+        owner = repository.user.username
+        name = repository.name
+        clause_list = [
+            and_(
+                app.model.Repository.table.c.deprecated == false(),
+                app.model.Repository.table.c.deleted == false(),
+                app.model.Repository.table.c.name == name,
+                app.model.User.table.c.username == owner,
+                app.model.Repository.table.c.user_id == app.model.User.table.c.id,
+            )
+        ]
+        repository = app.model.context.current.sa_session.query(app.model.Repository).filter(*clause_list).first()
+        if not repository:
+            log.warning(f"Repository {owner}/{name} does not exist, skipping")
+            continue
+        for changeset, changehash in repository.installable_revisions(app):
+            metadata = get_current_repository_metadata_for_changeset_revision(app, repository, changehash)
+            tools: Optional[List[Dict[str, Any]]] = metadata.metadata.get("tools")
+            if not tools:
+                log.warning(f"Repository {owner}/{name}/{changehash} does not contain valid tools, skipping")
+                continue
+            for tool_metadata in tools:
+                if tool_metadata["guid"] in tool_ids:
+                    repository_found.append("%d:%s" % (int(changeset), changehash))
+            metadata = get_current_repository_metadata_for_changeset_revision(app, repository, changehash)
+            if metadata is None:
+                continue
+            metadata_dict = metadata.to_dict(
+                value_mapper={"id": app.security.encode_id, "repository_id": app.security.encode_id}
+            )
+            metadata_dict["repository"] = repository.to_dict(value_mapper={"id": app.security.encode_id})
+            if metadata.has_repository_dependencies:
+                metadata_dict["repository_dependencies"] = get_all_dependencies(
+                    app, metadata, processed_dependency_links=[]
+                )
+            else:
+                metadata_dict["repository_dependencies"] = []
+            if metadata.includes_tool_dependencies:
+                metadata_dict["tool_dependencies"] = repository.get_tool_dependencies(app, changehash)
+            else:
+                metadata_dict["tool_dependencies"] = {}
+            if metadata.includes_tools:
+                metadata_dict["tools"] = metadata.metadata["tools"]
+            all_metadata[f"{int(changeset)}:{changehash}"] = metadata_dict
+    if repository_found:
+        all_metadata["current_changeset"] = repository_found[0]
+        # all_metadata[ 'found_changesets' ] = repository_found
+        return all_metadata
+    else:
+        return {}
+
+
+def index_repositories(app: ToolShedApp, name: Optional[str], owner: Optional[str], deleted: bool):
+    clause_list = [
+        and_(
+            app.model.Repository.table.c.deprecated == false(),
+            app.model.Repository.table.c.deleted == deleted,
+        )
+    ]
+    if owner is not None:
+        clause_list.append(
+            and_(
+                app.model.User.table.c.username == owner,
+                app.model.Repository.table.c.user_id == app.model.User.table.c.id,
+            )
+        )
+    if name is not None:
+        clause_list.append(app.model.Repository.table.c.name == name)
+    repositories = []
+    for repository in (
+        app.model.context.query(app.model.Repository).filter(*clause_list).order_by(app.model.Repository.table.c.name)
+    ):
+        repositories.append(repository)
+    return repositories
+
+
+def can_manage_repo(trans: ProvidesUserContext, repository: Repository) -> bool:
+    security_agent = trans.app.security_agent
+    return trans.user_is_admin or security_agent.user_can_administer_repository(trans.user, repository)
+
+
+def get_install_info(app: ToolShedApp, name, owner, changeset_revision) -> LegacyInstallInfoTuple:
+    value_mapper = get_value_mapper(app)
+    # Example URL:
+    # http://<xyz>/api/repositories/get_repository_revision_install_info?name=<n>&owner=<o>&changeset_revision=<cr>
+    if name and owner and changeset_revision:
+        # Get the repository information.
+        repository = get_repository_by_name_and_owner(
+            app, name, owner, eagerload_columns=[Repository.downloadable_revisions]
+        )
+        if repository is None:
+            log.debug(f"Cannot locate repository {name} owned by {owner}")
+            return {}, {}, {}
+        encoded_repository_id = app.security.encode_id(repository.id)
+        repository_dict: dict = repository.to_dict(view="element", value_mapper=value_mapper)
+        repository_dict["url"] = web.url_for(controller="repositories", action="show", id=encoded_repository_id)
+        # Get the repository_metadata information.
+        repository_metadata = get_repository_metadata_by_changeset_revision(
+            app, encoded_repository_id, changeset_revision
+        )
+        if repository_metadata is None:
+            # The changeset_revision column in the repository_metadata table has been updated with a new
+            # value value, so find the changeset_revision to which we need to update.
+            new_changeset_revision = get_next_downloadable_changeset_revision(app, repository, changeset_revision)
+            repository_metadata = get_repository_metadata_by_changeset_revision(
+                app, encoded_repository_id, new_changeset_revision
+            )
+            changeset_revision = new_changeset_revision
+        if repository_metadata is not None:
+            encoded_repository_metadata_id = app.security.encode_id(repository_metadata.id)
+            repository_metadata_dict: RepositoryMetadataInstallInfoDict = cast(
+                RepositoryMetadataInstallInfoDict,
+                repository_metadata.to_dict(view="collection", value_mapper=value_mapper),
+            )
+            repository_metadata_dict["url"] = web.url_for(
+                controller="repository_revisions", action="show", id=encoded_repository_metadata_id
+            )
+            if "tools" in repository_metadata.metadata:
+                repository_metadata_dict["valid_tools"] = repository_metadata.metadata["tools"]
+            # Get the repo_info_dict for installing the repository.
+            repo_info_dict: ExtraRepoInfo
+            (
+                repo_info_dict,
+                includes_tools,
+                includes_tool_dependencies,
+                includes_tools_for_display_in_tool_panel,
+                has_repository_dependencies,
+                has_repository_dependencies_only_if_compiling_contained_td,
+            ) = get_repo_info_dict(app, None, encoded_repository_id, changeset_revision)
+            return repository_dict, repository_metadata_dict, repo_info_dict
+        else:
+            log.debug(
+                "Unable to locate repository_metadata record for repository id %s and changeset_revision %s",
+                repository.id,
+                changeset_revision,
+            )
+            return repository_dict, {}, {}
+    else:
+        debug_msg = "Error in the Tool Shed repositories API in get_repository_revision_install_info: "
+        debug_msg += f"Invalid name {name} or owner {owner} or changeset_revision {changeset_revision} received."
+        log.debug(debug_msg)
+        return {}, {}, {}
+
+
+def get_value_mapper(app: ToolShedApp) -> Dict[str, Callable]:
+    value_mapper = {
+        "id": app.security.encode_id,
+        "repository_id": app.security.encode_id,
+        "user_id": app.security.encode_id,
+    }
+    return value_mapper
+
+
+def get_ordered_installable_revisions(
+    app: ToolShedApp, name: Optional[str], owner: Optional[str], tsr_id: Optional[str]
+) -> List[str]:
+    eagerload_columns = [Repository.downloadable_revisions]
+    if None not in [name, owner]:
+        # Get the repository information.
+        repository = get_repository_by_name_and_owner(app, name, owner, eagerload_columns=eagerload_columns)
+        if repository is None:
+            raise ObjectNotFound(f"No repository named {name} found with owner {owner}")
+    elif tsr_id is not None:
+        repository = get_repository_in_tool_shed(app, tsr_id, eagerload_columns=eagerload_columns)
+    else:
+        error_message = "Error in the Tool Shed repositories API in get_ordered_installable_revisions: "
+        error_message += "invalid parameters received."
+        log.debug(error_message)
+        return []
+    return [revision[1] for revision in repository.installable_revisions(app, sort_revisions=True)]
+
+
+def get_repository_metadata_dict(app: ToolShedApp, id: str, recursive: bool, downloadable_only: bool) -> Dict[str, Any]:
+    all_metadata = {}
+    repository = get_repository_in_tool_shed(app, id, eagerload_columns=[Repository.downloadable_revisions])
+    for changeset, changehash in get_metadata_revisions(
+        app, repository, sort_revisions=True, downloadable=downloadable_only
+    ):
+        metadata = get_current_repository_metadata_for_changeset_revision(app, repository, changehash)
+        if metadata is None:
+            continue
+        metadata_dict = metadata.to_dict(
+            value_mapper={"id": app.security.encode_id, "repository_id": app.security.encode_id}
+        )
+        metadata_dict["repository"] = repository.to_dict(value_mapper={"id": app.security.encode_id})
+        if metadata.has_repository_dependencies and recursive:
+            metadata_dict["repository_dependencies"] = get_all_dependencies(
+                app, metadata, processed_dependency_links=[]
+            )
+        else:
+            metadata_dict["repository_dependencies"] = []
+        if metadata.includes_tools:
+            metadata_dict["tools"] = metadata.metadata["tools"]
+        all_metadata[f"{int(changeset)}:{changehash}"] = metadata_dict
+    return all_metadata
+
+
+def readmes(app: ToolShedApp, repository: Repository, changeset_revision: str) -> dict:
+    encoded_repository_id = app.security.encode_id(repository.id)
+    repository_metadata = get_repository_metadata_by_changeset_revision(app, encoded_repository_id, changeset_revision)
+    if repository_metadata:
+        metadata = repository_metadata.metadata
+        if metadata:
+            return build_readme_files_dict(app, repository, changeset_revision, repository_metadata.metadata)
+    return {}
+
+
+def reset_metadata_on_repository(trans: ProvidesUserContext, repository_id) -> ResetMetadataOnRepositoryResponse:
+    app: ToolShedApp = trans.app
+
+    def handle_repository(trans, start_time, repository):
+        results = dict(start_time=start_time, repository_status=[])
+        try:
+            rmm = repository_metadata_manager.RepositoryMetadataManager(
+                app=app,
+                user=trans.user,
+                repository=repository,
+                resetting_all_metadata_on_repository=True,
+                updating_installed_repository=False,
+                persist=False,
+            )
+            rmm.reset_all_metadata_on_repository_in_tool_shed()
+            rmm_invalid_file_tups = rmm.get_invalid_file_tups()
+            if rmm_invalid_file_tups:
+                message = generate_message_for_invalid_tools(
+                    app, rmm_invalid_file_tups, repository, None, as_html=False
+                )
+                results["status"] = "warning"
+            else:
+                message = (
+                    f"Successfully reset metadata on repository {repository.name} owned by {repository.user.username}"
+                )
+                results["status"] = "ok"
+        except Exception as e:
+            message = (
+                f"Error resetting metadata on repository {repository.name} owned by {repository.user.username}: {e}"
+            )
+            results["status"] = "error"
+        status = f"{repository.name} : {message}"
+        results["repository_status"].append(status)
+        return results
+
+    if repository_id is not None:
+        repository = get_repository_in_tool_shed(app, repository_id)
+        start_time = strftime("%Y-%m-%d %H:%M:%S")
+        log.debug(f"{start_time}...resetting metadata on repository {repository.name}")
+        results = handle_repository(trans, start_time, repository)
+        stop_time = strftime("%Y-%m-%d %H:%M:%S")
+        results["stop_time"] = stop_time
+    return ResetMetadataOnRepositoryResponse(**results)
+
+
+def create_repository(trans: ProvidesUserContext, request: CreateRepositoryRequest) -> Repository:
+    app: ToolShedApp = trans.app
+    user = trans.user
+    assert user
+    category_ids = listify(request.category_ids)
+    name = request.name
+    invalid_message = validate_repository_name(app, name, user)
+    if invalid_message:
+        raise RequestParameterInvalidException(invalid_message)
+
+    repo, _ = low_level_create_repository(
+        app=app,
+        name=name,
+        type=request.type_,
+        description=request.synopsis,
+        long_description=request.description,
+        user_id=user.id,
+        category_ids=category_ids,
+        remote_repository_url=request.remote_repository_url,
+        homepage_url=request.homepage_url,
+    )
+    return repo
+
+
+def to_element_dict(app, repository: Repository, include_categories: bool = False) -> Dict[str, Any]:
+    value_mapper = get_value_mapper(app)
+    repository_dict = repository.to_dict(view="element", value_mapper=value_mapper)
+    if include_categories:
+        repository_dict["category_ids"] = [app.security.encode_id(x.category.id) for x in repository.categories]
+    return repository_dict
+
+
+def repositories_by_category(
+    app: ToolShedApp,
+    category_id: str,
+    page: Optional[int] = None,
+    sort_key: str = "name",
+    sort_order: str = "asc",
+    installable: bool = True,
+):
+    category = get_category(app, category_id)
+    category_dict: Dict[str, Any]
+    if category is None:
+        category_dict = dict(message=f"Unable to locate category record for id {str(id)}.", status="error")
+        return category_dict
+    category_dict = category.to_dict(view="element", value_mapper=category_value_mapper(app))
+    category_dict["repository_count"] = count_repositories_in_category(app, category_id)
+    repositories = get_repositories_by_category(
+        app, category.id, installable=installable, sort_order=sort_order, sort_key=sort_key, page=page
+    )
+    category_dict["repositories"] = repositories
+    return category_dict
+
+
+def to_model(app, repository: Repository) -> SchemaRepository:
+    return SchemaRepository(**to_element_dict(app, repository))
+
+
 def upload_tar_and_set_metadata(
     app: ToolShedApp,
     host: str,

lib/tool_shed/managers/tools.py

@@ -0,0 +1,44 @@
+from collections import namedtuple
+
+from galaxy import exceptions
+from tool_shed.context import SessionRequestContext
+from tool_shed.webapp.search.tool_search import ToolSearch
+
+
+def search(trans: SessionRequestContext, q: str, page: int = 1, page_size: int = 10) -> dict:
+    """
+    Perform the search over TS tools index.
+    Note that search works over the Whoosh index which you have
+    to pre-create with scripts/tool_shed/build_ts_whoosh_index.sh manually.
+    Also TS config option toolshed_search_on has to be True and
+    whoosh_index_dir has to be specified.
+    """
+    app = trans.app
+    conf = app.config
+    if not conf.toolshed_search_on:
+        raise exceptions.ConfigDoesNotAllowException(
+            "Searching the TS through the API is turned off for this instance."
+        )
+    if not conf.whoosh_index_dir:
+        raise exceptions.ConfigDoesNotAllowException(
+            "There is no directory for the search index specified. Please contact the administrator."
+        )
+    search_term = q.strip()
+    if len(search_term) < 1:
+        raise exceptions.RequestParameterInvalidException("The search term has to be at least one character long.")
+
+    tool_search = ToolSearch()
+
+    Boosts = namedtuple(
+        "Boosts", ["tool_name_boost", "tool_description_boost", "tool_help_boost", "tool_repo_owner_username_boost"]
+    )
+    boosts = Boosts(
+        float(conf.get("tool_name_boost", 1.2)),
+        float(conf.get("tool_description_boost", 0.6)),
+        float(conf.get("tool_help_boost", 0.4)),
+        float(conf.get("tool_repo_owner_username_boost", 0.3)),
+    )
+
+    results = tool_search.search(trans.app, search_term, page, page_size, boosts)
+    results["hostname"] = trans.url_builder("/", qualified=True)
+    return results

lib/tool_shed/managers/users.py

@@ -1,8 +1,35 @@
+from typing import List
+
+from galaxy.exceptions import RequestParameterInvalidException
+from galaxy.model.base import transaction
+from galaxy.security.validate_user_input import (
+    validate_email,
+    validate_password,
+    validate_publicname,
+)
+from tool_shed.context import ProvidesUserContext
 from tool_shed.structured_app import ToolShedApp
 from tool_shed.webapp.model import User
+from tool_shed_client.schema import (
+    CreateUserRequest,
+    User as ApiUser,
+)
+
+
+def index(app: ToolShedApp, deleted: bool) -> List[ApiUser]:
+    users: List[ApiUser] = []
+    for user in (
+        app.model.context.query(app.model.User)
+        .filter(app.model.User.table.c.deleted == deleted)
+        .order_by(app.model.User.table.c.username)
+    ):
+        users.append(get_api_user(app, user))
+    return users
 
 
 def create_user(app: ToolShedApp, email: str, username: str, password: str) -> User:
+    if username == "repos":
+        raise RequestParameterInvalidException("Cannot create a tool shed user with the username repos")
     sa_session = app.model.context
     user = User(email=email)
     user.set_password_cleartext(password)
@@ -13,6 +40,38 @@ def create_user(app: ToolShedApp, email: str, username: str, password: str) -> U
     # else:
     #    user.active = True  # Activation is off, every new user is active by default.
     sa_session.add(user)
-    sa_session.flush()
+    with transaction(sa_session):
+        sa_session.commit()
     app.security_agent.create_private_user_role(user)
     return user
+
+
+def api_create_user(trans: ProvidesUserContext, request: CreateUserRequest) -> ApiUser:
+    app = trans.app
+    message = _validate(
+        trans, email=request.email, password=request.password, confirm=request.password, username=request.username
+    )
+    if message:
+        raise RequestParameterInvalidException(message)
+    user = create_user(app, request.email, request.username, request.password)
+    return get_api_user(app, user)
+
+
+def get_api_user(app: ToolShedApp, user: User) -> ApiUser:
+    return ApiUser(
+        id=app.security.encode_id(user.id),
+        username=user.username,
+    )
+
+
+def _validate(trans: ProvidesUserContext, email: str, password: str, confirm: str, username: str) -> str:
+    if username in ["repos"]:
+        return f"The term '{username}' is a reserved word in the Tool Shed, so it cannot be used as a public user name."
+    message = "\n".join(
+        (
+            validate_email(trans, email),
+            validate_password(trans, password, confirm),
+            validate_publicname(trans, username),
+        )
+    ).rstrip()
+    return message

lib/tool_shed/test/functional/api_notes.md

@@ -41,4 +41,10 @@ for deletion instead of writing new tests and modernizing the API.
 | repositories/{repository_id}/changeset_revision | NO | YES | NO | YES | NO | |
 | POST repositories | NO | YES | NO | YES | NO | |
 | GET repositories (without search query) | ? | ? |? | True | True | |
+| GET /repositories/updates/ | YES | NO | NO | NO | YES | |
 
+Research if searching by tool_ids is used with the repository index API.
+
+Added in:
+- https://github.com/galaxyproject/galaxy/pull/3626/files
+- Likely no longer used?

lib/tool_shed/test/functional/test_shed_repositories.py

@@ -52,6 +52,8 @@ class TestShedRepositoriesApi(ShedApiTestCase):
         assert not only_revision.malicious
 
     def test_index_simple(self):
+        # Logic and typing is pretty different if given a tool id to search for - this should
+        # be tested or dropped in v2.
         populator = self.populator
         repo = populator.setup_column_maker_repo(prefix="repoforindex")
         repository_id = repo.id

lib/tool_shed/webapp/api/categories.py

@@ -1,13 +1,13 @@
 import logging
 from typing import (
     Any,
-    Callable,
     Dict,
+    List,
 )
 
 import tool_shed.util.shed_util_common as suc
+import tool_shed_client.schema
 from galaxy import (
-    exceptions,
     util,
     web,
 )
@@ -17,18 +17,22 @@ from galaxy.web import (
     expose_api_anonymous_and_sessionless,
     require_admin,
 )
-from galaxy.webapps.base.controller import BaseAPIController
-from tool_shed.util import repository_util
+from galaxy.webapps.galaxy.api import depends
+from tool_shed.managers.categories import (
+    CategoryManager,
+    get_value_mapper,
+)
+from tool_shed.managers.repositories import repositories_by_category
+from tool_shed.webapp.model import Category
+from . import BaseShedAPIController
 
 log = logging.getLogger(__name__)
 
 
-class CategoriesController(BaseAPIController):
+class CategoriesController(BaseShedAPIController):
     """RESTful controller for interactions with categories in the Tool Shed."""
 
-    def __get_value_mapper(self, trans) -> Dict[str, Callable]:
-        value_mapper = {"id": trans.security.encode_id}
-        return value_mapper
+    category_manager: CategoryManager = depends(CategoryManager)
 
     @expose_api
     @require_admin
@@ -46,27 +50,16 @@ class CategoriesController(BaseAPIController):
         Content-Disposition: form-data; name="description" Category_Description
         """
         category_dict = dict(message="", status="ok")
-        name = payload.get("name", "")
-        if name:
-            description = payload.get("description", "")
-            if not description:
-                # Default the description to the name.
-                description = name
-            if suc.get_category_by_name(self.app, name):
-                raise exceptions.Conflict("A category with that name already exists.")
-            else:
-                # Create the category
-                category = self.app.model.Category(name=name, description=description)
-                trans.sa_session.add(category)
-                with transaction(trans.sa_session):
-                    trans.sa_session.commit()
-                category_dict = category.to_dict(view="element", value_mapper=self.__get_value_mapper(trans))
-                category_dict["message"] = f"Category '{str(category.name)}' has been created"
-                category_dict["url"] = web.url_for(
-                    controller="categories", action="show", id=trans.security.encode_id(category.id)
-                )
-        else:
-            raise exceptions.RequestParameterMissingException('Missing required parameter "name".')
+        request = tool_shed_client.schema.CreateCategoryRequest(
+            name=payload.get("name"),
+            description=payload.get("description", ""),
+        )
+        category: Category = self.category_manager.create(trans, request)
+        category_dict = category.to_dict(view="element", value_mapper=get_value_mapper(trans.app))
+        category_dict["message"] = f"Category '{str(category.name)}' has been created"
+        category_dict["url"] = web.url_for(
+            controller="categories", action="show", id=trans.security.encode_id(category.id)
+        )
         return category_dict
 
     @expose_api_anonymous_and_sessionless
@@ -86,24 +79,19 @@ class CategoriesController(BaseAPIController):
         sort_key = kwd.get("sort_key", "name")
         sort_order = kwd.get("sort_order", "asc")
         page = kwd.get("page", None)
-        category = suc.get_category(self.app, category_id)
-        category_dict: Dict[str, Any]
-        if category is None:
-            category_dict = dict(message=f"Unable to locate category record for id {str(id)}.", status="error")
-            return category_dict
-        category_dict = category.to_dict(view="element", value_mapper=self.__get_value_mapper(trans))
-        category_dict["repository_count"] = suc.count_repositories_in_category(self.app, category_id)
-        category_dict["url"] = web.url_for(
-            controller="categories", action="show", id=trans.security.encode_id(category.id)
+        category_dict = repositories_by_category(
+            self.app,
+            category_id,
+            page=page,
+            sort_key=sort_key,
+            sort_order=sort_order,
+            installable=installable,
         )
-        repositories = repository_util.get_repositories_by_category(
-            self.app, category.id, installable=installable, sort_order=sort_order, sort_key=sort_key, page=page
-        )
-        category_dict["repositories"] = repositories
+        category_dict["url"] = web.url_for(controller="categories", action="show", id=category_dict["id"])
         return category_dict
 
     @expose_api_anonymous_and_sessionless
-    def index(self, trans, deleted=False, **kwd):
+    def index(self, trans, deleted=False, **kwd) -> List[Dict[str, Any]]:
         """
         GET /api/categories
         Return a list of dictionaries that contain information about each Category.
@@ -112,24 +100,8 @@ class CategoriesController(BaseAPIController):
 
         Example: GET localhost:9009/api/categories
         """
-        category_dicts = []
         deleted = util.asbool(deleted)
-        if deleted and not trans.user_is_admin:
-            raise exceptions.AdminRequiredException("Only administrators can query deleted categories.")
-        for category in (
-            trans.sa_session.query(self.app.model.Category)
-            .filter(self.app.model.Category.table.c.deleted == deleted)
-            .order_by(self.app.model.Category.table.c.name)
-        ):
-            category_dict = category.to_dict(view="collection", value_mapper=self.__get_value_mapper(trans))
-            category_dict["url"] = web.url_for(
-                controller="categories", action="show", id=trans.security.encode_id(category.id)
-            )
-            category_dict[
-                "repositories"
-            ] = self.app.repository_registry.viewable_repositories_and_suites_by_category.get(category.name, 0)
-            category_dicts.append(category_dict)
-        return category_dicts
+        return self.category_manager.index(trans, deleted)
 
     @expose_api_anonymous_and_sessionless
     def show(self, trans, id, **kwd):
@@ -145,7 +117,7 @@ class CategoriesController(BaseAPIController):
         if category is None:
             category_dict = dict(message=f"Unable to locate category record for id {str(id)}.", status="error")
             return category_dict
-        category_dict = category.to_dict(view="element", value_mapper=self.__get_value_mapper(trans))
+        category_dict = category.to_dict(view="element", value_mapper=get_value_mapper(trans.app))
         category_dict["url"] = web.url_for(
             controller="categories", action="show", id=trans.security.encode_id(category.id)
         )

lib/tool_shed/webapp/api/configuration.py

@@ -5,15 +5,12 @@ capabilities and configuration settings.
 import logging
 
 from galaxy.web import expose_api_anonymous_and_sessionless
-from galaxy.webapps.base.controller import BaseAPIController
+from . import BaseShedAPIController
 
 log = logging.getLogger(__name__)
 
 
-class ConfigurationController(BaseAPIController):
-    def __init__(self, app):
-        super().__init__(app)
-
+class ConfigurationController(BaseShedAPIController):
     @expose_api_anonymous_and_sessionless
     def version(self, trans, **kwds):
         """

lib/tool_shed/webapp/api/groups.py

@@ -22,16 +22,17 @@ from galaxy.web import (
     expose_api_anonymous_and_sessionless,
     require_admin,
 )
-from galaxy.webapps.base.controller import BaseAPIController
 from tool_shed.managers import groups
+from tool_shed.structured_app import ToolShedApp
+from . import BaseShedAPIController
 
 log = logging.getLogger(__name__)
 
 
-class GroupsController(BaseAPIController):
+class GroupsController(BaseShedAPIController):
     """RESTful controller for interactions with groups in the Tool Shed."""
 
-    def __init__(self, app):
+    def __init__(self, app: ToolShedApp):
         super().__init__(app)
         self.group_manager = groups.GroupManager()
 

lib/tool_shed/webapp/api/repositories.py

@@ -1,7 +1,6 @@
 import json
 import logging
 import os
-from collections import namedtuple
 from io import StringIO
 from time import strftime
 from typing import (
@@ -9,10 +8,6 @@ from typing import (
     Dict,
 )
 
-from sqlalchemy import (
-    and_,
-    false,
-)
 from webob.compat import cgi_FieldStorage
 
 from galaxy import (
@@ -21,7 +16,6 @@ from galaxy import (
 )
 from galaxy.exceptions import (
     ActionInputError,
-    ConfigDoesNotAllowException,
     InsufficientPermissionsException,
     MessageException,
     ObjectNotFound,
@@ -33,26 +27,41 @@ from galaxy.web import (
     expose_api_anonymous_and_sessionless,
     expose_api_raw_anonymous_and_sessionless,
 )
-from galaxy.webapps.base.controller import (
-    BaseAPIController,
-    HTTPBadRequest,
+from galaxy.webapps.base.controller import HTTPBadRequest
+from tool_shed.managers.repositories import (
+    can_update_repo,
+    check_updates,
+    create_repository,
+    get_install_info,
+    get_ordered_installable_revisions,
+    get_repository_metadata_dict,
+    get_value_mapper,
+    index_repositories,
+    index_tool_ids,
+    reset_metadata_on_repository,
+    search,
+    to_element_dict,
+    UpdatesRequest,
+    upload_tar_and_set_metadata,
 )
-from tool_shed.managers.repositories import upload_tar_and_set_metadata
 from tool_shed.metadata import repository_metadata_manager
 from tool_shed.repository_types import util as rt_util
 from tool_shed.util import (
-    encoding_util,
     metadata_util,
     repository_util,
     tool_util,
 )
 from tool_shed.webapp import model
-from tool_shed.webapp.search.repo_search import RepoSearch
+from tool_shed_client.schema import (
+    CreateRepositoryRequest,
+    LegacyInstallInfoTuple,
+)
+from . import BaseShedAPIController
 
 log = logging.getLogger(__name__)
 
 
-class RepositoriesController(BaseAPIController):
+class RepositoriesController(BaseShedAPIController):
     """RESTful controller for interactions with repositories in the Tool Shed."""
 
     @web.legacy_expose_api
@@ -115,28 +124,12 @@ class RepositoriesController(BaseAPIController):
         if owner is None:
             owner = kwd.get("owner", None)
         tsr_id = kwd.get("tsr_id", None)
-        eagerload_columns = [model.Repository.downloadable_revisions]
-        if None not in [name, owner]:
-            # Get the repository information.
-            repository = repository_util.get_repository_by_name_and_owner(
-                self.app, name, owner, eagerload_columns=eagerload_columns
-            )
-            if repository is None:
-                trans.response.status = 404
-                return {"status": "error", "message": f"No repository named {name} found with owner {owner}"}
-        elif tsr_id is not None:
-            repository = repository_util.get_repository_in_tool_shed(
-                self.app, tsr_id, eagerload_columns=eagerload_columns
-            )
-        else:
-            error_message = "Error in the Tool Shed repositories API in get_ordered_installable_revisions: "
-            error_message += "invalid parameters received."
-            log.debug(error_message)
-            return []
-        return [revision[1] for revision in repository.installable_revisions(self.app, sort_revisions=True)]
+        return get_ordered_installable_revisions(self.app, name, owner, tsr_id)
 
     @web.legacy_expose_api_anonymous
-    def get_repository_revision_install_info(self, trans, name, owner, changeset_revision, **kwd):
+    def get_repository_revision_install_info(
+        self, trans, name, owner, changeset_revision, **kwd
+    ) -> LegacyInstallInfoTuple:
         """
         GET /api/repositories/get_repository_revision_install_info
 
@@ -209,65 +202,7 @@ class RepositoriesController(BaseAPIController):
             }
 
         """
-        # Example URL:
-        # http://<xyz>/api/repositories/get_repository_revision_install_info?name=<n>&owner=<o>&changeset_revision=<cr>
-        if name and owner and changeset_revision:
-            # Get the repository information.
-            repository = repository_util.get_repository_by_name_and_owner(
-                self.app, name, owner, eagerload_columns=[model.Repository.downloadable_revisions]
-            )
-            if repository is None:
-                log.debug(f"Cannot locate repository {name} owned by {owner}")
-                return {}, {}, {}
-            encoded_repository_id = trans.security.encode_id(repository.id)
-            repository_dict = repository.to_dict(view="element", value_mapper=self.__get_value_mapper(trans))
-            repository_dict["url"] = web.url_for(controller="repositories", action="show", id=encoded_repository_id)
-            # Get the repository_metadata information.
-            repository_metadata = metadata_util.get_repository_metadata_by_changeset_revision(
-                self.app, encoded_repository_id, changeset_revision
-            )
-            if repository_metadata is None:
-                # The changeset_revision column in the repository_metadata table has been updated with a new
-                # value value, so find the changeset_revision to which we need to update.
-                new_changeset_revision = metadata_util.get_next_downloadable_changeset_revision(
-                    self.app, repository, changeset_revision
-                )
-                repository_metadata = metadata_util.get_repository_metadata_by_changeset_revision(
-                    self.app, encoded_repository_id, new_changeset_revision
-                )
-                changeset_revision = new_changeset_revision
-            if repository_metadata is not None:
-                encoded_repository_metadata_id = trans.security.encode_id(repository_metadata.id)
-                repository_metadata_dict = repository_metadata.to_dict(
-                    view="collection", value_mapper=self.__get_value_mapper(trans)
-                )
-                repository_metadata_dict["url"] = web.url_for(
-                    controller="repository_revisions", action="show", id=encoded_repository_metadata_id
-                )
-                if "tools" in repository_metadata.metadata:
-                    repository_metadata_dict["valid_tools"] = repository_metadata.metadata["tools"]
-                # Get the repo_info_dict for installing the repository.
-                (
-                    repo_info_dict,
-                    includes_tools,
-                    includes_tool_dependencies,
-                    includes_tools_for_display_in_tool_panel,
-                    has_repository_dependencies,
-                    has_repository_dependencies_only_if_compiling_contained_td,
-                ) = repository_util.get_repo_info_dict(self.app, trans.user, encoded_repository_id, changeset_revision)
-                return repository_dict, repository_metadata_dict, repo_info_dict
-            else:
-                log.debug(
-                    "Unable to locate repository_metadata record for repository id %s and changeset_revision %s",
-                    repository.id,
-                    changeset_revision,
-                )
-                return repository_dict, {}, {}
-        else:
-            debug_msg = "Error in the Tool Shed repositories API in get_repository_revision_install_info: "
-            debug_msg += f"Invalid name {name} or owner {owner} or changeset_revision {changeset_revision} received."
-            log.debug(debug_msg)
-            return {}, {}, {}
+        return get_install_info(self.app, name, owner, changeset_revision)
 
     @web.legacy_expose_api_anonymous
     def get_installable_revisions(self, trans, **kwd):
@@ -292,12 +227,7 @@ class RepositoriesController(BaseAPIController):
         return repository.installable_revisions(self.app)
 
     def __get_value_mapper(self, trans) -> Dict[str, Callable]:
-        value_mapper = {
-            "id": trans.security.encode_id,
-            "repository_id": trans.security.encode_id,
-            "user_id": trans.security.encode_id,
-        }
-        return value_mapper
+        return get_value_mapper(self.app)
 
     @expose_api_raw_anonymous_and_sessionless
     def index(self, trans, deleted=False, owner=None, name=None, **kwd):
@@ -352,7 +282,7 @@ class RepositoriesController(BaseAPIController):
                 raise RequestParameterInvalidException('The "page" and "page_size" parameters have to be integers.')
             return_jsonp = util.asbool(kwd.get("jsonp", False))
             callback = kwd.get("callback", "callback")
-            search_results = self._search(trans, q, page, page_size)
+            search_results = search(trans, q, page, page_size)
             if return_jsonp:
                 response = str(f"{callback}({json.dumps(search_results)});")
             else:
@@ -361,134 +291,18 @@ class RepositoriesController(BaseAPIController):
         tool_ids = kwd.get("tool_ids", None)
         if tool_ids is not None:
             tool_ids = util.listify(tool_ids)
-            repository_found = []
-            all_metadata = dict()
-            for tool_id in tool_ids:
-                # A valid GUID looks like toolshed.g2.bx.psu.edu/repos/bgruening/deeptools/deeptools_computeMatrix/1.1.0
-                shed, _, owner, name, tool, version = tool_id.split("/")
-                clause_list = [
-                    and_(
-                        self.app.model.Repository.table.c.deprecated == false(),
-                        self.app.model.Repository.table.c.deleted == false(),
-                        self.app.model.Repository.table.c.name == name,
-                        self.app.model.User.table.c.username == owner,
-                        self.app.model.Repository.table.c.user_id == self.app.model.User.table.c.id,
-                    )
+            response = index_tool_ids(self.app, tool_ids)
+            return json.dumps(response)
+        else:
+            repositories = index_repositories(self.app, name, owner, deleted)
+            repository_dicts = []
+            for repository in repositories:
+                repository_dict = repository.to_dict(view="collection", value_mapper=self.__get_value_mapper(trans))
+                repository_dict["category_ids"] = [
+                    trans.security.encode_id(x.category.id) for x in repository.categories
                 ]
-                repository = trans.sa_session.query(self.app.model.Repository).filter(*clause_list).first()
-                if not repository:
-                    log.warning(f"Repository {owner}/{name} does not exist, skipping")
-                    continue
-                for changeset, changehash in repository.installable_revisions(self.app):
-                    metadata = metadata_util.get_current_repository_metadata_for_changeset_revision(
-                        self.app, repository, changehash
-                    )
-                    tools = metadata.metadata.get("tools")
-                    if not tools:
-                        log.warning(f"Repository {owner}/{name}/{changehash} does not contain valid tools, skipping")
-                        continue
-                    for tool in tools:
-                        if tool["guid"] in tool_ids:
-                            repository_found.append("%d:%s" % (int(changeset), changehash))
-                    metadata = metadata_util.get_current_repository_metadata_for_changeset_revision(
-                        self.app, repository, changehash
-                    )
-                    if metadata is None:
-                        continue
-                    metadata_dict = metadata.to_dict(
-                        value_mapper={"id": self.app.security.encode_id, "repository_id": self.app.security.encode_id}
-                    )
-                    metadata_dict["repository"] = repository.to_dict(value_mapper={"id": self.app.security.encode_id})
-                    if metadata.has_repository_dependencies:
-                        metadata_dict["repository_dependencies"] = metadata_util.get_all_dependencies(
-                            self.app, metadata, processed_dependency_links=[]
-                        )
-                    else:
-                        metadata_dict["repository_dependencies"] = []
-                    if metadata.includes_tool_dependencies:
-                        metadata_dict["tool_dependencies"] = repository.get_tool_dependencies(self.app, changehash)
-                    else:
-                        metadata_dict["tool_dependencies"] = {}
-                    if metadata.includes_tools:
-                        metadata_dict["tools"] = metadata.metadata["tools"]
-                    all_metadata[f"{int(changeset)}:{changehash}"] = metadata_dict
-            if repository_found:
-                all_metadata["current_changeset"] = repository_found[0]
-                # all_metadata[ 'found_changesets' ] = repository_found
-                return json.dumps(all_metadata)
-            return "{}"
-
-        clause_list = [
-            and_(
-                self.app.model.Repository.table.c.deprecated == false(),
-                self.app.model.Repository.table.c.deleted == deleted,
-            )
-        ]
-        if owner is not None:
-            clause_list.append(
-                and_(
-                    self.app.model.User.table.c.username == owner,
-                    self.app.model.Repository.table.c.user_id == self.app.model.User.table.c.id,
-                )
-            )
-        if name is not None:
-            clause_list.append(self.app.model.Repository.table.c.name == name)
-        for repository in (
-            trans.sa_session.query(self.app.model.Repository)
-            .filter(*clause_list)
-            .order_by(self.app.model.Repository.table.c.name)
-        ):
-            repository_dict = repository.to_dict(view="collection", value_mapper=self.__get_value_mapper(trans))
-            repository_dict["category_ids"] = [trans.security.encode_id(x.category.id) for x in repository.categories]
-            repository_dicts.append(repository_dict)
-        return json.dumps(repository_dicts)
-
-    def _search(self, trans, q, page=1, page_size=10):
-        """
-        Perform the search over TS repositories.
-        Note that search works over the Whoosh index which you have
-        to pre-create with scripts/tool_shed/build_ts_whoosh_index.sh manually.
-        Also TS config option toolshed_search_on has to be True and
-        whoosh_index_dir has to be specified.
-        """
-        conf = self.app.config
-        if not conf.toolshed_search_on:
-            raise ConfigDoesNotAllowException("Searching the TS through the API is turned off for this instance.")
-        if not conf.whoosh_index_dir:
-            raise ConfigDoesNotAllowException(
-                "There is no directory for the search index specified. Please contact the administrator."
-            )
-        search_term = q.strip()
-        if len(search_term) < 1:
-            raise RequestParameterInvalidException("The search term has to be at least one character long.")
-
-        repo_search = RepoSearch()
-
-        Boosts = namedtuple(
-            "Boosts",
-            [
-                "repo_name_boost",
-                "repo_description_boost",
-                "repo_long_description_boost",
-                "repo_homepage_url_boost",
-                "repo_remote_repository_url_boost",
-                "categories_boost",
-                "repo_owner_username_boost",
-            ],
-        )
-        boosts = Boosts(
-            float(conf.get("repo_name_boost", 0.9)),
-            float(conf.get("repo_description_boost", 0.6)),
-            float(conf.get("repo_long_description_boost", 0.5)),
-            float(conf.get("repo_homepage_url_boost", 0.3)),
-            float(conf.get("repo_remote_repository_url_boost", 0.2)),
-            float(conf.get("categories_boost", 0.5)),
-            float(conf.get("repo_owner_username_boost", 0.3)),
-        )
-
-        results = repo_search.search(trans, search_term, page, page_size, boosts)
-        results["hostname"] = web.url_for("/", qualified=True)
-        return results
+                repository_dicts.append(repository_dict)
+            return json.dumps(repository_dicts)
 
     @web.legacy_expose_api
     def remove_repository_registry_entry(self, trans, payload, **kwd):
@@ -654,46 +468,8 @@ class RepositoriesController(BaseAPIController):
         The following parameters must be included in the payload.
         :param repository_id: the encoded id of the repository on which metadata is to be reset.
         """
-
-        def handle_repository(trans, start_time, repository):
-            results = dict(start_time=start_time, repository_status=[])
-            try:
-                rmm = repository_metadata_manager.RepositoryMetadataManager(
-                    app=self.app,
-                    user=trans.user,
-                    repository=repository,
-                    resetting_all_metadata_on_repository=True,
-                    updating_installed_repository=False,
-                    persist=False,
-                )
-                rmm.reset_all_metadata_on_repository_in_tool_shed()
-                rmm_invalid_file_tups = rmm.get_invalid_file_tups()
-                if rmm_invalid_file_tups:
-                    message = tool_util.generate_message_for_invalid_tools(
-                        self.app, rmm_invalid_file_tups, repository, None, as_html=False
-                    )
-                    results["status"] = "warning"
-                else:
-                    message = f"Successfully reset metadata on repository {repository.name} owned by {repository.user.username}"
-                    results["status"] = "ok"
-            except Exception as e:
-                message = (
-                    f"Error resetting metadata on repository {repository.name} owned by {repository.user.username}: {e}"
-                )
-                results["status"] = "error"
-            status = f"{repository.name} : {message}"
-            results["repository_status"].append(status)
-            return results
-
         repository_id = payload.get("repository_id", None)
-        if repository_id is not None:
-            repository = repository_util.get_repository_in_tool_shed(self.app, repository_id)
-            start_time = strftime("%Y-%m-%d %H:%M:%S")
-            log.debug(f"{start_time}...resetting metadata on repository {repository.name}")
-            results = handle_repository(trans, start_time, repository)
-            stop_time = strftime("%Y-%m-%d %H:%M:%S")
-            results["stop_time"] = stop_time
-        return results
+        return reset_metadata_on_repository(trans, repository_id).dict()
 
     @expose_api_anonymous_and_sessionless
     def show(self, trans, id, **kwd):
@@ -743,63 +519,13 @@ class RepositoriesController(BaseAPIController):
         owner = kwd.get("owner", None)
         changeset_revision = kwd.get("changeset_revision", None)
         hexlify_this = util.asbool(kwd.get("hexlify", True))
-        repository = repository_util.get_repository_by_name_and_owner(
-            trans.app, name, owner, eagerload_columns=[model.Repository.downloadable_revisions]
+        request = UpdatesRequest(
+            name=name,
+            owner=owner,
+            changeset_revision=changeset_revision,
+            hexlify=hexlify_this,
         )
-        if repository and repository.downloadable_revisions:
-            repository_metadata = metadata_util.get_repository_metadata_by_changeset_revision(
-                trans.app, trans.security.encode_id(repository.id), changeset_revision
-            )
-            tool_shed_status_dict = {}
-            # Handle repository deprecation.
-            tool_shed_status_dict["repository_deprecated"] = str(repository.deprecated)
-            tip_revision = repository.downloadable_revisions[0]
-            # Handle latest installable revision.
-            if changeset_revision == tip_revision:
-                tool_shed_status_dict["latest_installable_revision"] = "True"
-            else:
-                next_installable_revision = metadata_util.get_next_downloadable_changeset_revision(
-                    trans.app, repository, changeset_revision
-                )
-                if repository_metadata is None:
-                    if next_installable_revision and next_installable_revision != changeset_revision:
-                        tool_shed_status_dict["latest_installable_revision"] = "True"
-                    else:
-                        tool_shed_status_dict["latest_installable_revision"] = "False"
-                else:
-                    if next_installable_revision and next_installable_revision != changeset_revision:
-                        tool_shed_status_dict["latest_installable_revision"] = "False"
-                    else:
-                        tool_shed_status_dict["latest_installable_revision"] = "True"
-            # Handle revision updates.
-            if changeset_revision == tip_revision:
-                tool_shed_status_dict["revision_update"] = "False"
-            else:
-                if repository_metadata is None:
-                    tool_shed_status_dict["revision_update"] = "True"
-                else:
-                    tool_shed_status_dict["revision_update"] = "False"
-            # Handle revision upgrades.
-            metadata_revisions = [
-                revision[1] for revision in metadata_util.get_metadata_revisions(trans.app, repository)
-            ]
-            num_metadata_revisions = len(metadata_revisions)
-            for index, metadata_revision in enumerate(metadata_revisions):
-                if index == num_metadata_revisions:
-                    tool_shed_status_dict["revision_upgrade"] = "False"
-                    break
-                if metadata_revision == changeset_revision:
-                    if num_metadata_revisions - index > 1:
-                        tool_shed_status_dict["revision_upgrade"] = "True"
-                    else:
-                        tool_shed_status_dict["revision_upgrade"] = "False"
-                    break
-            return (
-                encoding_util.tool_shed_encode(tool_shed_status_dict)
-                if hexlify_this
-                else json.dumps(tool_shed_status_dict)
-            )
-        return encoding_util.tool_shed_encode({}) if hexlify_this else json.dumps({})
+        return check_updates(trans.app, request)
 
     @expose_api_anonymous_and_sessionless
     def show_tools(self, trans, id, changeset, **kwd):
@@ -843,32 +569,7 @@ class RepositoriesController(BaseAPIController):
         """
         recursive = util.asbool(kwd.get("recursive", "True"))
         downloadable_only = util.asbool(kwd.get("downloadable_only", "True"))
-        all_metadata = {}
-        repository = repository_util.get_repository_in_tool_shed(
-            self.app, id, eagerload_columns=[model.Repository.downloadable_revisions]
-        )
-        for changeset, changehash in metadata_util.get_metadata_revisions(
-            self.app, repository, sort_revisions=True, downloadable=downloadable_only
-        ):
-            metadata = metadata_util.get_current_repository_metadata_for_changeset_revision(
-                self.app, repository, changehash
-            )
-            if metadata is None:
-                continue
-            metadata_dict = metadata.to_dict(
-                value_mapper={"id": self.app.security.encode_id, "repository_id": self.app.security.encode_id}
-            )
-            metadata_dict["repository"] = repository.to_dict(value_mapper={"id": self.app.security.encode_id})
-            if metadata.has_repository_dependencies and recursive:
-                metadata_dict["repository_dependencies"] = metadata_util.get_all_dependencies(
-                    self.app, metadata, processed_dependency_links=[]
-                )
-            else:
-                metadata_dict["repository_dependencies"] = []
-            if metadata.includes_tools:
-                metadata_dict["tools"] = metadata.metadata["tools"]
-            all_metadata[f"{int(changeset)}:{changehash}"] = metadata_dict
-        return all_metadata
+        return get_repository_metadata_dict(self.app, id, recursive, downloadable_only)
 
     @expose_api
     def update(self, trans, id, **kwd):
@@ -966,31 +667,22 @@ class RepositoriesController(BaseAPIController):
         description = payload.get("description", "")
         remote_repository_url = payload.get("remote_repository_url", "")
         homepage_url = payload.get("homepage_url", "")
-        category_ids = util.listify(payload.get("category_ids[]", ""))
 
         repo_type = payload.get("type", rt_util.UNRESTRICTED)
         if repo_type not in rt_util.types:
             raise RequestParameterInvalidException("This repository type is not valid")
 
-        invalid_message = repository_util.validate_repository_name(self.app, name, trans.user)
-        if invalid_message:
-            raise RequestParameterInvalidException(invalid_message)
-
-        repo, message = repository_util.create_repository(
-            app=self.app,
+        request = CreateRepositoryRequest(
             name=name,
-            type=repo_type,
-            description=synopsis,
-            long_description=description,
-            user_id=trans.user.id,
-            category_ids=category_ids,
+            synopsis=synopsis,
+            description=description,
             remote_repository_url=remote_repository_url,
             homepage_url=homepage_url,
+            category_ids=payload.get("category_ids[]", ""),
+            type_=repo_type,
         )
-
-        repository_dict = repo.to_dict(view="element", value_mapper=self.__get_value_mapper(trans))
-        repository_dict["category_ids"] = [trans.security.encode_id(x.category.id) for x in repo.categories]
-        return repository_dict
+        repo = create_repository(trans, request)
+        return to_element_dict(self.app, repo, include_categories=True)
 
     @web.legacy_expose_api
     def create_changeset_revision(self, trans, id, payload, **kwd):
@@ -1011,11 +703,7 @@ class RepositoriesController(BaseAPIController):
         # Example URL: http://localhost:9009/api/repositories/f9cad7b01a472135
         repository = repository_util.get_repository_in_tool_shed(self.app, id)
 
-        if not (
-            trans.user_is_admin
-            or self.app.security_agent.user_can_administer_repository(trans.user, repository)
-            or self.app.security_agent.can_push(self.app, trans.user, repository)
-        ):
+        if not can_update_repo(trans, repository):
             trans.response.status = 400
             return {
                 "err_msg": "You do not have permission to update this repository.",

lib/tool_shed/webapp/api/repository_revisions.py

@@ -11,19 +11,17 @@ from galaxy import (
     web,
 )
 from galaxy.model.base import transaction
-from galaxy.webapps.base.controller import (
-    BaseAPIController,
-    HTTPBadRequest,
-)
+from galaxy.webapps.base.controller import HTTPBadRequest
 from tool_shed.util import (
     metadata_util,
     repository_util,
 )
+from . import BaseShedAPIController
 
 log = logging.getLogger(__name__)
 
 
-class RepositoryRevisionsController(BaseAPIController):
+class RepositoryRevisionsController(BaseShedAPIController):
     """RESTful controller for interactions with tool shed repository revisions."""
 
     def __get_value_mapper(self, trans) -> Dict[str, Callable]:

lib/tool_shed/webapp/api/tools.py

@@ -1,25 +1,23 @@
 import json
 import logging
-from collections import namedtuple
 
 from galaxy import (
     exceptions,
     util,
-    web,
 )
 from galaxy.web import (
     expose_api,
     expose_api_raw_anonymous_and_sessionless,
     require_admin,
 )
-from galaxy.webapps.base.controller import BaseAPIController
+from tool_shed.managers.tools import search
 from tool_shed.util.shed_index import build_index
-from tool_shed.webapp.search.tool_search import ToolSearch
+from . import BaseShedAPIController
 
 log = logging.getLogger(__name__)
 
 
-class ToolsController(BaseAPIController):
+class ToolsController(BaseShedAPIController):
     """RESTful controller for interactions with tools in the Tool Shed."""
 
     @expose_api
@@ -85,46 +83,9 @@ class ToolsController(BaseAPIController):
                 raise exceptions.RequestParameterInvalidException('The "page" and "page_size" have to be integers.')
             return_jsonp = util.asbool(kwd.get("jsonp", False))
             callback = kwd.get("callback", "callback")
-            search_results = self._search(trans, q, page, page_size)
+            search_results = search(trans, q, page, page_size)
             if return_jsonp:
                 response = str(f"{callback}({json.dumps(search_results)});")
             else:
                 response = json.dumps(search_results)
             return response
-
-    def _search(self, trans, q, page=1, page_size=10):
-        """
-        Perform the search over TS tools index.
-        Note that search works over the Whoosh index which you have
-        to pre-create with scripts/tool_shed/build_ts_whoosh_index.sh manually.
-        Also TS config option toolshed_search_on has to be True and
-        whoosh_index_dir has to be specified.
-        """
-        conf = self.app.config
-        if not conf.toolshed_search_on:
-            raise exceptions.ConfigDoesNotAllowException(
-                "Searching the TS through the API is turned off for this instance."
-            )
-        if not conf.whoosh_index_dir:
-            raise exceptions.ConfigDoesNotAllowException(
-                "There is no directory for the search index specified. Please contact the administrator."
-            )
-        search_term = q.strip()
-        if len(search_term) < 1:
-            raise exceptions.RequestParameterInvalidException("The search term has to be at least one character long.")
-
-        tool_search = ToolSearch()
-
-        Boosts = namedtuple(
-            "Boosts", ["tool_name_boost", "tool_description_boost", "tool_help_boost", "tool_repo_owner_username_boost"]
-        )
-        boosts = Boosts(
-            float(conf.get("tool_name_boost", 1.2)),
-            float(conf.get("tool_description_boost", 0.6)),
-            float(conf.get("tool_help_boost", 0.4)),
-            float(conf.get("tool_repo_owner_username_boost", 0.3)),
-        )
-
-        results = tool_search.search(trans, search_term, page, page_size, boosts)
-        results["hostname"] = web.url_for("/", qualified=True)
-        return results

lib/tool_shed/webapp/api/users.py

@@ -2,23 +2,20 @@ import logging
 
 import tool_shed.util.shed_util_common as suc
 from galaxy import (
-    exceptions,
     util,
     web,
 )
-from galaxy.model.base import transaction
-from galaxy.security.validate_user_input import (
-    validate_email,
-    validate_password,
-    validate_publicname,
+from tool_shed.managers.users import (
+    api_create_user,
+    index,
 )
-from galaxy.webapps.base.controller import BaseAPIController
-from tool_shed.managers.users import create_user
+from tool_shed_client.schema import CreateUserRequest
+from . import BaseShedAPIController
 
 log = logging.getLogger(__name__)
 
 
-class UsersController(BaseAPIController):
+class UsersController(BaseShedAPIController):
     """RESTful controller for interactions with users in the Tool Shed."""
 
     @web.expose_api
@@ -39,20 +36,18 @@ class UsersController(BaseAPIController):
         email = payload.get("email", "")
         password = payload.get("password", "")
         username = payload.get("username", "")
-        message = self.__validate(trans, email=email, password=password, confirm=password, username=username)
-        if message:
-            raise exceptions.RequestParameterInvalidException(message)
-
         # Create the user.
-        user = self.__create_user(trans, email, username, password)
-        user_dict = user.to_dict(view="element", value_mapper=self.__get_value_mapper(trans))
+        request = CreateUserRequest(
+            email=email,
+            username=username,
+            password=password,
+        )
+        user = api_create_user(trans, request)
+        user_dict = user.dict()
         user_dict["message"] = f"User '{str(user.username)}' has been created."
         user_dict["url"] = web.url_for(controller="users", action="show", id=trans.security.encode_id(user.id))
         return user_dict
 
-    def __create_user(self, trans, email, username, password):
-        return create_user(trans.app, email, username, password)
-
     def __get_value_mapper(self, trans):
         value_mapper = {"id": trans.security.encode_id}
         return value_mapper
@@ -66,12 +61,8 @@ class UsersController(BaseAPIController):
         # Example URL: http://localhost:9009/api/users
         user_dicts = []
         deleted = util.asbool(deleted)
-        for user in (
-            trans.sa_session.query(trans.app.model.User)
-            .filter(trans.app.model.User.table.c.deleted == deleted)
-            .order_by(trans.app.model.User.table.c.username)
-        ):
-            user_dict = user.to_dict(view="collection", value_mapper=self.__get_value_mapper(trans))
+        for user in index(trans.app, deleted):
+            user_dict = user.dict()
             user_dict["url"] = web.url_for(controller="users", action="show", id=trans.security.encode_id(user.id))
             user_dicts.append(user_dict)
         return user_dicts
@@ -94,15 +85,3 @@ class UsersController(BaseAPIController):
         user_dict = user.to_dict(view="element", value_mapper=self.__get_value_mapper(trans))
         user_dict["url"] = web.url_for(controller="users", action="show", id=trans.security.encode_id(user.id))
         return user_dict
-
-    def __validate(self, trans, email, password, confirm, username):
-        if username in ["repos"]:
-            return f"The term '{username}' is a reserved word in the Tool Shed, so it cannot be used as a public user name."
-        message = "\n".join(
-            (
-                validate_email(trans, email),
-                validate_password(trans, password, confirm),
-                validate_publicname(trans, username),
-            )
-        ).rstrip()
-        return message

lib/tool_shed/webapp/controllers/repository.py

@@ -35,6 +35,7 @@ from galaxy.web.form_builder import (
 from galaxy.web.legacy_framework import grids
 from galaxy.webapps.base.controller import BaseUIController
 from tool_shed.dependencies.repository import relation_builder
+from tool_shed.managers.repositories import readmes
 from tool_shed.metadata import repository_metadata_manager
 from tool_shed.tools import (
     tool_validator,
@@ -1182,16 +1183,7 @@ class RepositoryController(BaseUIController, ratings_util.ItemRatings):
         changeset_revision = kwd.get("changeset_revision", None)
         if repository_name is not None and repository_owner is not None and changeset_revision is not None:
             repository = repository_util.get_repository_by_name_and_owner(trans.app, repository_name, repository_owner)
-            if repository:
-                repository_metadata = metadata_util.get_repository_metadata_by_changeset_revision(
-                    trans.app, trans.security.encode_id(repository.id), changeset_revision
-                )
-                if repository_metadata:
-                    metadata = repository_metadata.metadata
-                    if metadata:
-                        return readme_util.build_readme_files_dict(
-                            trans.app, repository, changeset_revision, repository_metadata.metadata
-                        )
+            return readmes(trans.app, repository, changeset_revision)
         return {}
 
     @web.json

lib/tool_shed/webapp/search/tool_search.py

@@ -31,7 +31,7 @@ schema = Schema(
 
 
 class ToolSearch:
-    def search(self, trans, search_term, page, page_size, boosts):
+    def search(self, app, search_term, page, page_size, boosts):
         """
         Perform the search on the given search_term
 
@@ -39,7 +39,7 @@ class ToolSearch:
 
         :returns results: dictionary containing number of hits, hits themselves and matched terms for each
         """
-        tool_index_dir = os.path.join(trans.app.config.whoosh_index_dir, "tools")
+        tool_index_dir = os.path.join(app.config.whoosh_index_dir, "tools")
         index_exists = whoosh.index.exists_in(tool_index_dir)
         if index_exists:
             index = whoosh.index.open_dir(tool_index_dir)

lib/tool_shed/webapp/security/__init__.py

@@ -1,5 +1,6 @@
 """Tool Shed Security"""
 import logging
+from typing import List
 
 from sqlalchemy import (
     and_,
@@ -239,9 +240,12 @@ class CommunityRBACAgent(RBACAgent):
             for group in groups:
                 self.associate_components(user=user, group=group)
 
+    def usernames_that_can_push(self, repository) -> List[str]:
+        return listify(repository.allow_push())
+
     def can_push(self, app, user, repository):
         if user:
-            return user.username in listify(repository.allow_push())
+            return user.username in self.usernames_that_can_push(repository)
         return False
 
     def user_can_administer_repository(self, user, repository):

lib/tool_shed_client/schema/__init__.py

@@ -35,6 +35,10 @@ class Repository(BaseModel):
     create_time: str
 
 
+class RepositoryRevisionReadmes(BaseModel):
+    __root__: Dict[str, str]
+
+
 class CreateUserRequest(BaseModel):
     username: str
     email: str