From 85100edff05ff8ddbb6fa9435c59e49802cd24f7 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Thu, 17 Aug 2023 15:05:45 -0400 Subject: [PATCH] Tool Shed API Modernization ahead of FastAPI - Use the controller base that allow DI. - Move a bunch functionality we want to port to FastAPI into manage layer. - Some typing enhancements. --- lib/tool_shed/managers/categories.py | 66 +++ lib/tool_shed/managers/repositories.py | 506 +++++++++++++++++- lib/tool_shed/managers/tools.py | 44 ++ lib/tool_shed/managers/users.py | 61 ++- lib/tool_shed/test/functional/api_notes.md | 6 + .../test/functional/test_shed_repositories.py | 2 + lib/tool_shed/webapp/api/categories.py | 94 ++-- lib/tool_shed/webapp/api/configuration.py | 7 +- lib/tool_shed/webapp/api/groups.py | 7 +- lib/tool_shed/webapp/api/repositories.py | 424 ++------------- .../webapp/api/repository_revisions.py | 8 +- lib/tool_shed/webapp/api/tools.py | 47 +- lib/tool_shed/webapp/api/users.py | 51 +- .../webapp/controllers/repository.py | 12 +- lib/tool_shed/webapp/search/tool_search.py | 4 +- lib/tool_shed/webapp/security/__init__.py | 6 +- lib/tool_shed_client/schema/__init__.py | 4 + 17 files changed, 813 insertions(+), 536 deletions(-) create mode 100644 lib/tool_shed/managers/categories.py create mode 100644 lib/tool_shed/managers/tools.py diff --git a/lib/tool_shed/managers/categories.py b/lib/tool_shed/managers/categories.py new file mode 100644 index 00000000000..b9ae0535b03 --- /dev/null +++ b/lib/tool_shed/managers/categories.py @@ -0,0 +1,66 @@ +from typing import ( + Any, + Callable, + Dict, + List, +) + +import tool_shed.util.shed_util_common as suc +from galaxy import ( + exceptions, + web, +) +from galaxy.model.base import transaction +from tool_shed.context import ProvidesUserContext +from tool_shed.structured_app import ToolShedApp +from tool_shed.webapp.model import Category +from tool_shed_client.schema import CreateCategoryRequest + + +class CategoryManager: + def __init__(self, app: ToolShedApp): + self.app = app + + def create(self, trans: ProvidesUserContext, category_request: CreateCategoryRequest) -> Category: + name = category_request.name + description = category_request.description or name + if name: + if suc.get_category_by_name(self.app, name): + raise exceptions.Conflict("A category with that name already exists.") + else: + # Create the category + category = self.app.model.Category(name=name, description=description) + trans.sa_session.add(category) + with transaction(trans.sa_session): + trans.sa_session.commit() + return category + else: + raise exceptions.RequestParameterMissingException('Missing required parameter "name".') + + def index_db(self, trans: ProvidesUserContext, deleted: bool) -> List[Category]: + category_db_objects: List[Category] = [] + if deleted and not trans.user_is_admin: + raise exceptions.AdminRequiredException("Only administrators can query deleted categories.") + for category in ( + trans.sa_session.query(Category).filter(Category.table.c.deleted == deleted).order_by(Category.table.c.name) + ): + category_db_objects.append(category) + return category_db_objects + + def index(self, trans: ProvidesUserContext, deleted: bool) -> List[Dict[str, Any]]: + category_dicts: List[Dict[str, Any]] = [] + for category in self.index_db(trans, deleted): + category_dict = category.to_dict(view="collection", value_mapper=get_value_mapper(trans.app)) + category_dict["url"] = web.url_for( + controller="categories", action="show", id=trans.security.encode_id(category.id) + ) + category_dict[ + "repositories" + ] = self.app.repository_registry.viewable_repositories_and_suites_by_category.get(category.name, 0) + category_dicts.append(category_dict) + return category_dicts + + +def get_value_mapper(app: ToolShedApp) -> Dict[str, Callable]: + value_mapper = {"id": app.security.encode_id} + return value_mapper diff --git a/lib/tool_shed/managers/repositories.py b/lib/tool_shed/managers/repositories.py index 61625a505de..010232e830b 100644 --- a/lib/tool_shed/managers/repositories.py +++ b/lib/tool_shed/managers/repositories.py @@ -1,26 +1,530 @@ """ Manager and Serializer for TS repositories. """ +import json import logging +from collections import namedtuple +from time import strftime +from typing import ( + Any, + Callable, + cast, + Dict, + List, + Optional, + Union, +) +from pydantic import BaseModel +from sqlalchemy import ( + and_, + false, +) + +from galaxy import web from galaxy.exceptions import ( + ConfigDoesNotAllowException, InternalServerError, MalformedContents, + ObjectNotFound, + RequestParameterInvalidException, ) from galaxy.tool_shed.util import dependency_display +from galaxy.util import listify +from tool_shed.context import ProvidesUserContext from tool_shed.metadata import repository_metadata_manager from tool_shed.repository_types import util as rt_util from tool_shed.structured_app import ToolShedApp -from tool_shed.util import hg_util +from tool_shed.util import ( + encoding_util, + hg_util, +) +from tool_shed.util.metadata_util import ( + get_all_dependencies, + get_current_repository_metadata_for_changeset_revision, + get_metadata_revisions, + get_next_downloadable_changeset_revision, + get_repository_metadata_by_changeset_revision, +) +from tool_shed.util.readme_util import build_readme_files_dict from tool_shed.util.repository_content_util import upload_tar +from tool_shed.util.repository_util import ( + create_repository as low_level_create_repository, + get_repo_info_dict, + get_repositories_by_category, + get_repository_by_name_and_owner, + get_repository_in_tool_shed, + validate_repository_name, +) +from tool_shed.util.shed_util_common import ( + get_category, + count_repositories_in_category, +) +from tool_shed.util.tool_util import generate_message_for_invalid_tools from tool_shed.webapp.model import ( Repository, User, ) +from tool_shed.webapp.search.repo_search import RepoSearch +from tool_shed_client.schema import ( + CreateRepositoryRequest, + ExtraRepoInfo, + LegacyInstallInfoTuple, + Repository as SchemaRepository, + RepositoryMetadataInstallInfoDict, + ResetMetadataOnRepositoryResponse, +) +from .categories import get_value_mapper as category_value_mapper log = logging.getLogger(__name__) +def search(trans: ProvidesUserContext, q: str, page: int = 1, page_size: int = 10): + """ + Perform the search over TS repositories. + Note that search works over the Whoosh index which you have + to pre-create with scripts/tool_shed/build_ts_whoosh_index.sh manually. + Also TS config option toolshed_search_on has to be True and + whoosh_index_dir has to be specified. + """ + app = trans.app + conf = app.config + if not conf.toolshed_search_on: + raise ConfigDoesNotAllowException("Searching the TS through the API is turned off for this instance.") + if not conf.whoosh_index_dir: + raise ConfigDoesNotAllowException( + "There is no directory for the search index specified. Please contact the administrator." + ) + + search_term = q.strip() + if len(search_term) < 1: + raise RequestParameterInvalidException("The search term has to be at least one character long.") + + repo_search = RepoSearch() + + Boosts = namedtuple( + "Boosts", + [ + "repo_name_boost", + "repo_description_boost", + "repo_long_description_boost", + "repo_homepage_url_boost", + "repo_remote_repository_url_boost", + "categories_boost", + "repo_owner_username_boost", + ], + ) + boosts = Boosts( + float(conf.get("repo_name_boost", 0.9)), + float(conf.get("repo_description_boost", 0.6)), + float(conf.get("repo_long_description_boost", 0.5)), + float(conf.get("repo_homepage_url_boost", 0.3)), + float(conf.get("repo_remote_repository_url_boost", 0.2)), + float(conf.get("categories_boost", 0.5)), + float(conf.get("repo_owner_username_boost", 0.3)), + ) + + results = repo_search.search(trans, search_term, page, page_size, boosts) + results["hostname"] = web.url_for("/", qualified=True) + return results + + +class UpdatesRequest(BaseModel): + name: Optional[str] = None + owner: Optional[str] = None + changeset_revision: str + hexlify: bool = True + + +def check_updates(app: ToolShedApp, request: UpdatesRequest) -> Union[str, Dict[str, Any]]: + name = request.name + owner = request.owner + changeset_revision = request.changeset_revision + hexlify_this = request.hexlify + repository = get_repository_by_name_and_owner( + app, name, owner, eagerload_columns=[Repository.downloadable_revisions] + ) + if repository and repository.downloadable_revisions: + repository_metadata = get_repository_metadata_by_changeset_revision( + app, app.security.encode_id(repository.id), changeset_revision + ) + tool_shed_status_dict = {} + # Handle repository deprecation. + tool_shed_status_dict["repository_deprecated"] = str(repository.deprecated) + tip_revision = repository.downloadable_revisions[0] + # Handle latest installable revision. + if changeset_revision == tip_revision: + tool_shed_status_dict["latest_installable_revision"] = "True" + else: + next_installable_revision = get_next_downloadable_changeset_revision(app, repository, changeset_revision) + if repository_metadata is None: + if next_installable_revision and next_installable_revision != changeset_revision: + tool_shed_status_dict["latest_installable_revision"] = "True" + else: + tool_shed_status_dict["latest_installable_revision"] = "False" + else: + if next_installable_revision and next_installable_revision != changeset_revision: + tool_shed_status_dict["latest_installable_revision"] = "False" + else: + tool_shed_status_dict["latest_installable_revision"] = "True" + # Handle revision updates. + if changeset_revision == tip_revision: + tool_shed_status_dict["revision_update"] = "False" + else: + if repository_metadata is None: + tool_shed_status_dict["revision_update"] = "True" + else: + tool_shed_status_dict["revision_update"] = "False" + # Handle revision upgrades. + metadata_revisions = [revision[1] for revision in get_metadata_revisions(app, repository)] + num_metadata_revisions = len(metadata_revisions) + for index, metadata_revision in enumerate(metadata_revisions): + if index == num_metadata_revisions: + tool_shed_status_dict["revision_upgrade"] = "False" + break + if metadata_revision == changeset_revision: + if num_metadata_revisions - index > 1: + tool_shed_status_dict["revision_upgrade"] = "True" + else: + tool_shed_status_dict["revision_upgrade"] = "False" + break + return ( + encoding_util.tool_shed_encode(tool_shed_status_dict) if hexlify_this else json.dumps(tool_shed_status_dict) + ) + return encoding_util.tool_shed_encode({}) if hexlify_this else json.dumps({}) + + +def guid_to_repository(app: ToolShedApp, tool_id: str) -> "Repository": + # tool_id = remove_protocol_and_user_from_clone_url(tool_id) + shed, _, owner, name, rest = tool_id.split("/", 5) + clause_list = [ + and_( + app.model.Repository.table.c.deprecated == false(), + app.model.Repository.table.c.deleted == false(), + app.model.Repository.table.c.name == name, + app.model.User.table.c.username == owner, + app.model.Repository.table.c.user_id == app.model.User.table.c.id, + ) + ] + repository = app.model.context.query(app.model.Repository).filter(*clause_list).first() + return repository + + +def index_tool_ids(app: ToolShedApp, tool_ids: List[str]) -> Dict[str, Any]: + repository_found = [] + all_metadata = dict() + for tool_id in tool_ids: + repository = guid_to_repository(app, tool_id) + owner = repository.user.username + name = repository.name + clause_list = [ + and_( + app.model.Repository.table.c.deprecated == false(), + app.model.Repository.table.c.deleted == false(), + app.model.Repository.table.c.name == name, + app.model.User.table.c.username == owner, + app.model.Repository.table.c.user_id == app.model.User.table.c.id, + ) + ] + repository = app.model.context.current.sa_session.query(app.model.Repository).filter(*clause_list).first() + if not repository: + log.warning(f"Repository {owner}/{name} does not exist, skipping") + continue + for changeset, changehash in repository.installable_revisions(app): + metadata = get_current_repository_metadata_for_changeset_revision(app, repository, changehash) + tools: Optional[List[Dict[str, Any]]] = metadata.metadata.get("tools") + if not tools: + log.warning(f"Repository {owner}/{name}/{changehash} does not contain valid tools, skipping") + continue + for tool_metadata in tools: + if tool_metadata["guid"] in tool_ids: + repository_found.append("%d:%s" % (int(changeset), changehash)) + metadata = get_current_repository_metadata_for_changeset_revision(app, repository, changehash) + if metadata is None: + continue + metadata_dict = metadata.to_dict( + value_mapper={"id": app.security.encode_id, "repository_id": app.security.encode_id} + ) + metadata_dict["repository"] = repository.to_dict(value_mapper={"id": app.security.encode_id}) + if metadata.has_repository_dependencies: + metadata_dict["repository_dependencies"] = get_all_dependencies( + app, metadata, processed_dependency_links=[] + ) + else: + metadata_dict["repository_dependencies"] = [] + if metadata.includes_tool_dependencies: + metadata_dict["tool_dependencies"] = repository.get_tool_dependencies(app, changehash) + else: + metadata_dict["tool_dependencies"] = {} + if metadata.includes_tools: + metadata_dict["tools"] = metadata.metadata["tools"] + all_metadata[f"{int(changeset)}:{changehash}"] = metadata_dict + if repository_found: + all_metadata["current_changeset"] = repository_found[0] + # all_metadata[ 'found_changesets' ] = repository_found + return all_metadata + else: + return {} + + +def index_repositories(app: ToolShedApp, name: Optional[str], owner: Optional[str], deleted: bool): + clause_list = [ + and_( + app.model.Repository.table.c.deprecated == false(), + app.model.Repository.table.c.deleted == deleted, + ) + ] + if owner is not None: + clause_list.append( + and_( + app.model.User.table.c.username == owner, + app.model.Repository.table.c.user_id == app.model.User.table.c.id, + ) + ) + if name is not None: + clause_list.append(app.model.Repository.table.c.name == name) + repositories = [] + for repository in ( + app.model.context.query(app.model.Repository).filter(*clause_list).order_by(app.model.Repository.table.c.name) + ): + repositories.append(repository) + return repositories + + +def can_manage_repo(trans: ProvidesUserContext, repository: Repository) -> bool: + security_agent = trans.app.security_agent + return trans.user_is_admin or security_agent.user_can_administer_repository(trans.user, repository) + + +def get_install_info(app: ToolShedApp, name, owner, changeset_revision) -> LegacyInstallInfoTuple: + value_mapper = get_value_mapper(app) + # Example URL: + # http:///api/repositories/get_repository_revision_install_info?name=&owner=&changeset_revision= + if name and owner and changeset_revision: + # Get the repository information. + repository = get_repository_by_name_and_owner( + app, name, owner, eagerload_columns=[Repository.downloadable_revisions] + ) + if repository is None: + log.debug(f"Cannot locate repository {name} owned by {owner}") + return {}, {}, {} + encoded_repository_id = app.security.encode_id(repository.id) + repository_dict: dict = repository.to_dict(view="element", value_mapper=value_mapper) + repository_dict["url"] = web.url_for(controller="repositories", action="show", id=encoded_repository_id) + # Get the repository_metadata information. + repository_metadata = get_repository_metadata_by_changeset_revision( + app, encoded_repository_id, changeset_revision + ) + if repository_metadata is None: + # The changeset_revision column in the repository_metadata table has been updated with a new + # value value, so find the changeset_revision to which we need to update. + new_changeset_revision = get_next_downloadable_changeset_revision(app, repository, changeset_revision) + repository_metadata = get_repository_metadata_by_changeset_revision( + app, encoded_repository_id, new_changeset_revision + ) + changeset_revision = new_changeset_revision + if repository_metadata is not None: + encoded_repository_metadata_id = app.security.encode_id(repository_metadata.id) + repository_metadata_dict: RepositoryMetadataInstallInfoDict = cast( + RepositoryMetadataInstallInfoDict, + repository_metadata.to_dict(view="collection", value_mapper=value_mapper), + ) + repository_metadata_dict["url"] = web.url_for( + controller="repository_revisions", action="show", id=encoded_repository_metadata_id + ) + if "tools" in repository_metadata.metadata: + repository_metadata_dict["valid_tools"] = repository_metadata.metadata["tools"] + # Get the repo_info_dict for installing the repository. + repo_info_dict: ExtraRepoInfo + ( + repo_info_dict, + includes_tools, + includes_tool_dependencies, + includes_tools_for_display_in_tool_panel, + has_repository_dependencies, + has_repository_dependencies_only_if_compiling_contained_td, + ) = get_repo_info_dict(app, None, encoded_repository_id, changeset_revision) + return repository_dict, repository_metadata_dict, repo_info_dict + else: + log.debug( + "Unable to locate repository_metadata record for repository id %s and changeset_revision %s", + repository.id, + changeset_revision, + ) + return repository_dict, {}, {} + else: + debug_msg = "Error in the Tool Shed repositories API in get_repository_revision_install_info: " + debug_msg += f"Invalid name {name} or owner {owner} or changeset_revision {changeset_revision} received." + log.debug(debug_msg) + return {}, {}, {} + + +def get_value_mapper(app: ToolShedApp) -> Dict[str, Callable]: + value_mapper = { + "id": app.security.encode_id, + "repository_id": app.security.encode_id, + "user_id": app.security.encode_id, + } + return value_mapper + + +def get_ordered_installable_revisions( + app: ToolShedApp, name: Optional[str], owner: Optional[str], tsr_id: Optional[str] +) -> List[str]: + eagerload_columns = [Repository.downloadable_revisions] + if None not in [name, owner]: + # Get the repository information. + repository = get_repository_by_name_and_owner(app, name, owner, eagerload_columns=eagerload_columns) + if repository is None: + raise ObjectNotFound(f"No repository named {name} found with owner {owner}") + elif tsr_id is not None: + repository = get_repository_in_tool_shed(app, tsr_id, eagerload_columns=eagerload_columns) + else: + error_message = "Error in the Tool Shed repositories API in get_ordered_installable_revisions: " + error_message += "invalid parameters received." + log.debug(error_message) + return [] + return [revision[1] for revision in repository.installable_revisions(app, sort_revisions=True)] + + +def get_repository_metadata_dict(app: ToolShedApp, id: str, recursive: bool, downloadable_only: bool) -> Dict[str, Any]: + all_metadata = {} + repository = get_repository_in_tool_shed(app, id, eagerload_columns=[Repository.downloadable_revisions]) + for changeset, changehash in get_metadata_revisions( + app, repository, sort_revisions=True, downloadable=downloadable_only + ): + metadata = get_current_repository_metadata_for_changeset_revision(app, repository, changehash) + if metadata is None: + continue + metadata_dict = metadata.to_dict( + value_mapper={"id": app.security.encode_id, "repository_id": app.security.encode_id} + ) + metadata_dict["repository"] = repository.to_dict(value_mapper={"id": app.security.encode_id}) + if metadata.has_repository_dependencies and recursive: + metadata_dict["repository_dependencies"] = get_all_dependencies( + app, metadata, processed_dependency_links=[] + ) + else: + metadata_dict["repository_dependencies"] = [] + if metadata.includes_tools: + metadata_dict["tools"] = metadata.metadata["tools"] + all_metadata[f"{int(changeset)}:{changehash}"] = metadata_dict + return all_metadata + + +def readmes(app: ToolShedApp, repository: Repository, changeset_revision: str) -> dict: + encoded_repository_id = app.security.encode_id(repository.id) + repository_metadata = get_repository_metadata_by_changeset_revision(app, encoded_repository_id, changeset_revision) + if repository_metadata: + metadata = repository_metadata.metadata + if metadata: + return build_readme_files_dict(app, repository, changeset_revision, repository_metadata.metadata) + return {} + + +def reset_metadata_on_repository(trans: ProvidesUserContext, repository_id) -> ResetMetadataOnRepositoryResponse: + app: ToolShedApp = trans.app + + def handle_repository(trans, start_time, repository): + results = dict(start_time=start_time, repository_status=[]) + try: + rmm = repository_metadata_manager.RepositoryMetadataManager( + app=app, + user=trans.user, + repository=repository, + resetting_all_metadata_on_repository=True, + updating_installed_repository=False, + persist=False, + ) + rmm.reset_all_metadata_on_repository_in_tool_shed() + rmm_invalid_file_tups = rmm.get_invalid_file_tups() + if rmm_invalid_file_tups: + message = generate_message_for_invalid_tools( + app, rmm_invalid_file_tups, repository, None, as_html=False + ) + results["status"] = "warning" + else: + message = ( + f"Successfully reset metadata on repository {repository.name} owned by {repository.user.username}" + ) + results["status"] = "ok" + except Exception as e: + message = ( + f"Error resetting metadata on repository {repository.name} owned by {repository.user.username}: {e}" + ) + results["status"] = "error" + status = f"{repository.name} : {message}" + results["repository_status"].append(status) + return results + + if repository_id is not None: + repository = get_repository_in_tool_shed(app, repository_id) + start_time = strftime("%Y-%m-%d %H:%M:%S") + log.debug(f"{start_time}...resetting metadata on repository {repository.name}") + results = handle_repository(trans, start_time, repository) + stop_time = strftime("%Y-%m-%d %H:%M:%S") + results["stop_time"] = stop_time + return ResetMetadataOnRepositoryResponse(**results) + + +def create_repository(trans: ProvidesUserContext, request: CreateRepositoryRequest) -> Repository: + app: ToolShedApp = trans.app + user = trans.user + assert user + category_ids = listify(request.category_ids) + name = request.name + invalid_message = validate_repository_name(app, name, user) + if invalid_message: + raise RequestParameterInvalidException(invalid_message) + + repo, _ = low_level_create_repository( + app=app, + name=name, + type=request.type_, + description=request.synopsis, + long_description=request.description, + user_id=user.id, + category_ids=category_ids, + remote_repository_url=request.remote_repository_url, + homepage_url=request.homepage_url, + ) + return repo + + +def to_element_dict(app, repository: Repository, include_categories: bool = False) -> Dict[str, Any]: + value_mapper = get_value_mapper(app) + repository_dict = repository.to_dict(view="element", value_mapper=value_mapper) + if include_categories: + repository_dict["category_ids"] = [app.security.encode_id(x.category.id) for x in repository.categories] + return repository_dict + + +def repositories_by_category( + app: ToolShedApp, + category_id: str, + page: Optional[int] = None, + sort_key: str = "name", + sort_order: str = "asc", + installable: bool = True, +): + category = get_category(app, category_id) + category_dict: Dict[str, Any] + if category is None: + category_dict = dict(message=f"Unable to locate category record for id {str(id)}.", status="error") + return category_dict + category_dict = category.to_dict(view="element", value_mapper=category_value_mapper(app)) + category_dict["repository_count"] = count_repositories_in_category(app, category_id) + repositories = get_repositories_by_category( + app, category.id, installable=installable, sort_order=sort_order, sort_key=sort_key, page=page + ) + category_dict["repositories"] = repositories + return category_dict + + +def to_model(app, repository: Repository) -> SchemaRepository: + return SchemaRepository(**to_element_dict(app, repository)) + + def upload_tar_and_set_metadata( app: ToolShedApp, host: str, diff --git a/lib/tool_shed/managers/tools.py b/lib/tool_shed/managers/tools.py new file mode 100644 index 00000000000..84dfc9c6d0f --- /dev/null +++ b/lib/tool_shed/managers/tools.py @@ -0,0 +1,44 @@ +from collections import namedtuple + +from galaxy import exceptions +from tool_shed.context import SessionRequestContext +from tool_shed.webapp.search.tool_search import ToolSearch + + +def search(trans: SessionRequestContext, q: str, page: int = 1, page_size: int = 10) -> dict: + """ + Perform the search over TS tools index. + Note that search works over the Whoosh index which you have + to pre-create with scripts/tool_shed/build_ts_whoosh_index.sh manually. + Also TS config option toolshed_search_on has to be True and + whoosh_index_dir has to be specified. + """ + app = trans.app + conf = app.config + if not conf.toolshed_search_on: + raise exceptions.ConfigDoesNotAllowException( + "Searching the TS through the API is turned off for this instance." + ) + if not conf.whoosh_index_dir: + raise exceptions.ConfigDoesNotAllowException( + "There is no directory for the search index specified. Please contact the administrator." + ) + search_term = q.strip() + if len(search_term) < 1: + raise exceptions.RequestParameterInvalidException("The search term has to be at least one character long.") + + tool_search = ToolSearch() + + Boosts = namedtuple( + "Boosts", ["tool_name_boost", "tool_description_boost", "tool_help_boost", "tool_repo_owner_username_boost"] + ) + boosts = Boosts( + float(conf.get("tool_name_boost", 1.2)), + float(conf.get("tool_description_boost", 0.6)), + float(conf.get("tool_help_boost", 0.4)), + float(conf.get("tool_repo_owner_username_boost", 0.3)), + ) + + results = tool_search.search(trans.app, search_term, page, page_size, boosts) + results["hostname"] = trans.url_builder("/", qualified=True) + return results diff --git a/lib/tool_shed/managers/users.py b/lib/tool_shed/managers/users.py index a6cc02a1dc0..055aaf1945d 100644 --- a/lib/tool_shed/managers/users.py +++ b/lib/tool_shed/managers/users.py @@ -1,8 +1,35 @@ +from typing import List + +from galaxy.exceptions import RequestParameterInvalidException +from galaxy.model.base import transaction +from galaxy.security.validate_user_input import ( + validate_email, + validate_password, + validate_publicname, +) +from tool_shed.context import ProvidesUserContext from tool_shed.structured_app import ToolShedApp from tool_shed.webapp.model import User +from tool_shed_client.schema import ( + CreateUserRequest, + User as ApiUser, +) + + +def index(app: ToolShedApp, deleted: bool) -> List[ApiUser]: + users: List[ApiUser] = [] + for user in ( + app.model.context.query(app.model.User) + .filter(app.model.User.table.c.deleted == deleted) + .order_by(app.model.User.table.c.username) + ): + users.append(get_api_user(app, user)) + return users def create_user(app: ToolShedApp, email: str, username: str, password: str) -> User: + if username == "repos": + raise RequestParameterInvalidException("Cannot create a tool shed user with the username repos") sa_session = app.model.context user = User(email=email) user.set_password_cleartext(password) @@ -13,6 +40,38 @@ def create_user(app: ToolShedApp, email: str, username: str, password: str) -> U # else: # user.active = True # Activation is off, every new user is active by default. sa_session.add(user) - sa_session.flush() + with transaction(sa_session): + sa_session.commit() app.security_agent.create_private_user_role(user) return user + + +def api_create_user(trans: ProvidesUserContext, request: CreateUserRequest) -> ApiUser: + app = trans.app + message = _validate( + trans, email=request.email, password=request.password, confirm=request.password, username=request.username + ) + if message: + raise RequestParameterInvalidException(message) + user = create_user(app, request.email, request.username, request.password) + return get_api_user(app, user) + + +def get_api_user(app: ToolShedApp, user: User) -> ApiUser: + return ApiUser( + id=app.security.encode_id(user.id), + username=user.username, + ) + + +def _validate(trans: ProvidesUserContext, email: str, password: str, confirm: str, username: str) -> str: + if username in ["repos"]: + return f"The term '{username}' is a reserved word in the Tool Shed, so it cannot be used as a public user name." + message = "\n".join( + ( + validate_email(trans, email), + validate_password(trans, password, confirm), + validate_publicname(trans, username), + ) + ).rstrip() + return message diff --git a/lib/tool_shed/test/functional/api_notes.md b/lib/tool_shed/test/functional/api_notes.md index 0e225c08478..dc207aa658f 100644 --- a/lib/tool_shed/test/functional/api_notes.md +++ b/lib/tool_shed/test/functional/api_notes.md @@ -41,4 +41,10 @@ for deletion instead of writing new tests and modernizing the API. | repositories/{repository_id}/changeset_revision | NO | YES | NO | YES | NO | | | POST repositories | NO | YES | NO | YES | NO | | | GET repositories (without search query) | ? | ? |? | True | True | | +| GET /repositories/updates/ | YES | NO | NO | NO | YES | | +Research if searching by tool_ids is used with the repository index API. + +Added in: +- https://github.com/galaxyproject/galaxy/pull/3626/files +- Likely no longer used? diff --git a/lib/tool_shed/test/functional/test_shed_repositories.py b/lib/tool_shed/test/functional/test_shed_repositories.py index f2fba654d2a..8fefcddea00 100644 --- a/lib/tool_shed/test/functional/test_shed_repositories.py +++ b/lib/tool_shed/test/functional/test_shed_repositories.py @@ -52,6 +52,8 @@ class TestShedRepositoriesApi(ShedApiTestCase): assert not only_revision.malicious def test_index_simple(self): + # Logic and typing is pretty different if given a tool id to search for - this should + # be tested or dropped in v2. populator = self.populator repo = populator.setup_column_maker_repo(prefix="repoforindex") repository_id = repo.id diff --git a/lib/tool_shed/webapp/api/categories.py b/lib/tool_shed/webapp/api/categories.py index c0ca91ef870..15183b2a62a 100644 --- a/lib/tool_shed/webapp/api/categories.py +++ b/lib/tool_shed/webapp/api/categories.py @@ -1,13 +1,13 @@ import logging from typing import ( Any, - Callable, Dict, + List, ) import tool_shed.util.shed_util_common as suc +import tool_shed_client.schema from galaxy import ( - exceptions, util, web, ) @@ -17,18 +17,22 @@ from galaxy.web import ( expose_api_anonymous_and_sessionless, require_admin, ) -from galaxy.webapps.base.controller import BaseAPIController -from tool_shed.util import repository_util +from galaxy.webapps.galaxy.api import depends +from tool_shed.managers.categories import ( + CategoryManager, + get_value_mapper, +) +from tool_shed.managers.repositories import repositories_by_category +from tool_shed.webapp.model import Category +from . import BaseShedAPIController log = logging.getLogger(__name__) -class CategoriesController(BaseAPIController): +class CategoriesController(BaseShedAPIController): """RESTful controller for interactions with categories in the Tool Shed.""" - def __get_value_mapper(self, trans) -> Dict[str, Callable]: - value_mapper = {"id": trans.security.encode_id} - return value_mapper + category_manager: CategoryManager = depends(CategoryManager) @expose_api @require_admin @@ -46,27 +50,16 @@ class CategoriesController(BaseAPIController): Content-Disposition: form-data; name="description" Category_Description """ category_dict = dict(message="", status="ok") - name = payload.get("name", "") - if name: - description = payload.get("description", "") - if not description: - # Default the description to the name. - description = name - if suc.get_category_by_name(self.app, name): - raise exceptions.Conflict("A category with that name already exists.") - else: - # Create the category - category = self.app.model.Category(name=name, description=description) - trans.sa_session.add(category) - with transaction(trans.sa_session): - trans.sa_session.commit() - category_dict = category.to_dict(view="element", value_mapper=self.__get_value_mapper(trans)) - category_dict["message"] = f"Category '{str(category.name)}' has been created" - category_dict["url"] = web.url_for( - controller="categories", action="show", id=trans.security.encode_id(category.id) - ) - else: - raise exceptions.RequestParameterMissingException('Missing required parameter "name".') + request = tool_shed_client.schema.CreateCategoryRequest( + name=payload.get("name"), + description=payload.get("description", ""), + ) + category: Category = self.category_manager.create(trans, request) + category_dict = category.to_dict(view="element", value_mapper=get_value_mapper(trans.app)) + category_dict["message"] = f"Category '{str(category.name)}' has been created" + category_dict["url"] = web.url_for( + controller="categories", action="show", id=trans.security.encode_id(category.id) + ) return category_dict @expose_api_anonymous_and_sessionless @@ -86,24 +79,19 @@ class CategoriesController(BaseAPIController): sort_key = kwd.get("sort_key", "name") sort_order = kwd.get("sort_order", "asc") page = kwd.get("page", None) - category = suc.get_category(self.app, category_id) - category_dict: Dict[str, Any] - if category is None: - category_dict = dict(message=f"Unable to locate category record for id {str(id)}.", status="error") - return category_dict - category_dict = category.to_dict(view="element", value_mapper=self.__get_value_mapper(trans)) - category_dict["repository_count"] = suc.count_repositories_in_category(self.app, category_id) - category_dict["url"] = web.url_for( - controller="categories", action="show", id=trans.security.encode_id(category.id) + category_dict = repositories_by_category( + self.app, + category_id, + page=page, + sort_key=sort_key, + sort_order=sort_order, + installable=installable, ) - repositories = repository_util.get_repositories_by_category( - self.app, category.id, installable=installable, sort_order=sort_order, sort_key=sort_key, page=page - ) - category_dict["repositories"] = repositories + category_dict["url"] = web.url_for(controller="categories", action="show", id=category_dict["id"]) return category_dict @expose_api_anonymous_and_sessionless - def index(self, trans, deleted=False, **kwd): + def index(self, trans, deleted=False, **kwd) -> List[Dict[str, Any]]: """ GET /api/categories Return a list of dictionaries that contain information about each Category. @@ -112,24 +100,8 @@ class CategoriesController(BaseAPIController): Example: GET localhost:9009/api/categories """ - category_dicts = [] deleted = util.asbool(deleted) - if deleted and not trans.user_is_admin: - raise exceptions.AdminRequiredException("Only administrators can query deleted categories.") - for category in ( - trans.sa_session.query(self.app.model.Category) - .filter(self.app.model.Category.table.c.deleted == deleted) - .order_by(self.app.model.Category.table.c.name) - ): - category_dict = category.to_dict(view="collection", value_mapper=self.__get_value_mapper(trans)) - category_dict["url"] = web.url_for( - controller="categories", action="show", id=trans.security.encode_id(category.id) - ) - category_dict[ - "repositories" - ] = self.app.repository_registry.viewable_repositories_and_suites_by_category.get(category.name, 0) - category_dicts.append(category_dict) - return category_dicts + return self.category_manager.index(trans, deleted) @expose_api_anonymous_and_sessionless def show(self, trans, id, **kwd): @@ -145,7 +117,7 @@ class CategoriesController(BaseAPIController): if category is None: category_dict = dict(message=f"Unable to locate category record for id {str(id)}.", status="error") return category_dict - category_dict = category.to_dict(view="element", value_mapper=self.__get_value_mapper(trans)) + category_dict = category.to_dict(view="element", value_mapper=get_value_mapper(trans.app)) category_dict["url"] = web.url_for( controller="categories", action="show", id=trans.security.encode_id(category.id) ) diff --git a/lib/tool_shed/webapp/api/configuration.py b/lib/tool_shed/webapp/api/configuration.py index fb449597607..920baeb670b 100644 --- a/lib/tool_shed/webapp/api/configuration.py +++ b/lib/tool_shed/webapp/api/configuration.py @@ -5,15 +5,12 @@ capabilities and configuration settings. import logging from galaxy.web import expose_api_anonymous_and_sessionless -from galaxy.webapps.base.controller import BaseAPIController +from . import BaseShedAPIController log = logging.getLogger(__name__) -class ConfigurationController(BaseAPIController): - def __init__(self, app): - super().__init__(app) - +class ConfigurationController(BaseShedAPIController): @expose_api_anonymous_and_sessionless def version(self, trans, **kwds): """ diff --git a/lib/tool_shed/webapp/api/groups.py b/lib/tool_shed/webapp/api/groups.py index 05f46673fc8..72329e20b78 100644 --- a/lib/tool_shed/webapp/api/groups.py +++ b/lib/tool_shed/webapp/api/groups.py @@ -22,16 +22,17 @@ from galaxy.web import ( expose_api_anonymous_and_sessionless, require_admin, ) -from galaxy.webapps.base.controller import BaseAPIController from tool_shed.managers import groups +from tool_shed.structured_app import ToolShedApp +from . import BaseShedAPIController log = logging.getLogger(__name__) -class GroupsController(BaseAPIController): +class GroupsController(BaseShedAPIController): """RESTful controller for interactions with groups in the Tool Shed.""" - def __init__(self, app): + def __init__(self, app: ToolShedApp): super().__init__(app) self.group_manager = groups.GroupManager() diff --git a/lib/tool_shed/webapp/api/repositories.py b/lib/tool_shed/webapp/api/repositories.py index 28094114fca..403d0418219 100644 --- a/lib/tool_shed/webapp/api/repositories.py +++ b/lib/tool_shed/webapp/api/repositories.py @@ -1,7 +1,6 @@ import json import logging import os -from collections import namedtuple from io import StringIO from time import strftime from typing import ( @@ -9,10 +8,6 @@ from typing import ( Dict, ) -from sqlalchemy import ( - and_, - false, -) from webob.compat import cgi_FieldStorage from galaxy import ( @@ -21,7 +16,6 @@ from galaxy import ( ) from galaxy.exceptions import ( ActionInputError, - ConfigDoesNotAllowException, InsufficientPermissionsException, MessageException, ObjectNotFound, @@ -33,26 +27,41 @@ from galaxy.web import ( expose_api_anonymous_and_sessionless, expose_api_raw_anonymous_and_sessionless, ) -from galaxy.webapps.base.controller import ( - BaseAPIController, - HTTPBadRequest, +from galaxy.webapps.base.controller import HTTPBadRequest +from tool_shed.managers.repositories import ( + can_update_repo, + check_updates, + create_repository, + get_install_info, + get_ordered_installable_revisions, + get_repository_metadata_dict, + get_value_mapper, + index_repositories, + index_tool_ids, + reset_metadata_on_repository, + search, + to_element_dict, + UpdatesRequest, + upload_tar_and_set_metadata, ) -from tool_shed.managers.repositories import upload_tar_and_set_metadata from tool_shed.metadata import repository_metadata_manager from tool_shed.repository_types import util as rt_util from tool_shed.util import ( - encoding_util, metadata_util, repository_util, tool_util, ) from tool_shed.webapp import model -from tool_shed.webapp.search.repo_search import RepoSearch +from tool_shed_client.schema import ( + CreateRepositoryRequest, + LegacyInstallInfoTuple, +) +from . import BaseShedAPIController log = logging.getLogger(__name__) -class RepositoriesController(BaseAPIController): +class RepositoriesController(BaseShedAPIController): """RESTful controller for interactions with repositories in the Tool Shed.""" @web.legacy_expose_api @@ -115,28 +124,12 @@ class RepositoriesController(BaseAPIController): if owner is None: owner = kwd.get("owner", None) tsr_id = kwd.get("tsr_id", None) - eagerload_columns = [model.Repository.downloadable_revisions] - if None not in [name, owner]: - # Get the repository information. - repository = repository_util.get_repository_by_name_and_owner( - self.app, name, owner, eagerload_columns=eagerload_columns - ) - if repository is None: - trans.response.status = 404 - return {"status": "error", "message": f"No repository named {name} found with owner {owner}"} - elif tsr_id is not None: - repository = repository_util.get_repository_in_tool_shed( - self.app, tsr_id, eagerload_columns=eagerload_columns - ) - else: - error_message = "Error in the Tool Shed repositories API in get_ordered_installable_revisions: " - error_message += "invalid parameters received." - log.debug(error_message) - return [] - return [revision[1] for revision in repository.installable_revisions(self.app, sort_revisions=True)] + return get_ordered_installable_revisions(self.app, name, owner, tsr_id) @web.legacy_expose_api_anonymous - def get_repository_revision_install_info(self, trans, name, owner, changeset_revision, **kwd): + def get_repository_revision_install_info( + self, trans, name, owner, changeset_revision, **kwd + ) -> LegacyInstallInfoTuple: """ GET /api/repositories/get_repository_revision_install_info @@ -209,65 +202,7 @@ class RepositoriesController(BaseAPIController): } """ - # Example URL: - # http:///api/repositories/get_repository_revision_install_info?name=&owner=&changeset_revision= - if name and owner and changeset_revision: - # Get the repository information. - repository = repository_util.get_repository_by_name_and_owner( - self.app, name, owner, eagerload_columns=[model.Repository.downloadable_revisions] - ) - if repository is None: - log.debug(f"Cannot locate repository {name} owned by {owner}") - return {}, {}, {} - encoded_repository_id = trans.security.encode_id(repository.id) - repository_dict = repository.to_dict(view="element", value_mapper=self.__get_value_mapper(trans)) - repository_dict["url"] = web.url_for(controller="repositories", action="show", id=encoded_repository_id) - # Get the repository_metadata information. - repository_metadata = metadata_util.get_repository_metadata_by_changeset_revision( - self.app, encoded_repository_id, changeset_revision - ) - if repository_metadata is None: - # The changeset_revision column in the repository_metadata table has been updated with a new - # value value, so find the changeset_revision to which we need to update. - new_changeset_revision = metadata_util.get_next_downloadable_changeset_revision( - self.app, repository, changeset_revision - ) - repository_metadata = metadata_util.get_repository_metadata_by_changeset_revision( - self.app, encoded_repository_id, new_changeset_revision - ) - changeset_revision = new_changeset_revision - if repository_metadata is not None: - encoded_repository_metadata_id = trans.security.encode_id(repository_metadata.id) - repository_metadata_dict = repository_metadata.to_dict( - view="collection", value_mapper=self.__get_value_mapper(trans) - ) - repository_metadata_dict["url"] = web.url_for( - controller="repository_revisions", action="show", id=encoded_repository_metadata_id - ) - if "tools" in repository_metadata.metadata: - repository_metadata_dict["valid_tools"] = repository_metadata.metadata["tools"] - # Get the repo_info_dict for installing the repository. - ( - repo_info_dict, - includes_tools, - includes_tool_dependencies, - includes_tools_for_display_in_tool_panel, - has_repository_dependencies, - has_repository_dependencies_only_if_compiling_contained_td, - ) = repository_util.get_repo_info_dict(self.app, trans.user, encoded_repository_id, changeset_revision) - return repository_dict, repository_metadata_dict, repo_info_dict - else: - log.debug( - "Unable to locate repository_metadata record for repository id %s and changeset_revision %s", - repository.id, - changeset_revision, - ) - return repository_dict, {}, {} - else: - debug_msg = "Error in the Tool Shed repositories API in get_repository_revision_install_info: " - debug_msg += f"Invalid name {name} or owner {owner} or changeset_revision {changeset_revision} received." - log.debug(debug_msg) - return {}, {}, {} + return get_install_info(self.app, name, owner, changeset_revision) @web.legacy_expose_api_anonymous def get_installable_revisions(self, trans, **kwd): @@ -292,12 +227,7 @@ class RepositoriesController(BaseAPIController): return repository.installable_revisions(self.app) def __get_value_mapper(self, trans) -> Dict[str, Callable]: - value_mapper = { - "id": trans.security.encode_id, - "repository_id": trans.security.encode_id, - "user_id": trans.security.encode_id, - } - return value_mapper + return get_value_mapper(self.app) @expose_api_raw_anonymous_and_sessionless def index(self, trans, deleted=False, owner=None, name=None, **kwd): @@ -352,7 +282,7 @@ class RepositoriesController(BaseAPIController): raise RequestParameterInvalidException('The "page" and "page_size" parameters have to be integers.') return_jsonp = util.asbool(kwd.get("jsonp", False)) callback = kwd.get("callback", "callback") - search_results = self._search(trans, q, page, page_size) + search_results = search(trans, q, page, page_size) if return_jsonp: response = str(f"{callback}({json.dumps(search_results)});") else: @@ -361,134 +291,18 @@ class RepositoriesController(BaseAPIController): tool_ids = kwd.get("tool_ids", None) if tool_ids is not None: tool_ids = util.listify(tool_ids) - repository_found = [] - all_metadata = dict() - for tool_id in tool_ids: - # A valid GUID looks like toolshed.g2.bx.psu.edu/repos/bgruening/deeptools/deeptools_computeMatrix/1.1.0 - shed, _, owner, name, tool, version = tool_id.split("/") - clause_list = [ - and_( - self.app.model.Repository.table.c.deprecated == false(), - self.app.model.Repository.table.c.deleted == false(), - self.app.model.Repository.table.c.name == name, - self.app.model.User.table.c.username == owner, - self.app.model.Repository.table.c.user_id == self.app.model.User.table.c.id, - ) + response = index_tool_ids(self.app, tool_ids) + return json.dumps(response) + else: + repositories = index_repositories(self.app, name, owner, deleted) + repository_dicts = [] + for repository in repositories: + repository_dict = repository.to_dict(view="collection", value_mapper=self.__get_value_mapper(trans)) + repository_dict["category_ids"] = [ + trans.security.encode_id(x.category.id) for x in repository.categories ] - repository = trans.sa_session.query(self.app.model.Repository).filter(*clause_list).first() - if not repository: - log.warning(f"Repository {owner}/{name} does not exist, skipping") - continue - for changeset, changehash in repository.installable_revisions(self.app): - metadata = metadata_util.get_current_repository_metadata_for_changeset_revision( - self.app, repository, changehash - ) - tools = metadata.metadata.get("tools") - if not tools: - log.warning(f"Repository {owner}/{name}/{changehash} does not contain valid tools, skipping") - continue - for tool in tools: - if tool["guid"] in tool_ids: - repository_found.append("%d:%s" % (int(changeset), changehash)) - metadata = metadata_util.get_current_repository_metadata_for_changeset_revision( - self.app, repository, changehash - ) - if metadata is None: - continue - metadata_dict = metadata.to_dict( - value_mapper={"id": self.app.security.encode_id, "repository_id": self.app.security.encode_id} - ) - metadata_dict["repository"] = repository.to_dict(value_mapper={"id": self.app.security.encode_id}) - if metadata.has_repository_dependencies: - metadata_dict["repository_dependencies"] = metadata_util.get_all_dependencies( - self.app, metadata, processed_dependency_links=[] - ) - else: - metadata_dict["repository_dependencies"] = [] - if metadata.includes_tool_dependencies: - metadata_dict["tool_dependencies"] = repository.get_tool_dependencies(self.app, changehash) - else: - metadata_dict["tool_dependencies"] = {} - if metadata.includes_tools: - metadata_dict["tools"] = metadata.metadata["tools"] - all_metadata[f"{int(changeset)}:{changehash}"] = metadata_dict - if repository_found: - all_metadata["current_changeset"] = repository_found[0] - # all_metadata[ 'found_changesets' ] = repository_found - return json.dumps(all_metadata) - return "{}" - - clause_list = [ - and_( - self.app.model.Repository.table.c.deprecated == false(), - self.app.model.Repository.table.c.deleted == deleted, - ) - ] - if owner is not None: - clause_list.append( - and_( - self.app.model.User.table.c.username == owner, - self.app.model.Repository.table.c.user_id == self.app.model.User.table.c.id, - ) - ) - if name is not None: - clause_list.append(self.app.model.Repository.table.c.name == name) - for repository in ( - trans.sa_session.query(self.app.model.Repository) - .filter(*clause_list) - .order_by(self.app.model.Repository.table.c.name) - ): - repository_dict = repository.to_dict(view="collection", value_mapper=self.__get_value_mapper(trans)) - repository_dict["category_ids"] = [trans.security.encode_id(x.category.id) for x in repository.categories] - repository_dicts.append(repository_dict) - return json.dumps(repository_dicts) - - def _search(self, trans, q, page=1, page_size=10): - """ - Perform the search over TS repositories. - Note that search works over the Whoosh index which you have - to pre-create with scripts/tool_shed/build_ts_whoosh_index.sh manually. - Also TS config option toolshed_search_on has to be True and - whoosh_index_dir has to be specified. - """ - conf = self.app.config - if not conf.toolshed_search_on: - raise ConfigDoesNotAllowException("Searching the TS through the API is turned off for this instance.") - if not conf.whoosh_index_dir: - raise ConfigDoesNotAllowException( - "There is no directory for the search index specified. Please contact the administrator." - ) - search_term = q.strip() - if len(search_term) < 1: - raise RequestParameterInvalidException("The search term has to be at least one character long.") - - repo_search = RepoSearch() - - Boosts = namedtuple( - "Boosts", - [ - "repo_name_boost", - "repo_description_boost", - "repo_long_description_boost", - "repo_homepage_url_boost", - "repo_remote_repository_url_boost", - "categories_boost", - "repo_owner_username_boost", - ], - ) - boosts = Boosts( - float(conf.get("repo_name_boost", 0.9)), - float(conf.get("repo_description_boost", 0.6)), - float(conf.get("repo_long_description_boost", 0.5)), - float(conf.get("repo_homepage_url_boost", 0.3)), - float(conf.get("repo_remote_repository_url_boost", 0.2)), - float(conf.get("categories_boost", 0.5)), - float(conf.get("repo_owner_username_boost", 0.3)), - ) - - results = repo_search.search(trans, search_term, page, page_size, boosts) - results["hostname"] = web.url_for("/", qualified=True) - return results + repository_dicts.append(repository_dict) + return json.dumps(repository_dicts) @web.legacy_expose_api def remove_repository_registry_entry(self, trans, payload, **kwd): @@ -654,46 +468,8 @@ class RepositoriesController(BaseAPIController): The following parameters must be included in the payload. :param repository_id: the encoded id of the repository on which metadata is to be reset. """ - - def handle_repository(trans, start_time, repository): - results = dict(start_time=start_time, repository_status=[]) - try: - rmm = repository_metadata_manager.RepositoryMetadataManager( - app=self.app, - user=trans.user, - repository=repository, - resetting_all_metadata_on_repository=True, - updating_installed_repository=False, - persist=False, - ) - rmm.reset_all_metadata_on_repository_in_tool_shed() - rmm_invalid_file_tups = rmm.get_invalid_file_tups() - if rmm_invalid_file_tups: - message = tool_util.generate_message_for_invalid_tools( - self.app, rmm_invalid_file_tups, repository, None, as_html=False - ) - results["status"] = "warning" - else: - message = f"Successfully reset metadata on repository {repository.name} owned by {repository.user.username}" - results["status"] = "ok" - except Exception as e: - message = ( - f"Error resetting metadata on repository {repository.name} owned by {repository.user.username}: {e}" - ) - results["status"] = "error" - status = f"{repository.name} : {message}" - results["repository_status"].append(status) - return results - repository_id = payload.get("repository_id", None) - if repository_id is not None: - repository = repository_util.get_repository_in_tool_shed(self.app, repository_id) - start_time = strftime("%Y-%m-%d %H:%M:%S") - log.debug(f"{start_time}...resetting metadata on repository {repository.name}") - results = handle_repository(trans, start_time, repository) - stop_time = strftime("%Y-%m-%d %H:%M:%S") - results["stop_time"] = stop_time - return results + return reset_metadata_on_repository(trans, repository_id).dict() @expose_api_anonymous_and_sessionless def show(self, trans, id, **kwd): @@ -743,63 +519,13 @@ class RepositoriesController(BaseAPIController): owner = kwd.get("owner", None) changeset_revision = kwd.get("changeset_revision", None) hexlify_this = util.asbool(kwd.get("hexlify", True)) - repository = repository_util.get_repository_by_name_and_owner( - trans.app, name, owner, eagerload_columns=[model.Repository.downloadable_revisions] + request = UpdatesRequest( + name=name, + owner=owner, + changeset_revision=changeset_revision, + hexlify=hexlify_this, ) - if repository and repository.downloadable_revisions: - repository_metadata = metadata_util.get_repository_metadata_by_changeset_revision( - trans.app, trans.security.encode_id(repository.id), changeset_revision - ) - tool_shed_status_dict = {} - # Handle repository deprecation. - tool_shed_status_dict["repository_deprecated"] = str(repository.deprecated) - tip_revision = repository.downloadable_revisions[0] - # Handle latest installable revision. - if changeset_revision == tip_revision: - tool_shed_status_dict["latest_installable_revision"] = "True" - else: - next_installable_revision = metadata_util.get_next_downloadable_changeset_revision( - trans.app, repository, changeset_revision - ) - if repository_metadata is None: - if next_installable_revision and next_installable_revision != changeset_revision: - tool_shed_status_dict["latest_installable_revision"] = "True" - else: - tool_shed_status_dict["latest_installable_revision"] = "False" - else: - if next_installable_revision and next_installable_revision != changeset_revision: - tool_shed_status_dict["latest_installable_revision"] = "False" - else: - tool_shed_status_dict["latest_installable_revision"] = "True" - # Handle revision updates. - if changeset_revision == tip_revision: - tool_shed_status_dict["revision_update"] = "False" - else: - if repository_metadata is None: - tool_shed_status_dict["revision_update"] = "True" - else: - tool_shed_status_dict["revision_update"] = "False" - # Handle revision upgrades. - metadata_revisions = [ - revision[1] for revision in metadata_util.get_metadata_revisions(trans.app, repository) - ] - num_metadata_revisions = len(metadata_revisions) - for index, metadata_revision in enumerate(metadata_revisions): - if index == num_metadata_revisions: - tool_shed_status_dict["revision_upgrade"] = "False" - break - if metadata_revision == changeset_revision: - if num_metadata_revisions - index > 1: - tool_shed_status_dict["revision_upgrade"] = "True" - else: - tool_shed_status_dict["revision_upgrade"] = "False" - break - return ( - encoding_util.tool_shed_encode(tool_shed_status_dict) - if hexlify_this - else json.dumps(tool_shed_status_dict) - ) - return encoding_util.tool_shed_encode({}) if hexlify_this else json.dumps({}) + return check_updates(trans.app, request) @expose_api_anonymous_and_sessionless def show_tools(self, trans, id, changeset, **kwd): @@ -843,32 +569,7 @@ class RepositoriesController(BaseAPIController): """ recursive = util.asbool(kwd.get("recursive", "True")) downloadable_only = util.asbool(kwd.get("downloadable_only", "True")) - all_metadata = {} - repository = repository_util.get_repository_in_tool_shed( - self.app, id, eagerload_columns=[model.Repository.downloadable_revisions] - ) - for changeset, changehash in metadata_util.get_metadata_revisions( - self.app, repository, sort_revisions=True, downloadable=downloadable_only - ): - metadata = metadata_util.get_current_repository_metadata_for_changeset_revision( - self.app, repository, changehash - ) - if metadata is None: - continue - metadata_dict = metadata.to_dict( - value_mapper={"id": self.app.security.encode_id, "repository_id": self.app.security.encode_id} - ) - metadata_dict["repository"] = repository.to_dict(value_mapper={"id": self.app.security.encode_id}) - if metadata.has_repository_dependencies and recursive: - metadata_dict["repository_dependencies"] = metadata_util.get_all_dependencies( - self.app, metadata, processed_dependency_links=[] - ) - else: - metadata_dict["repository_dependencies"] = [] - if metadata.includes_tools: - metadata_dict["tools"] = metadata.metadata["tools"] - all_metadata[f"{int(changeset)}:{changehash}"] = metadata_dict - return all_metadata + return get_repository_metadata_dict(self.app, id, recursive, downloadable_only) @expose_api def update(self, trans, id, **kwd): @@ -966,31 +667,22 @@ class RepositoriesController(BaseAPIController): description = payload.get("description", "") remote_repository_url = payload.get("remote_repository_url", "") homepage_url = payload.get("homepage_url", "") - category_ids = util.listify(payload.get("category_ids[]", "")) repo_type = payload.get("type", rt_util.UNRESTRICTED) if repo_type not in rt_util.types: raise RequestParameterInvalidException("This repository type is not valid") - invalid_message = repository_util.validate_repository_name(self.app, name, trans.user) - if invalid_message: - raise RequestParameterInvalidException(invalid_message) - - repo, message = repository_util.create_repository( - app=self.app, + request = CreateRepositoryRequest( name=name, - type=repo_type, - description=synopsis, - long_description=description, - user_id=trans.user.id, - category_ids=category_ids, + synopsis=synopsis, + description=description, remote_repository_url=remote_repository_url, homepage_url=homepage_url, + category_ids=payload.get("category_ids[]", ""), + type_=repo_type, ) - - repository_dict = repo.to_dict(view="element", value_mapper=self.__get_value_mapper(trans)) - repository_dict["category_ids"] = [trans.security.encode_id(x.category.id) for x in repo.categories] - return repository_dict + repo = create_repository(trans, request) + return to_element_dict(self.app, repo, include_categories=True) @web.legacy_expose_api def create_changeset_revision(self, trans, id, payload, **kwd): @@ -1011,11 +703,7 @@ class RepositoriesController(BaseAPIController): # Example URL: http://localhost:9009/api/repositories/f9cad7b01a472135 repository = repository_util.get_repository_in_tool_shed(self.app, id) - if not ( - trans.user_is_admin - or self.app.security_agent.user_can_administer_repository(trans.user, repository) - or self.app.security_agent.can_push(self.app, trans.user, repository) - ): + if not can_update_repo(trans, repository): trans.response.status = 400 return { "err_msg": "You do not have permission to update this repository.", diff --git a/lib/tool_shed/webapp/api/repository_revisions.py b/lib/tool_shed/webapp/api/repository_revisions.py index 4c66fbeb3b4..1b413827ace 100644 --- a/lib/tool_shed/webapp/api/repository_revisions.py +++ b/lib/tool_shed/webapp/api/repository_revisions.py @@ -11,19 +11,17 @@ from galaxy import ( web, ) from galaxy.model.base import transaction -from galaxy.webapps.base.controller import ( - BaseAPIController, - HTTPBadRequest, -) +from galaxy.webapps.base.controller import HTTPBadRequest from tool_shed.util import ( metadata_util, repository_util, ) +from . import BaseShedAPIController log = logging.getLogger(__name__) -class RepositoryRevisionsController(BaseAPIController): +class RepositoryRevisionsController(BaseShedAPIController): """RESTful controller for interactions with tool shed repository revisions.""" def __get_value_mapper(self, trans) -> Dict[str, Callable]: diff --git a/lib/tool_shed/webapp/api/tools.py b/lib/tool_shed/webapp/api/tools.py index fdbdc1043a5..33099f5e2ad 100644 --- a/lib/tool_shed/webapp/api/tools.py +++ b/lib/tool_shed/webapp/api/tools.py @@ -1,25 +1,23 @@ import json import logging -from collections import namedtuple from galaxy import ( exceptions, util, - web, ) from galaxy.web import ( expose_api, expose_api_raw_anonymous_and_sessionless, require_admin, ) -from galaxy.webapps.base.controller import BaseAPIController +from tool_shed.managers.tools import search from tool_shed.util.shed_index import build_index -from tool_shed.webapp.search.tool_search import ToolSearch +from . import BaseShedAPIController log = logging.getLogger(__name__) -class ToolsController(BaseAPIController): +class ToolsController(BaseShedAPIController): """RESTful controller for interactions with tools in the Tool Shed.""" @expose_api @@ -85,46 +83,9 @@ class ToolsController(BaseAPIController): raise exceptions.RequestParameterInvalidException('The "page" and "page_size" have to be integers.') return_jsonp = util.asbool(kwd.get("jsonp", False)) callback = kwd.get("callback", "callback") - search_results = self._search(trans, q, page, page_size) + search_results = search(trans, q, page, page_size) if return_jsonp: response = str(f"{callback}({json.dumps(search_results)});") else: response = json.dumps(search_results) return response - - def _search(self, trans, q, page=1, page_size=10): - """ - Perform the search over TS tools index. - Note that search works over the Whoosh index which you have - to pre-create with scripts/tool_shed/build_ts_whoosh_index.sh manually. - Also TS config option toolshed_search_on has to be True and - whoosh_index_dir has to be specified. - """ - conf = self.app.config - if not conf.toolshed_search_on: - raise exceptions.ConfigDoesNotAllowException( - "Searching the TS through the API is turned off for this instance." - ) - if not conf.whoosh_index_dir: - raise exceptions.ConfigDoesNotAllowException( - "There is no directory for the search index specified. Please contact the administrator." - ) - search_term = q.strip() - if len(search_term) < 1: - raise exceptions.RequestParameterInvalidException("The search term has to be at least one character long.") - - tool_search = ToolSearch() - - Boosts = namedtuple( - "Boosts", ["tool_name_boost", "tool_description_boost", "tool_help_boost", "tool_repo_owner_username_boost"] - ) - boosts = Boosts( - float(conf.get("tool_name_boost", 1.2)), - float(conf.get("tool_description_boost", 0.6)), - float(conf.get("tool_help_boost", 0.4)), - float(conf.get("tool_repo_owner_username_boost", 0.3)), - ) - - results = tool_search.search(trans, search_term, page, page_size, boosts) - results["hostname"] = web.url_for("/", qualified=True) - return results diff --git a/lib/tool_shed/webapp/api/users.py b/lib/tool_shed/webapp/api/users.py index 558c2518aad..54f1fba3ffe 100644 --- a/lib/tool_shed/webapp/api/users.py +++ b/lib/tool_shed/webapp/api/users.py @@ -2,23 +2,20 @@ import logging import tool_shed.util.shed_util_common as suc from galaxy import ( - exceptions, util, web, ) -from galaxy.model.base import transaction -from galaxy.security.validate_user_input import ( - validate_email, - validate_password, - validate_publicname, +from tool_shed.managers.users import ( + api_create_user, + index, ) -from galaxy.webapps.base.controller import BaseAPIController -from tool_shed.managers.users import create_user +from tool_shed_client.schema import CreateUserRequest +from . import BaseShedAPIController log = logging.getLogger(__name__) -class UsersController(BaseAPIController): +class UsersController(BaseShedAPIController): """RESTful controller for interactions with users in the Tool Shed.""" @web.expose_api @@ -39,20 +36,18 @@ class UsersController(BaseAPIController): email = payload.get("email", "") password = payload.get("password", "") username = payload.get("username", "") - message = self.__validate(trans, email=email, password=password, confirm=password, username=username) - if message: - raise exceptions.RequestParameterInvalidException(message) - # Create the user. - user = self.__create_user(trans, email, username, password) - user_dict = user.to_dict(view="element", value_mapper=self.__get_value_mapper(trans)) + request = CreateUserRequest( + email=email, + username=username, + password=password, + ) + user = api_create_user(trans, request) + user_dict = user.dict() user_dict["message"] = f"User '{str(user.username)}' has been created." user_dict["url"] = web.url_for(controller="users", action="show", id=trans.security.encode_id(user.id)) return user_dict - def __create_user(self, trans, email, username, password): - return create_user(trans.app, email, username, password) - def __get_value_mapper(self, trans): value_mapper = {"id": trans.security.encode_id} return value_mapper @@ -66,12 +61,8 @@ class UsersController(BaseAPIController): # Example URL: http://localhost:9009/api/users user_dicts = [] deleted = util.asbool(deleted) - for user in ( - trans.sa_session.query(trans.app.model.User) - .filter(trans.app.model.User.table.c.deleted == deleted) - .order_by(trans.app.model.User.table.c.username) - ): - user_dict = user.to_dict(view="collection", value_mapper=self.__get_value_mapper(trans)) + for user in index(trans.app, deleted): + user_dict = user.dict() user_dict["url"] = web.url_for(controller="users", action="show", id=trans.security.encode_id(user.id)) user_dicts.append(user_dict) return user_dicts @@ -94,15 +85,3 @@ class UsersController(BaseAPIController): user_dict = user.to_dict(view="element", value_mapper=self.__get_value_mapper(trans)) user_dict["url"] = web.url_for(controller="users", action="show", id=trans.security.encode_id(user.id)) return user_dict - - def __validate(self, trans, email, password, confirm, username): - if username in ["repos"]: - return f"The term '{username}' is a reserved word in the Tool Shed, so it cannot be used as a public user name." - message = "\n".join( - ( - validate_email(trans, email), - validate_password(trans, password, confirm), - validate_publicname(trans, username), - ) - ).rstrip() - return message diff --git a/lib/tool_shed/webapp/controllers/repository.py b/lib/tool_shed/webapp/controllers/repository.py index 16c4bffdfd6..b6b67d2bd0f 100644 --- a/lib/tool_shed/webapp/controllers/repository.py +++ b/lib/tool_shed/webapp/controllers/repository.py @@ -35,6 +35,7 @@ from galaxy.web.form_builder import ( from galaxy.web.legacy_framework import grids from galaxy.webapps.base.controller import BaseUIController from tool_shed.dependencies.repository import relation_builder +from tool_shed.managers.repositories import readmes from tool_shed.metadata import repository_metadata_manager from tool_shed.tools import ( tool_validator, @@ -1182,16 +1183,7 @@ class RepositoryController(BaseUIController, ratings_util.ItemRatings): changeset_revision = kwd.get("changeset_revision", None) if repository_name is not None and repository_owner is not None and changeset_revision is not None: repository = repository_util.get_repository_by_name_and_owner(trans.app, repository_name, repository_owner) - if repository: - repository_metadata = metadata_util.get_repository_metadata_by_changeset_revision( - trans.app, trans.security.encode_id(repository.id), changeset_revision - ) - if repository_metadata: - metadata = repository_metadata.metadata - if metadata: - return readme_util.build_readme_files_dict( - trans.app, repository, changeset_revision, repository_metadata.metadata - ) + return readmes(trans.app, repository, changeset_revision) return {} @web.json diff --git a/lib/tool_shed/webapp/search/tool_search.py b/lib/tool_shed/webapp/search/tool_search.py index bbe53d7df1d..bf3617c4cd3 100644 --- a/lib/tool_shed/webapp/search/tool_search.py +++ b/lib/tool_shed/webapp/search/tool_search.py @@ -31,7 +31,7 @@ schema = Schema( class ToolSearch: - def search(self, trans, search_term, page, page_size, boosts): + def search(self, app, search_term, page, page_size, boosts): """ Perform the search on the given search_term @@ -39,7 +39,7 @@ class ToolSearch: :returns results: dictionary containing number of hits, hits themselves and matched terms for each """ - tool_index_dir = os.path.join(trans.app.config.whoosh_index_dir, "tools") + tool_index_dir = os.path.join(app.config.whoosh_index_dir, "tools") index_exists = whoosh.index.exists_in(tool_index_dir) if index_exists: index = whoosh.index.open_dir(tool_index_dir) diff --git a/lib/tool_shed/webapp/security/__init__.py b/lib/tool_shed/webapp/security/__init__.py index b8bbbd30cc6..a2cea3ade3b 100644 --- a/lib/tool_shed/webapp/security/__init__.py +++ b/lib/tool_shed/webapp/security/__init__.py @@ -1,5 +1,6 @@ """Tool Shed Security""" import logging +from typing import List from sqlalchemy import ( and_, @@ -239,9 +240,12 @@ class CommunityRBACAgent(RBACAgent): for group in groups: self.associate_components(user=user, group=group) + def usernames_that_can_push(self, repository) -> List[str]: + return listify(repository.allow_push()) + def can_push(self, app, user, repository): if user: - return user.username in listify(repository.allow_push()) + return user.username in self.usernames_that_can_push(repository) return False def user_can_administer_repository(self, user, repository): diff --git a/lib/tool_shed_client/schema/__init__.py b/lib/tool_shed_client/schema/__init__.py index 870bacd5ea9..2651ff1c81b 100644 --- a/lib/tool_shed_client/schema/__init__.py +++ b/lib/tool_shed_client/schema/__init__.py @@ -35,6 +35,10 @@ class Repository(BaseModel): create_time: str +class RepositoryRevisionReadmes(BaseModel): + __root__: Dict[str, str] + + class CreateUserRequest(BaseModel): username: str email: str