From 32b2d38c4e24a292c2f86737e4c88e3009d358e1 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Thu, 15 Sep 2022 10:43:46 -0400 Subject: [PATCH] metadata_generator: move tool shed impl into tool shed package the correct home and allows better typing with actual Repository type annotations --- .../tool_shed/metadata/metadata_generator.py | 193 ---------------- .../metadata/repository_metadata_manager.py | 206 +++++++++++++++++- 2 files changed, 205 insertions(+), 194 deletions(-) diff --git a/lib/galaxy/tool_shed/metadata/metadata_generator.py b/lib/galaxy/tool_shed/metadata/metadata_generator.py index 82e2798ea9d..184ee767c81 100644 --- a/lib/galaxy/tool_shed/metadata/metadata_generator.py +++ b/lib/galaxy/tool_shed/metadata/metadata_generator.py @@ -9,7 +9,6 @@ from typing import ( Tuple, ) -from sqlalchemy import and_ from typing_extensions import Protocol from galaxy import util @@ -38,7 +37,6 @@ from galaxy.tools.data_manager.manager import DataManager from galaxy.tools.repositories import ValidationContext from galaxy.util.tool_shed.common_util import ( generate_clone_url_for_installed_repository, - generate_clone_url_for_repository_in_tool_shed, remove_protocol_and_user_from_clone_url, remove_protocol_from_tool_shed_url, ) @@ -63,12 +61,6 @@ class RepositoryProtocol(Protocol): ... -class ToolShedRepositoryProtocol(RepositoryProtocol): - - def tip(self) -> str: - ... - - class BaseMetadataGenerator: app: MinimalManagerApp repository: Optional[RepositoryProtocol] @@ -1010,191 +1002,6 @@ class GalaxyMetadataGenerator(BaseMetadataGenerator): return repository_dependency_tup, is_valid, error_message - -class ToolShedMetadataGenerator(BaseMetadataGenerator): - """A MetadataGenerator building on ToolShed's app and repository constructs.""" - repository: Optional[ToolShedRepositoryProtocol] - - def __init__( - self, - app: MinimalManagerApp, - repository: Optional[ToolShedRepositoryProtocol] = None, - changeset_revision: Optional[str] = None, - repository_clone_url: Optional[str] = None, - shed_config_dict: Optional[Dict[str, Any]] = None, - relative_install_dir=None, - repository_files_dir=None, - resetting_all_metadata_on_repository=False, - updating_installed_repository=False, - persist=False, - metadata_dict=None, - user=None, - ): - self.app = app - self.user = user - self.repository = repository - if changeset_revision is None and self.repository is not None: - self.changeset_revision = self.repository.tip() - else: - self.changeset_revision = changeset_revision - if repository_clone_url is None and self.repository is not None: - self.repository_clone_url = generate_clone_url_for_repository_in_tool_shed(self.user, self.repository) - else: - self.repository_clone_url = repository_clone_url - if shed_config_dict is None: - self.shed_config_dict = {} - else: - self.shed_config_dict = shed_config_dict - if relative_install_dir is None and self.repository is not None: - relative_install_dir = self.repository.repo_path(self.app) - if repository_files_dir is None and self.repository is not None: - repository_files_dir = self.repository.repo_path(self.app) - if metadata_dict is None: - self.metadata_dict = {} - else: - self.metadata_dict = metadata_dict - self.relative_install_dir = relative_install_dir - self.repository_files_dir = repository_files_dir - self.resetting_all_metadata_on_repository = resetting_all_metadata_on_repository - self.updating_installed_repository = updating_installed_repository - self.persist = persist - self.invalid_file_tups = [] - self.sa_session = app.model.session - - def initial_metadata_dict(self) -> Dict[str, Any]: - return {} - - def set_repository( - self, repository, relative_install_dir: Optional[str] = None, changeset_revision: Optional[str] = None - ): - self.repository = repository - if relative_install_dir is None and self.repository is not None: - relative_install_dir = repository.repo_path(self.app) - if changeset_revision is None and self.repository is not None: - self.set_changeset_revision(self.repository.tip()) - else: - self.set_changeset_revision(changeset_revision) - self.shed_config_dict = {} - self._reset_attributes_after_repository_update(relative_install_dir) - - def handle_repository_elem(self, repository_elem, only_if_compiling_contained_td=False) -> HandleResultT: - """ - Process the received repository_elem which is a tag either from a - repository_dependencies.xml file or a tool_dependencies.xml file. If the former, - we're generating repository dependencies metadata for a repository in the Tool Shed. - If the latter, we're generating package dependency metadata within Galaxy or the - Tool Shed. - """ - is_valid = True - error_message = "" - toolshed = repository_elem.get("toolshed", None) - name = repository_elem.get("name", None) - owner = repository_elem.get("owner", None) - changeset_revision = repository_elem.get("changeset_revision", None) - prior_installation_required = str(repository_elem.get("prior_installation_required", False)) - repository_dependency_tup = [ - toolshed, - name, - owner, - changeset_revision, - prior_installation_required, - str(only_if_compiling_contained_td), - ] - if not toolshed: - # Default to the current tool shed. - toolshed = str(url_for("/", qualified=True)).rstrip("/") - repository_dependency_tup[0] = toolshed - toolshed = remove_protocol_from_tool_shed_url(toolshed) - - if suc.tool_shed_is_this_tool_shed(toolshed): - try: - user = ( - self.sa_session.query(self.app.model.User) - .filter(self.app.model.User.table.c.username == owner) - .one() - ) - except Exception: - error_message = ( - "Ignoring repository dependency definition for tool shed %s, name %s, owner %s, " - % (toolshed, name, owner) - ) - error_message += f"changeset revision {changeset_revision} because the owner is invalid." - log.debug(error_message) - is_valid = False - return repository_dependency_tup, is_valid, error_message - try: - repository = ( - self.sa_session.query(self.app.model.Repository) - .filter( - and_( - self.app.model.Repository.table.c.name == name, - self.app.model.Repository.table.c.user_id == user.id, - ) - ) - .one() - ) - except Exception: - error_message = ( - "Ignoring repository dependency definition for tool shed %s, name %s, owner %s, " - % (toolshed, name, owner) - ) - error_message += f"changeset revision {changeset_revision} because the name is invalid. " - log.debug(error_message) - is_valid = False - return repository_dependency_tup, is_valid, error_message - repo = repository.hg_repo - - # The received changeset_revision may be None since defining it in the dependency definition is optional. - # If this is the case, the default will be to set its value to the repository dependency tip revision. - # This probably occurs only when handling circular dependency definitions. - tip_ctx = repo[repo.changelog.tip()] - # Make sure the repo.changlog includes at least 1 revision. - if changeset_revision is None and tip_ctx.rev() >= 0: - changeset_revision = str(tip_ctx) - repository_dependency_tup = [ - toolshed, - name, - owner, - changeset_revision, - prior_installation_required, - str(only_if_compiling_contained_td), - ] - return repository_dependency_tup, is_valid, error_message - else: - # Find the specified changeset revision in the repository's changelog to see if it's valid. - found = False - for changeset in repo.changelog: - changeset_hash = str(repo[changeset]) - if changeset_hash == changeset_revision: - found = True - break - if not found: - error_message = ( - "Ignoring repository dependency definition for tool shed %s, name %s, owner %s, " - % (toolshed, name, owner) - ) - error_message += ( - f"changeset revision {changeset_revision} because the changeset revision is invalid. " - ) - log.debug(error_message) - is_valid = False - return repository_dependency_tup, is_valid, error_message - else: - # Repository dependencies are currently supported within a single tool shed. - error_message = ( - "Repository dependencies are currently supported only within the same tool shed. Ignoring " - ) - error_message += ( - "repository dependency definition for tool shed %s, name %s, owner %s, changeset revision %s. " - % (toolshed, name, owner, changeset_revision) - ) - log.debug(error_message) - is_valid = False - return repository_dependency_tup, is_valid, error_message - return repository_dependency_tup, is_valid, error_message - - - def _get_readme_file_names(repository_name: str) -> List[str]: """Return a list of file names that will be categorized as README files for the received repository_name.""" readme_files = ["readme", "read_me", "install"] diff --git a/lib/tool_shed/metadata/repository_metadata_manager.py b/lib/tool_shed/metadata/repository_metadata_manager.py index b4492186b1f..a717a5ae9fb 100644 --- a/lib/tool_shed/metadata/repository_metadata_manager.py +++ b/lib/tool_shed/metadata/repository_metadata_manager.py @@ -1,20 +1,29 @@ import logging import tempfile from typing import ( + Any, + Dict, List, Optional, ) from sqlalchemy import ( + and_, false, or_, ) from galaxy import util from galaxy.model.base import transaction +from galaxy.structured_app import MinimalManagerApp +from galaxy.tool_shed.metadata.metadata_generator import ( + BaseMetadataGenerator, + HandleResultT, + InvalidFileT, +) from galaxy.util import inflector +from galaxy.web import url_for from galaxy.web.form_builder import SelectField -from galaxy.tool_shed.galaxy_install.metadata_generator import ToolShedMetadataGenerator from tool_shed.repository_types import util as rt_util from tool_shed.repository_types.metadata import TipOnly from tool_shed.structured_app import ToolShedApp @@ -27,10 +36,199 @@ from tool_shed.util import ( shed_util_common as suc, tool_util, ) +from tool_shed.webapp.model import Repository log = logging.getLogger(__name__) +class ToolShedMetadataGenerator(BaseMetadataGenerator): + """A MetadataGenerator building on ToolShed's app and repository constructs.""" + + repository: Optional[Repository] + + # why is mypy making me re-annotate these things from the base class, it didn't + # when they were in the same file + invalid_file_tups: List[InvalidFileT] + repository_clone_url: Optional[str] + + def __init__( + self, + app: MinimalManagerApp, + repository: Optional[Repository] = None, + changeset_revision: Optional[str] = None, + repository_clone_url: Optional[str] = None, + shed_config_dict: Optional[Dict[str, Any]] = None, + relative_install_dir=None, + repository_files_dir=None, + resetting_all_metadata_on_repository=False, + updating_installed_repository=False, + persist=False, + metadata_dict=None, + user=None, + ): + self.app = app + self.user = user + self.repository = repository + if changeset_revision is None and self.repository is not None: + self.changeset_revision = self.repository.tip() + else: + self.changeset_revision = changeset_revision + if repository_clone_url is None and self.repository is not None: + self.repository_clone_url = common_util.generate_clone_url_for_repository_in_tool_shed( + self.user, self.repository + ) + else: + self.repository_clone_url = repository_clone_url + if shed_config_dict is None: + self.shed_config_dict = {} + else: + self.shed_config_dict = shed_config_dict + if relative_install_dir is None and self.repository is not None: + relative_install_dir = self.repository.repo_path(self.app) + if repository_files_dir is None and self.repository is not None: + repository_files_dir = self.repository.repo_path(self.app) + if metadata_dict is None: + self.metadata_dict = {} + else: + self.metadata_dict = metadata_dict + self.relative_install_dir = relative_install_dir + self.repository_files_dir = repository_files_dir + self.resetting_all_metadata_on_repository = resetting_all_metadata_on_repository + self.updating_installed_repository = updating_installed_repository + self.persist = persist + self.invalid_file_tups = [] + self.sa_session = app.model.session + + def initial_metadata_dict(self) -> Dict[str, Any]: + return {} + + def set_repository( + self, repository, relative_install_dir: Optional[str] = None, changeset_revision: Optional[str] = None + ): + self.repository = repository + if relative_install_dir is None and self.repository is not None: + relative_install_dir = repository.repo_path(self.app) + if changeset_revision is None and self.repository is not None: + self.set_changeset_revision(self.repository.tip()) + else: + self.set_changeset_revision(changeset_revision) + self.shed_config_dict = {} + self._reset_attributes_after_repository_update(relative_install_dir) + + def handle_repository_elem(self, repository_elem, only_if_compiling_contained_td=False) -> HandleResultT: + """ + Process the received repository_elem which is a tag either from a + repository_dependencies.xml file or a tool_dependencies.xml file. If the former, + we're generating repository dependencies metadata for a repository in the Tool Shed. + If the latter, we're generating package dependency metadata within Galaxy or the + Tool Shed. + """ + is_valid = True + error_message = "" + toolshed = repository_elem.get("toolshed", None) + name = repository_elem.get("name", None) + owner = repository_elem.get("owner", None) + changeset_revision = repository_elem.get("changeset_revision", None) + prior_installation_required = str(repository_elem.get("prior_installation_required", False)) + repository_dependency_tup = [ + toolshed, + name, + owner, + changeset_revision, + prior_installation_required, + str(only_if_compiling_contained_td), + ] + if not toolshed: + # Default to the current tool shed. + toolshed = str(url_for("/", qualified=True)).rstrip("/") + repository_dependency_tup[0] = toolshed + toolshed = common_util.remove_protocol_from_tool_shed_url(toolshed) + + if suc.tool_shed_is_this_tool_shed(toolshed): + try: + user = ( + self.sa_session.query(self.app.model.User) + .filter(self.app.model.User.table.c.username == owner) + .one() + ) + except Exception: + error_message = "Ignoring repository dependency definition for tool shed %s, name %s, owner %s, " % ( + toolshed, + name, + owner, + ) + error_message += f"changeset revision {changeset_revision} because the owner is invalid." + log.debug(error_message) + is_valid = False + return repository_dependency_tup, is_valid, error_message + try: + repository = ( + self.sa_session.query(self.app.model.Repository) + .filter( + and_( + self.app.model.Repository.table.c.name == name, + self.app.model.Repository.table.c.user_id == user.id, + ) + ) + .one() + ) + except Exception: + error_message = f"Ignoring repository dependency definition for tool shed {toolshed}," + error_message += f"name {name}, owner {owner}, " + error_message += f"changeset revision {changeset_revision} because the name is invalid. " + log.debug(error_message) + is_valid = False + return repository_dependency_tup, is_valid, error_message + repo = repository.hg_repo + + # The received changeset_revision may be None since defining it in the dependency definition is optional. + # If this is the case, the default will be to set its value to the repository dependency tip revision. + # This probably occurs only when handling circular dependency definitions. + tip_ctx = repo[repo.changelog.tip()] + # Make sure the repo.changlog includes at least 1 revision. + if changeset_revision is None and tip_ctx.rev() >= 0: + changeset_revision = str(tip_ctx) + repository_dependency_tup = [ + toolshed, + name, + owner, + changeset_revision, + prior_installation_required, + str(only_if_compiling_contained_td), + ] + return repository_dependency_tup, is_valid, error_message + else: + # Find the specified changeset revision in the repository's changelog to see if it's valid. + found = False + for changeset in repo.changelog: + changeset_hash = str(repo[changeset]) + if changeset_hash == changeset_revision: + found = True + break + if not found: + error_message = ( + "Ignoring repository dependency definition for tool shed %s, name %s, owner %s, " + % (toolshed, name, owner) + ) + error_message += ( + f"changeset revision {changeset_revision} because the changeset revision is invalid. " + ) + log.debug(error_message) + is_valid = False + return repository_dependency_tup, is_valid, error_message + else: + # Repository dependencies are currently supported within a single tool shed. + error_message = "Repository dependencies are currently supported only within the same tool shed. Ignoring " + error_message += ( + "repository dependency definition for tool shed %s, name %s, owner %s, changeset revision %s. " + % (toolshed, name, owner, changeset_revision) + ) + log.debug(error_message) + is_valid = False + return repository_dependency_tup, is_valid, error_message + return repository_dependency_tup, is_valid, error_message + + class RepositoryMetadataManager(ToolShedMetadataGenerator): def __init__( self, @@ -101,6 +299,7 @@ class RepositoryMetadataManager(ToolShedMetadataGenerator): return repositories_select_field def clean_repository_metadata(self, changeset_revisions): + assert self.repository # Delete all repository_metadata records associated with the repository that have # a changeset_revision that is not in changeset_revisions. We sometimes see multiple # records with the same changeset revision value - no idea how this happens. We'll @@ -431,6 +630,7 @@ class RepositoryMetadataManager(ToolShedMetadataGenerator): downloadable = True else: downloadable = False + assert self.repository repository_metadata = metadata_util.get_repository_metadata_by_changeset_revision( self.app, self.app.security.encode_id(self.repository.id), changeset_revision ) @@ -616,6 +816,7 @@ class RepositoryMetadataManager(ToolShedMetadataGenerator): self.repository because one or more Galaxy utilities may have been deleted from self.repository in the new tip. """ + assert self.repository repository_metadata = metadata_util.get_latest_repository_metadata( self.app, self.repository.id, downloadable=False ) @@ -843,6 +1044,7 @@ class RepositoryMetadataManager(ToolShedMetadataGenerator): def reset_all_metadata_on_repository_in_tool_shed(self): """Reset all metadata on a single repository in a tool shed.""" + assert self.repository log.debug(f"Resetting all metadata on repository: {self.repository.name}") repo = self.repository.hg_repo # The list of changeset_revisions refers to repository_metadata records that have been created @@ -920,6 +1122,7 @@ class RepositoryMetadataManager(ToolShedMetadataGenerator): def reset_all_tool_versions(self, repo): """Reset tool version lineage for those changeset revisions that include valid tools.""" + assert self.repository encoded_repository_id = self.app.security.encode_id(self.repository.id) changeset_revisions_that_contain_tools = [] for changeset in repo.changelog: @@ -1022,6 +1225,7 @@ class RepositoryMetadataManager(ToolShedMetadataGenerator): Set metadata using the self.repository's current disk files, returning specific error messages (if any) to alert the repository owner that the changeset has problems. """ + assert self.repository message = "" status = "done" encoded_id = self.app.security.encode_id(self.repository.id)