diff --git a/lib/galaxy/tool_shed/util/dependency_display.py b/lib/galaxy/tool_shed/util/dependency_display.py new file mode 100644 index 00000000000..1d27431e495 --- /dev/null +++ b/lib/galaxy/tool_shed/util/dependency_display.py @@ -0,0 +1,292 @@ +import logging +import os + +from galaxy import util +from galaxy.tool_shed.util import utility_container_manager +from galaxy.util.tool_shed.common_util import parse_repository_dependency_tuple + +log = logging.getLogger(__name__) + + +class DependencyDisplayer: + def __init__(self, app): + self.app = app + + def _add_installation_directories_to_tool_dependencies(self, tool_dependencies): + """ + Determine the path to the installation directory for each of the received + tool dependencies. This path will be displayed within the tool dependencies + container on the select_tool_panel_section or reselect_tool_panel_section + pages when installing or reinstalling repositories that contain tools with + the defined tool dependencies. The list of tool dependencies may be associated + with more than a single repository. + """ + for dependency_key, requirements_dict in tool_dependencies.items(): + if dependency_key in ["set_environment"]: + continue + repository_name = requirements_dict.get("repository_name", "unknown") + repository_owner = requirements_dict.get("repository_owner", "unknown") + changeset_revision = requirements_dict.get("changeset_revision", "unknown") + dependency_name = requirements_dict["name"] + version = requirements_dict["version"] + if self.app.tool_dependency_dir: + root_dir = self.app.tool_dependency_dir + else: + root_dir = "" + install_dir = os.path.join( + root_dir, dependency_name, version, repository_owner, repository_name, changeset_revision + ) + requirements_dict["install_dir"] = install_dir + tool_dependencies[dependency_key] = requirements_dict + return tool_dependencies + + def generate_message_for_invalid_repository_dependencies(self, metadata_dict, error_from_tuple=False): + """ + Get or generate and return an error message associated with an invalid repository dependency. + """ + message = "" + if metadata_dict: + if error_from_tuple: + # Return the error messages associated with a set of one or more invalid repository + # dependency tuples. + invalid_repository_dependencies_dict = metadata_dict.get("invalid_repository_dependencies", None) + if invalid_repository_dependencies_dict is not None: + invalid_repository_dependencies = invalid_repository_dependencies_dict.get( + "invalid_repository_dependencies", [] + ) + for repository_dependency_tup in invalid_repository_dependencies: + ( + toolshed, + name, + owner, + changeset_revision, + prior_installation_required, + only_if_compiling_contained_td, + error, + ) = parse_repository_dependency_tuple(repository_dependency_tup, contains_error=True) + if error: + message += f"{error} " + else: + # The complete dependency hierarchy could not be determined for a repository being installed into + # Galaxy. This is likely due to invalid repository dependency definitions, so we'll get them from + # the metadata and parse them for display in an error message. This will hopefully communicate the + # problem to the user in such a way that a resolution can be determined. + message += ( + "The complete dependency hierarchy could not be determined for this repository, so no required " + ) + message += "repositories will not be installed. This is likely due to invalid repository dependency definitions. " + repository_dependencies_dict = metadata_dict.get("repository_dependencies", None) + if repository_dependencies_dict is not None: + rd_tups = repository_dependencies_dict.get("repository_dependencies", None) + if rd_tups is not None: + message += "Here are the attributes of the dependencies defined for this repository to help determine the " + message += "cause of this problem.
" + message += '' + message += ( + "" + ) + message += "" + for rd_tup in rd_tups: + ( + tool_shed, + name, + owner, + changeset_revision, + pir, + oicct, + ) = parse_repository_dependency_tuple(rd_tup) + if util.asbool(pir): + pir_str = "True" + else: + pir_str = "" + message += f"" + message += "
Tool shedRepository nameOwnerChangeset revisionPrior install required
{tool_shed}{name}{owner}{changeset_revision}{pir_str}
" + return message + + def generate_message_for_invalid_tool_dependencies(self, metadata_dict): + """ + Tool dependency definitions can only be invalid if they include a definition for a complex + repository dependency and the repository dependency definition is invalid. This method + retrieves the error message associated with the invalid tool dependency for display in the + caller. + """ + message = "" + if metadata_dict: + invalid_tool_dependencies = metadata_dict.get("invalid_tool_dependencies", None) + if invalid_tool_dependencies: + for requirement_dict in invalid_tool_dependencies.values(): + error = requirement_dict.get("error", None) + if error: + message = f"{error} " + return message + + def generate_message_for_orphan_tool_dependencies(self, repository, metadata_dict): + """ + The designation of a ToolDependency into the "orphan" category has evolved over time, + and is significantly restricted since the introduction of the TOOL_DEPENDENCY_DEFINITION + repository type. This designation is still critical, however, in that it handles the + case where a repository contains both tools and a tool_dependencies.xml file, but the + definition in the tool_dependencies.xml file is in no way related to anything defined + by any of the contained tool's requirements tag sets. This is important in that it is + often a result of a typo (e.g., dependency name or version) that differs between the tool + dependency definition within the tool_dependencies.xml file and what is defined in the + tool config's tag sets. In these cases, the user should be presented with + a warning message, and this warning message is is in fact displayed if the following + is_orphan attribute is True. This is tricky because in some cases it may be intentional, + and tool dependencies that are categorized as "orphan" are in fact valid. + """ + has_orphan_package_dependencies = False + has_orphan_set_environment_dependencies = False + message = "" + package_orphans_str = "" + set_environment_orphans_str = "" + # Tool dependencies are categorized as orphan only if the repository contains tools. + if metadata_dict: + tools = metadata_dict.get("tools", []) + tool_dependencies = metadata_dict.get("tool_dependencies", {}) + # The use of the orphan_tool_dependencies category in metadata has been deprecated, + # but we still need to check in case the metadata is out of date. + orphan_tool_dependencies = metadata_dict.get("orphan_tool_dependencies", {}) + # Updating should cause no problems here since a tool dependency cannot be included + # in both dictionaries. + tool_dependencies.update(orphan_tool_dependencies) + if tool_dependencies and tools: + for td_key, requirements_dict in tool_dependencies.items(): + if td_key == "set_environment": + # "set_environment": [{"name": "R_SCRIPT_PATH", "type": "set_environment"}] + for env_requirements_dict in requirements_dict: + name = env_requirements_dict["name"] + type = env_requirements_dict["type"] + if self._tool_dependency_is_orphan(type, name, None, tools): + if not has_orphan_set_environment_dependencies: + has_orphan_set_environment_dependencies = True + set_environment_orphans_str += f"* name: {name}, type: {type}
" + else: + # "R/2.15.1": {"name": "R", "readme": "some string", "type": "package", "version": "2.15.1"} + name = requirements_dict["name"] + type = requirements_dict["type"] + version = requirements_dict["version"] + if self._tool_dependency_is_orphan(type, name, version, tools): + if not has_orphan_package_dependencies: + has_orphan_package_dependencies = True + package_orphans_str += ( + f"* name: {name}, type: {type}, version: {version}
" + ) + if has_orphan_package_dependencies: + message += "The settings for name, version and type from a " + message += "contained tool configuration file's requirement tag does not match " + message += "the information for the following tool dependency definitions in the " + message += "tool_dependencies.xml file, so these tool dependencies have no " + message += "relationship with any tools within this repository.
" + message += package_orphans_str + if has_orphan_set_environment_dependencies: + message += "The settings for name and type from a contained tool " + message += "configuration file's requirement tag does not match the information " + message += "for the following tool dependency definitions in the tool_dependencies.xml " + message += "file, so these tool dependencies have no relationship with any tools within " + message += "this repository.
" + message += set_environment_orphans_str + return message + + def populate_containers_dict_from_repository_metadata(self, repository): + """ + Retrieve necessary information from the received repository's metadata to populate the + containers_dict for display. This method is called only from Galaxy (not the tool shed) + when displaying repository dependencies for installed repositories and when displaying + them for uninstalled repositories that are being reinstalled. + """ + metadata = repository.metadata_ + if metadata: + # Handle repository dependencies. + ( + installed_repository_dependencies, + missing_repository_dependencies, + ) = self.app.installed_repository_manager.get_installed_and_missing_repository_dependencies(repository) + # Handle the current repository's tool dependencies. + repository_tool_dependencies = metadata.get("tool_dependencies", None) + # Make sure to display missing tool dependencies as well. + repository_invalid_tool_dependencies = metadata.get("invalid_tool_dependencies", None) + if repository_invalid_tool_dependencies is not None: + if repository_tool_dependencies is None: + repository_tool_dependencies = {} + repository_tool_dependencies.update(repository_invalid_tool_dependencies) + gucm = GalaxyUtilityContainerManager(self.app) + containers_dict = gucm.build_repository_containers( + missing_repository_dependencies=missing_repository_dependencies, + repository_dependencies=installed_repository_dependencies, + ) + else: + containers_dict = dict( + repository_dependencies=None, + ) + return containers_dict + + def _tool_dependency_is_orphan(self, type, name, version, tools): + """ + Determine if the combination of the received type, name and version is defined in the + tag for at least one tool in the received list of tools. If not, the tool dependency defined by the + combination is considered an orphan in its repository in the tool shed. + """ + if type == "package": + if name and version: + for tool_dict in tools: + requirements = tool_dict.get("requirements", []) + for requirement_dict in requirements: + req_name = requirement_dict.get("name", None) + req_version = requirement_dict.get("version", None) + req_type = requirement_dict.get("type", None) + if req_name == name and req_version == version and req_type == type: + return False + elif type == "set_environment": + if name: + for tool_dict in tools: + requirements = tool_dict.get("requirements", []) + for requirement_dict in requirements: + req_name = requirement_dict.get("name", None) + req_type = requirement_dict.get("type", None) + if req_name == name and req_type == type: + return False + return True + + +class GalaxyUtilityContainerManager(utility_container_manager.UtilityContainerManager): + def __init__(self, app): + self.app = app + + def build_repository_containers( + self, + missing_repository_dependencies, + repository_dependencies, + ): + """ + Return a dictionary of containers for the received repository's dependencies and readme files for + display during installation to Galaxy. + """ + containers_dict = dict( + repository_dependencies=None, + missing_repository_dependencies=None, + ) + # Some of the tool dependency folders will include links to display tool dependency information, and + # some of these links require the repository id. However we need to be careful because sometimes the + # repository object is None. + try: + folder_id = 0 + # Installed repository dependencies container. + if repository_dependencies: + label = "Installed repository dependencies" + folder_id, repository_dependencies_root_folder = self.build_repository_dependencies_folder( + folder_id=folder_id, repository_dependencies=repository_dependencies, label=label, installed=True + ) + containers_dict["repository_dependencies"] = repository_dependencies_root_folder + # Missing repository dependencies container. + if missing_repository_dependencies: + folder_id, missing_repository_dependencies_root_folder = self.build_repository_dependencies_folder( + folder_id=folder_id, + repository_dependencies=missing_repository_dependencies, + label="Missing repository dependencies", + installed=False, + ) + containers_dict["missing_repository_dependencies"] = missing_repository_dependencies_root_folder + except Exception as e: + log.debug(f"Exception in build_repository_containers: {str(e)}") + return containers_dict diff --git a/lib/tool_shed/utility_containers/utility_container_manager.py b/lib/galaxy/tool_shed/util/utility_container_manager.py similarity index 100% rename from lib/tool_shed/utility_containers/utility_container_manager.py rename to lib/galaxy/tool_shed/util/utility_container_manager.py diff --git a/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py b/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py index 4616f761d18..ea077071571 100644 --- a/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py +++ b/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py @@ -12,13 +12,19 @@ from galaxy.exceptions import ConfigDoesNotAllowException from galaxy.tool_shed.galaxy_install import install_manager from galaxy.tool_shed.galaxy_install.repository_dependencies import repository_dependency_manager from galaxy.tool_shed.galaxy_install.tools import tool_panel_manager -from galaxy.tool_shed.util import tool_util +from galaxy.tool_shed.util import dependency_display from galaxy.tool_shed.util.repository_util import ( create_or_update_tool_shed_repository, get_absolute_path_to_file_in_repository, get_installed_tool_shed_repository, get_tool_shed_repository_by_id, ) +from galaxy.tool_shed.util.shed_util_common import ( + clean_dependency_relationships, + get_ctx_rev, + get_tool_panel_config_tool_path_install_dir, + have_shed_tool_conf_for_install, +) from galaxy.tool_util.deps import views from galaxy.util import unicodify from galaxy.util.tool_shed import ( @@ -26,9 +32,12 @@ from galaxy.util.tool_shed import ( encoding_util, ) from galaxy.web.form_builder import CheckboxField -from tool_shed.galaxy_install import dependency_display -from tool_shed.util import shed_util_common as suc from tool_shed.util.repository_util import create_repo_info_dict +from tool_shed.util.shed_util_common import ( + get_repository_type_from_tool_shed, + get_requirements_from_repository, + get_tool_dependency_definition_metadata_from_tool_shed, +) from tool_shed.util.web_util import escape from .admin import AdminGalaxy @@ -151,10 +160,8 @@ class AdminToolshed(AdminGalaxy): repository = get_installed_tool_shed_repository(trans.app, repository_id) if repository is None: return trans.show_error_message("Invalid repository specified.") - tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry(trans.app, str(repository.tool_shed)) description = kwd.get("description", repository.description) status = "ok" - _, tool_path, _ = suc.get_tool_panel_config_tool_path_install_dir(trans.app, repository) if repository.in_error_state: message = "This repository is not installed correctly (see the Repository installation error below). Choose " message += "Reset to install from the Repository Actions menu, correct problems if necessary and try " @@ -171,14 +178,10 @@ class AdminToolshed(AdminGalaxy): message = "The repository information has been updated." dd = dependency_display.DependencyDisplayer(trans.app) containers_dict = dd.populate_containers_dict_from_repository_metadata( - tool_shed_url=tool_shed_url, - tool_path=tool_path, repository=repository, - reinstalling=False, - required_repo_info_dicts=None, ) view = views.DependencyResolversView(self.app) - tool_requirements_d = suc.get_requirements_from_repository(repository) + tool_requirements_d = get_requirements_from_repository(repository) requirements_status = view.get_requirements_status(tool_requirements_d, repository.installed_tool_dependencies) management_dict = { "status": status, @@ -211,9 +214,9 @@ class AdminToolshed(AdminGalaxy): install_repository_dependencies = CheckboxField.is_checked(kwd.get("install_repository_dependencies", "")) install_tool_dependencies = CheckboxField.is_checked(kwd.get("install_tool_dependencies", "")) install_resolver_dependencies = CheckboxField.is_checked(kwd.get("install_resolver_dependencies", "")) - if not suc.have_shed_tool_conf_for_install(trans.app): + if not have_shed_tool_conf_for_install(trans.app): raise Exception("No valid shed tool configuration file available, please configure one") - shed_tool_conf, tool_path, relative_install_dir = suc.get_tool_panel_config_tool_path_install_dir( + shed_tool_conf, tool_path, relative_install_dir = get_tool_panel_config_tool_path_install_dir( trans.app, tool_shed_repository ) repository_clone_url = common_util.generate_clone_url_for_installed_repository( @@ -245,19 +248,19 @@ class AdminToolshed(AdminGalaxy): # Just in case the tool_section.id differs from tool_panel_section_id, which it shouldn't... tool_panel_section_id = str(tool_section.id) if tool_shed_repository.status == trans.install_model.ToolShedRepository.installation_status.UNINSTALLED: - repository_type = suc.get_repository_type_from_tool_shed( + repository_type = get_repository_type_from_tool_shed( trans.app, tool_shed_url, tool_shed_repository.name, tool_shed_repository.owner ) if repository_type == rt_util.TOOL_DEPENDENCY_DEFINITION: # Repositories of type tool_dependency_definition must get the latest # metadata from the Tool Shed since they have only a single installable # revision. - raw_text = suc.get_tool_dependency_definition_metadata_from_tool_shed( + raw_text = get_tool_dependency_definition_metadata_from_tool_shed( trans.app, tool_shed_url, tool_shed_repository.name, tool_shed_repository.owner ) new_meta = json.loads(raw_text) # Clean up old repository dependency and tool dependency relationships. - suc.clean_dependency_relationships(trans, new_meta, tool_shed_repository, tool_shed_url) + clean_dependency_relationships(trans, new_meta, tool_shed_repository, tool_shed_url) # The repository's status must be updated from 'Uninstalled' to 'New' when initiating reinstall # so the repository_installation_updater will function. tool_shed_repository = create_or_update_tool_shed_repository( @@ -273,7 +276,7 @@ class AdminToolshed(AdminGalaxy): owner=tool_shed_repository.owner, dist_to_shed=tool_shed_repository.dist_to_shed, ) - ctx_rev = suc.get_ctx_rev( + ctx_rev = get_ctx_rev( trans.app, tool_shed_url, tool_shed_repository.name, @@ -368,172 +371,3 @@ class AdminToolshed(AdminGalaxy): except Exception: log.exception("Problem you know in here somewhere...") raise - - @web.expose - @web.require_admin - @legacy_tool_shed_endpoint - def reselect_tool_panel_section(self, trans, **kwd): - """ - Select or change the tool panel section to contain the tools included in the tool shed repository - being reinstalled. If there are updates available for the repository in the tool shed, the - tool_dependencies and repository_dependencies associated with the updated changeset revision will - have been retrieved from the tool shed and passed in the received kwd. In this case, the stored - tool shed repository metadata from the Galaxy database will not be used since it is outdated. - """ - message = "" - status = "done" - repository_id = kwd.get("id", None) - latest_changeset_revision = kwd.get("latest_changeset_revision", None) - latest_ctx_rev = kwd.get("latest_ctx_rev", None) - tool_shed_repository = get_installed_tool_shed_repository(trans.app, repository_id) - repository_clone_url = common_util.generate_clone_url_for_installed_repository(trans.app, tool_shed_repository) - metadata = tool_shed_repository.metadata_ - tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( - trans.app, str(tool_shed_repository.tool_shed) - ) - tool_path = tool_shed_repository.get_tool_relative_path(trans.app)[0] - if latest_changeset_revision and latest_ctx_rev: - # There are updates available in the tool shed for the repository, so use the received - # dependency information which was retrieved from the tool shed. - encoded_updated_repo_info_dict = kwd.get("updated_repo_info_dict", None) - updated_repo_info_dict = encoding_util.tool_shed_decode(encoded_updated_repo_info_dict) - readme_files_dict = updated_repo_info_dict.get("readme_files_dict", None) - includes_data_managers = updated_repo_info_dict.get("includes_data_managers", False) - includes_tool_dependencies = updated_repo_info_dict.get("includes_tool_dependencies", False) - repo_info_dict = updated_repo_info_dict["repo_info_dict"] - else: - # There are no updates available from the tool shed for the repository, so use its locally stored metadata. - includes_data_managers = False - includes_tool_dependencies = False - readme_files_dict = None - tool_dependencies = None - if metadata: - if "data_manager" in metadata: - includes_data_managers = True - if "tool_dependencies" in metadata: - includes_tool_dependencies = True - # Since we're reinstalling, we need to send a request to the tool shed to get the README files. - params = dict( - name=tool_shed_repository.name, - owner=tool_shed_repository.owner, - changeset_revision=tool_shed_repository.installed_changeset_revision, - ) - pathspec = ["repository", "get_readme_files"] - raw_text = util.url_get( - tool_shed_url, - auth=self.app.tool_shed_registry.url_auth(tool_shed_url), - pathspec=pathspec, - params=params, - ) - readme_files_dict = json.loads(raw_text) - tool_dependencies = metadata.get("tool_dependencies", None) - rdim = repository_dependency_manager.RepositoryDependencyInstallManager(trans.app) - repository_dependencies = rdim.get_repository_dependencies_for_installed_tool_shed_repository( - trans.app, tool_shed_repository - ) - repo_info_dict = create_repo_info_dict( - trans.app, - repository_clone_url=repository_clone_url, - changeset_revision=tool_shed_repository.installed_changeset_revision, - ctx_rev=tool_shed_repository.ctx_rev, - repository_owner=tool_shed_repository.owner, - repository_name=tool_shed_repository.name, - tool_dependencies=tool_dependencies, - repository_dependencies=repository_dependencies, - ) - irm = trans.app.installed_repository_manager - dependencies_for_repository_dict = irm.get_dependencies_for_repository( - tool_shed_url, repo_info_dict, includes_tool_dependencies, updating=True - ) - includes_tool_dependencies = dependencies_for_repository_dict.get("includes_tool_dependencies", False) - includes_tools = dependencies_for_repository_dict.get("includes_tools", False) - includes_tools_for_display_in_tool_panel = dependencies_for_repository_dict.get( - "includes_tools_for_display_in_tool_panel", False - ) - installed_repository_dependencies = dependencies_for_repository_dict.get( - "installed_repository_dependencies", None - ) - installed_tool_dependencies = dependencies_for_repository_dict.get("installed_tool_dependencies", None) - missing_repository_dependencies = dependencies_for_repository_dict.get("missing_repository_dependencies", None) - missing_tool_dependencies = dependencies_for_repository_dict.get("missing_tool_dependencies", None) - if installed_repository_dependencies or missing_repository_dependencies: - has_repository_dependencies = True - else: - has_repository_dependencies = False - if includes_tools_for_display_in_tool_panel: - # Get the location in the tool panel in which the tools were originally loaded. - if "tool_panel_section" in metadata: - tool_panel_dict = metadata["tool_panel_section"] - if tool_panel_dict: - if tool_util.panel_entry_per_tool(tool_panel_dict): - # The following forces everything to be loaded into 1 section (or no section) in the tool panel. - tool_section_dicts = tool_panel_dict[next(iter(tool_panel_dict.keys()))] - tool_section_dict = tool_section_dicts[0] - original_section_name = tool_section_dict["name"] - else: - original_section_name = tool_panel_dict["name"] - else: - original_section_name = "" - else: - original_section_name = "" - tool_panel_section_select_field = tool_util.build_tool_panel_section_select_field(trans.app) - no_changes_check_box = CheckboxField("no_changes", value=True) - if original_section_name: - message += ( - "The tools contained in your %s repository were last loaded into the tool panel section %s. " - % (escape(tool_shed_repository.name), original_section_name) - ) - message += "Uncheck the No changes check box and select a different tool panel section to load the tools in a " - message += "different section in the tool panel. " - status = "warning" - else: - message += f"The tools contained in your {escape(tool_shed_repository.name)} repository were last loaded into the tool panel outside of any sections. " - message += "Uncheck the No changes check box and select a tool panel section to load the tools into that section. " - status = "warning" - else: - no_changes_check_box = None - original_section_name = "" - tool_panel_section_select_field = None - shed_tool_conf_select_field = tool_util.build_shed_tool_conf_select_field(trans.app) - dd = dependency_display.DependencyDisplayer(trans.app) - containers_dict = dd.populate_containers_dict_for_new_install( - tool_shed_url=tool_shed_url, - tool_path=tool_path, - readme_files_dict=readme_files_dict, - installed_repository_dependencies=installed_repository_dependencies, - missing_repository_dependencies=missing_repository_dependencies, - installed_tool_dependencies=installed_tool_dependencies, - missing_tool_dependencies=missing_tool_dependencies, - updating=False, - ) - # Since we're reinstalling we'll merge the list of missing repository dependencies into the list of - # installed repository dependencies since each displayed repository dependency will display a status, - # whether installed or missing. - containers_dict = dd.merge_missing_repository_dependencies_to_installed_container(containers_dict) - # Handle repository dependencies check box. - install_repository_dependencies_check_box = CheckboxField("install_repository_dependencies", value=True) - view = views.DependencyResolversView(self.app) - if view.installable_resolvers: - install_resolver_dependencies_check_box = CheckboxField("install_resolver_dependencies", value=True) - else: - install_resolver_dependencies_check_box = None - return trans.fill_template( - "/admin/tool_shed_repository/reselect_tool_panel_section.mako", - repository=tool_shed_repository, - no_changes_check_box=no_changes_check_box, - original_section_name=original_section_name, - includes_data_managers=includes_data_managers, - includes_tools=includes_tools, - includes_tools_for_display_in_tool_panel=includes_tools_for_display_in_tool_panel, - includes_tool_dependencies=includes_tool_dependencies, - has_repository_dependencies=has_repository_dependencies, - install_repository_dependencies_check_box=install_repository_dependencies_check_box, - install_resolver_dependencies_check_box=install_resolver_dependencies_check_box, - containers_dict=containers_dict, - tool_panel_section_select_field=tool_panel_section_select_field, - shed_tool_conf_select_field=shed_tool_conf_select_field, - encoded_repo_info_dict=encoding_util.tool_shed_encode(repo_info_dict), - repo_info_dict=repo_info_dict, - message=message, - status=status, - ) diff --git a/lib/tool_shed/galaxy_install/dependency_display.py b/lib/tool_shed/galaxy_install/dependency_display.py deleted file mode 100644 index 5c4c8cf0e56..00000000000 --- a/lib/tool_shed/galaxy_install/dependency_display.py +++ /dev/null @@ -1,686 +0,0 @@ -import json -import logging -import os -import threading - -from galaxy import util -from tool_shed.galaxy_install.utility_containers import GalaxyUtilityContainerManager -from tool_shed.util import ( - common_util, - container_util, - readme_util, - repository_util, - tool_dependency_util, -) -from tool_shed.utility_containers import utility_container_manager - -log = logging.getLogger(__name__) - - -class DependencyDisplayer: - def __init__(self, app): - self.app = app - - def add_installation_directories_to_tool_dependencies(self, tool_dependencies): - """ - Determine the path to the installation directory for each of the received - tool dependencies. This path will be displayed within the tool dependencies - container on the select_tool_panel_section or reselect_tool_panel_section - pages when installing or reinstalling repositories that contain tools with - the defined tool dependencies. The list of tool dependencies may be associated - with more than a single repository. - """ - for dependency_key, requirements_dict in tool_dependencies.items(): - if dependency_key in ["set_environment"]: - continue - repository_name = requirements_dict.get("repository_name", "unknown") - repository_owner = requirements_dict.get("repository_owner", "unknown") - changeset_revision = requirements_dict.get("changeset_revision", "unknown") - dependency_name = requirements_dict["name"] - version = requirements_dict["version"] - if self.app.tool_dependency_dir: - root_dir = self.app.tool_dependency_dir - else: - root_dir = "" - install_dir = os.path.join( - root_dir, dependency_name, version, repository_owner, repository_name, changeset_revision - ) - requirements_dict["install_dir"] = install_dir - tool_dependencies[dependency_key] = requirements_dict - return tool_dependencies - - def generate_message_for_invalid_repository_dependencies(self, metadata_dict, error_from_tuple=False): - """ - Get or generate and return an error message associated with an invalid repository dependency. - """ - message = "" - if metadata_dict: - if error_from_tuple: - # Return the error messages associated with a set of one or more invalid repository - # dependency tuples. - invalid_repository_dependencies_dict = metadata_dict.get("invalid_repository_dependencies", None) - if invalid_repository_dependencies_dict is not None: - invalid_repository_dependencies = invalid_repository_dependencies_dict.get( - "invalid_repository_dependencies", [] - ) - for repository_dependency_tup in invalid_repository_dependencies: - ( - toolshed, - name, - owner, - changeset_revision, - prior_installation_required, - only_if_compiling_contained_td, - error, - ) = common_util.parse_repository_dependency_tuple( - repository_dependency_tup, contains_error=True - ) - if error: - message += f"{error} " - else: - # The complete dependency hierarchy could not be determined for a repository being installed into - # Galaxy. This is likely due to invalid repository dependency definitions, so we'll get them from - # the metadata and parse them for display in an error message. This will hopefully communicate the - # problem to the user in such a way that a resolution can be determined. - message += ( - "The complete dependency hierarchy could not be determined for this repository, so no required " - ) - message += "repositories will not be installed. This is likely due to invalid repository dependency definitions. " - repository_dependencies_dict = metadata_dict.get("repository_dependencies", None) - if repository_dependencies_dict is not None: - rd_tups = repository_dependencies_dict.get("repository_dependencies", None) - if rd_tups is not None: - message += "Here are the attributes of the dependencies defined for this repository to help determine the " - message += "cause of this problem.
" - message += '' - message += ( - "" - ) - message += "" - for rd_tup in rd_tups: - ( - tool_shed, - name, - owner, - changeset_revision, - pir, - oicct, - ) = common_util.parse_repository_dependency_tuple(rd_tup) - if util.asbool(pir): - pir_str = "True" - else: - pir_str = "" - message += f"" - message += "
Tool shedRepository nameOwnerChangeset revisionPrior install required
{tool_shed}{name}{owner}{changeset_revision}{pir_str}
" - return message - - def generate_message_for_invalid_tool_dependencies(self, metadata_dict): - """ - Tool dependency definitions can only be invalid if they include a definition for a complex - repository dependency and the repository dependency definition is invalid. This method - retrieves the error message associated with the invalid tool dependency for display in the - caller. - """ - message = "" - if metadata_dict: - invalid_tool_dependencies = metadata_dict.get("invalid_tool_dependencies", None) - if invalid_tool_dependencies: - for requirement_dict in invalid_tool_dependencies.values(): - error = requirement_dict.get("error", None) - if error: - message = f"{error} " - return message - - def generate_message_for_orphan_tool_dependencies(self, repository, metadata_dict): - """ - The designation of a ToolDependency into the "orphan" category has evolved over time, - and is significantly restricted since the introduction of the TOOL_DEPENDENCY_DEFINITION - repository type. This designation is still critical, however, in that it handles the - case where a repository contains both tools and a tool_dependencies.xml file, but the - definition in the tool_dependencies.xml file is in no way related to anything defined - by any of the contained tool's requirements tag sets. This is important in that it is - often a result of a typo (e.g., dependency name or version) that differs between the tool - dependency definition within the tool_dependencies.xml file and what is defined in the - tool config's tag sets. In these cases, the user should be presented with - a warning message, and this warning message is is in fact displayed if the following - is_orphan attribute is True. This is tricky because in some cases it may be intentional, - and tool dependencies that are categorized as "orphan" are in fact valid. - """ - has_orphan_package_dependencies = False - has_orphan_set_environment_dependencies = False - message = "" - package_orphans_str = "" - set_environment_orphans_str = "" - # Tool dependencies are categorized as orphan only if the repository contains tools. - if metadata_dict: - tools = metadata_dict.get("tools", []) - tool_dependencies = metadata_dict.get("tool_dependencies", {}) - # The use of the orphan_tool_dependencies category in metadata has been deprecated, - # but we still need to check in case the metadata is out of date. - orphan_tool_dependencies = metadata_dict.get("orphan_tool_dependencies", {}) - # Updating should cause no problems here since a tool dependency cannot be included - # in both dictionaries. - tool_dependencies.update(orphan_tool_dependencies) - if tool_dependencies and tools: - for td_key, requirements_dict in tool_dependencies.items(): - if td_key == "set_environment": - # "set_environment": [{"name": "R_SCRIPT_PATH", "type": "set_environment"}] - for env_requirements_dict in requirements_dict: - name = env_requirements_dict["name"] - type = env_requirements_dict["type"] - if self.tool_dependency_is_orphan(type, name, None, tools): - if not has_orphan_set_environment_dependencies: - has_orphan_set_environment_dependencies = True - set_environment_orphans_str += f"* name: {name}, type: {type}
" - else: - # "R/2.15.1": {"name": "R", "readme": "some string", "type": "package", "version": "2.15.1"} - name = requirements_dict["name"] - type = requirements_dict["type"] - version = requirements_dict["version"] - if self.tool_dependency_is_orphan(type, name, version, tools): - if not has_orphan_package_dependencies: - has_orphan_package_dependencies = True - package_orphans_str += ( - f"* name: {name}, type: {type}, version: {version}
" - ) - if has_orphan_package_dependencies: - message += "The settings for name, version and type from a " - message += "contained tool configuration file's requirement tag does not match " - message += "the information for the following tool dependency definitions in the " - message += "tool_dependencies.xml file, so these tool dependencies have no " - message += "relationship with any tools within this repository.
" - message += package_orphans_str - if has_orphan_set_environment_dependencies: - message += "The settings for name and type from a contained tool " - message += "configuration file's requirement tag does not match the information " - message += "for the following tool dependency definitions in the tool_dependencies.xml " - message += "file, so these tool dependencies have no relationship with any tools within " - message += "this repository.
" - message += set_environment_orphans_str - return message - - def get_installed_and_missing_tool_dependencies_for_installed_repository(self, repository, all_tool_dependencies): - """ - Return the lists of installed tool dependencies and missing tool dependencies for a Tool Shed - repository that has been installed into Galaxy. - """ - if all_tool_dependencies: - tool_dependencies = {} - missing_tool_dependencies = {} - for td_key, val in all_tool_dependencies.items(): - if td_key in ["set_environment"]: - for index, td_info_dict in enumerate(val): - name = td_info_dict["name"] - version = None - type = td_info_dict["type"] - tool_dependency = tool_dependency_util.get_tool_dependency_by_name_type_repository( - self.app, repository, name, type - ) - if tool_dependency: - td_info_dict["repository_id"] = repository.id - td_info_dict["tool_dependency_id"] = tool_dependency.id - if tool_dependency.status: - tool_dependency_status = str(tool_dependency.status) - else: - tool_dependency_status = "Never installed" - td_info_dict["status"] = tool_dependency_status - val[index] = td_info_dict - if ( - tool_dependency.status - == self.app.install_model.ToolDependency.installation_status.INSTALLED - ): - tool_dependencies[td_key] = val - else: - missing_tool_dependencies[td_key] = val - else: - name = val["name"] - version = val["version"] - type = val["type"] - tool_dependency = tool_dependency_util.get_tool_dependency_by_name_version_type_repository( - self.app, repository, name, version, type - ) - if tool_dependency: - val["repository_id"] = repository.id - val["tool_dependency_id"] = tool_dependency.id - if tool_dependency.status: - tool_dependency_status = str(tool_dependency.status) - else: - tool_dependency_status = "Never installed" - val["status"] = tool_dependency_status - if ( - tool_dependency.status - == self.app.install_model.ToolDependency.installation_status.INSTALLED - ): - tool_dependencies[td_key] = val - else: - missing_tool_dependencies[td_key] = val - else: - tool_dependencies = None - missing_tool_dependencies = None - return tool_dependencies, missing_tool_dependencies - - def merge_containers_dicts_for_new_install(self, containers_dicts): - """ - When installing one or more tool shed repositories for the first time, the received list of - containers_dicts contains a containers_dict for each repository being installed. Since the - repositories are being installed for the first time, all entries are None except the repository - dependencies and tool dependencies. The entries for missing dependencies are all None since - they have previously been merged into the installed dependencies. This method will merge the - dependencies entries into a single container and return it for display. - """ - new_containers_dict = dict( - readme_files=None, - datatypes=None, - missing_repository_dependencies=None, - repository_dependencies=None, - missing_tool_dependencies=None, - tool_dependencies=None, - invalid_tools=None, - valid_tools=None, - workflows=None, - ) - if containers_dicts: - lock = threading.Lock() - lock.acquire(True) - try: - repository_dependencies_root_folder = None - tool_dependencies_root_folder = None - # Use a unique folder id (hopefully the following is). - folder_id = 867 - for old_container_dict in containers_dicts: - # Merge repository_dependencies. - old_container_repository_dependencies_root = old_container_dict["repository_dependencies"] - if old_container_repository_dependencies_root: - if repository_dependencies_root_folder is None: - repository_dependencies_root_folder = utility_container_manager.Folder( - id=folder_id, key="root", label="root", parent=None - ) - folder_id += 1 - repository_dependencies_folder = utility_container_manager.Folder( - id=folder_id, - key="merged", - label="Repository dependencies", - parent=repository_dependencies_root_folder, - ) - folder_id += 1 - # The old_container_repository_dependencies_root will be a root folder containing a single sub_folder. - old_container_repository_dependencies_folder = ( - old_container_repository_dependencies_root.folders[0] - ) - # Change the folder id so it won't confict with others being merged. - old_container_repository_dependencies_folder.id = folder_id - folder_id += 1 - repository_components_tuple = container_util.get_components_from_key( - old_container_repository_dependencies_folder.key - ) - components_list = repository_util.extract_components_from_tuple(repository_components_tuple) - name = components_list[1] - # Generate the label by retrieving the repository name. - old_container_repository_dependencies_folder.label = str(name) - repository_dependencies_folder.folders.append(old_container_repository_dependencies_folder) - # Merge tool_dependencies. - old_container_tool_dependencies_root = old_container_dict["tool_dependencies"] - if old_container_tool_dependencies_root: - if tool_dependencies_root_folder is None: - tool_dependencies_root_folder = utility_container_manager.Folder( - id=folder_id, key="root", label="root", parent=None - ) - folder_id += 1 - tool_dependencies_folder = utility_container_manager.Folder( - id=folder_id, - key="merged", - label="Tool dependencies", - parent=tool_dependencies_root_folder, - ) - folder_id += 1 - else: - td_list = [td.listify for td in tool_dependencies_folder.tool_dependencies] - # The old_container_tool_dependencies_root will be a root folder containing a single sub_folder. - old_container_tool_dependencies_folder = old_container_tool_dependencies_root.folders[0] - for td in old_container_tool_dependencies_folder.tool_dependencies: - if td.listify not in td_list: - tool_dependencies_folder.tool_dependencies.append(td) - if repository_dependencies_root_folder: - repository_dependencies_root_folder.folders.append(repository_dependencies_folder) - new_containers_dict["repository_dependencies"] = repository_dependencies_root_folder - if tool_dependencies_root_folder: - tool_dependencies_root_folder.folders.append(tool_dependencies_folder) - new_containers_dict["tool_dependencies"] = tool_dependencies_root_folder - except Exception as e: - log.debug(f"Exception in merge_containers_dicts_for_new_install: {e}") - finally: - lock.release() - return new_containers_dict - - def merge_missing_repository_dependencies_to_installed_container(self, containers_dict): - """ - Merge the list of missing repository dependencies into the list of installed - repository dependencies. - """ - missing_rd_container_root = containers_dict.get("missing_repository_dependencies", None) - if missing_rd_container_root: - # The missing_rd_container_root will be a root folder containing a single sub_folder. - missing_rd_container = missing_rd_container_root.folders[0] - installed_rd_container_root = containers_dict.get("repository_dependencies", None) - # The installed_rd_container_root will be a root folder containing a single sub_folder. - if installed_rd_container_root: - installed_rd_container = installed_rd_container_root.folders[0] - installed_rd_container.label = "Repository dependencies" - for index, rd in enumerate(missing_rd_container.repository_dependencies): - # Skip the header row. - if index == 0: - continue - installed_rd_container.repository_dependencies.append(rd) - installed_rd_container_root.folders = [installed_rd_container] - containers_dict["repository_dependencies"] = installed_rd_container_root - else: - # Change the folder label from 'Missing repository dependencies' to be - # 'Repository dependencies' for display. - root_container = containers_dict["missing_repository_dependencies"] - for sub_container in root_container.folders: - # There should only be 1 sub-folder. - sub_container.label = "Repository dependencies" - containers_dict["repository_dependencies"] = root_container - containers_dict["missing_repository_dependencies"] = None - return containers_dict - - def merge_missing_tool_dependencies_to_installed_container(self, containers_dict): - """ - Merge the list of missing tool dependencies into the list of installed tool - dependencies. - """ - missing_td_container_root = containers_dict.get("missing_tool_dependencies", None) - if missing_td_container_root: - # The missing_td_container_root will be a root folder containing a single sub_folder. - missing_td_container = missing_td_container_root.folders[0] - installed_td_container_root = containers_dict.get("tool_dependencies", None) - # The installed_td_container_root will be a root folder containing a single sub_folder. - if installed_td_container_root: - installed_td_container = installed_td_container_root.folders[0] - installed_td_container.label = "Tool dependencies" - for index, td in enumerate(missing_td_container.tool_dependencies): - # Skip the header row. - if index == 0: - continue - installed_td_container.tool_dependencies.append(td) - installed_td_container_root.folders = [installed_td_container] - containers_dict["tool_dependencies"] = installed_td_container_root - else: - # Change the folder label from 'Missing tool dependencies' to be - # 'Tool dependencies' for display. - root_container = containers_dict["missing_tool_dependencies"] - for sub_container in root_container.folders: - # There should only be 1 subfolder. - sub_container.label = "Tool dependencies" - containers_dict["tool_dependencies"] = root_container - containers_dict["missing_tool_dependencies"] = None - return containers_dict - - def populate_containers_dict_for_new_install( - self, - tool_shed_url, - tool_path, - readme_files_dict, - installed_repository_dependencies, - missing_repository_dependencies, - installed_tool_dependencies, - missing_tool_dependencies, - updating=False, - ): - """ - Return the populated containers for a repository being installed for the first time - or for an installed repository that is being updated and the updates include newly - defined repository (and possibly tool) dependencies. - """ - installed_tool_dependencies, missing_tool_dependencies = self.populate_tool_dependencies_dicts( - tool_shed_url=tool_shed_url, - tool_path=tool_path, - repository_installed_tool_dependencies=installed_tool_dependencies, - repository_missing_tool_dependencies=missing_tool_dependencies, - required_repo_info_dicts=None, - ) - # Most of the repository contents are set to None since we don't yet know what they are. - gucm = GalaxyUtilityContainerManager(self.app) - containers_dict = gucm.build_repository_containers( - repository=None, - invalid_tools=None, - missing_repository_dependencies=missing_repository_dependencies, - missing_tool_dependencies=missing_tool_dependencies, - readme_files_dict=readme_files_dict, - repository_dependencies=installed_repository_dependencies, - tool_dependencies=installed_tool_dependencies, - valid_tools=None, - valid_data_managers=None, - invalid_data_managers=None, - data_managers_errors=None, - new_install=True, - reinstalling=False, - ) - if not updating: - # If we installing a new repository and not updaing an installed repository, we can merge - # the missing_repository_dependencies container contents to the installed_repository_dependencies - # container. When updating an installed repository, merging will result in losing newly defined - # dependencies included in the updates. - containers_dict = self.merge_missing_repository_dependencies_to_installed_container(containers_dict) - # Merge the missing_tool_dependencies container contents to the installed_tool_dependencies container. - containers_dict = self.merge_missing_tool_dependencies_to_installed_container(containers_dict) - return containers_dict - - def populate_containers_dict_from_repository_metadata( - self, tool_shed_url, tool_path, repository, reinstalling=False, required_repo_info_dicts=None - ): - """ - Retrieve necessary information from the received repository's metadata to populate the - containers_dict for display. This method is called only from Galaxy (not the tool shed) - when displaying repository dependencies for installed repositories and when displaying - them for uninstalled repositories that are being reinstalled. - """ - metadata = repository.metadata_ - if metadata: - # Handle invalid tools. - invalid_tools = metadata.get("invalid_tools", None) - # Handle README files. - if repository.has_readme_files: - if reinstalling or repository.status not in [ - self.app.install_model.ToolShedRepository.installation_status.DEACTIVATED, - self.app.install_model.ToolShedRepository.installation_status.INSTALLED, - ]: - # Since we're reinstalling, we need to send a request to the tool shed to get the README files. - tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry(self.app, tool_shed_url) - params = dict( - name=str(repository.name), - owner=str(repository.owner), - changeset_revision=str(repository.installed_changeset_revision), - ) - pathspec = ["repository", "get_readme_files"] - raw_text = util.url_get( - tool_shed_url, - auth=self.app.tool_shed_registry.url_auth(tool_shed_url), - pathspec=pathspec, - params=params, - ) - readme_files_dict = json.loads(raw_text) - else: - readme_files_dict = readme_util.build_readme_files_dict( - self.app, repository, repository.changeset_revision, repository.metadata_, tool_path - ) - else: - readme_files_dict = None - # Handle repository dependencies. - ( - installed_repository_dependencies, - missing_repository_dependencies, - ) = self.app.installed_repository_manager.get_installed_and_missing_repository_dependencies(repository) - # Handle the current repository's tool dependencies. - repository_tool_dependencies = metadata.get("tool_dependencies", None) - # Make sure to display missing tool dependencies as well. - repository_invalid_tool_dependencies = metadata.get("invalid_tool_dependencies", None) - if repository_invalid_tool_dependencies is not None: - if repository_tool_dependencies is None: - repository_tool_dependencies = {} - repository_tool_dependencies.update(repository_invalid_tool_dependencies) - ( - repository_installed_tool_dependencies, - repository_missing_tool_dependencies, - ) = self.get_installed_and_missing_tool_dependencies_for_installed_repository( - repository, repository_tool_dependencies - ) - if reinstalling: - installed_tool_dependencies, missing_tool_dependencies = self.populate_tool_dependencies_dicts( - tool_shed_url, - tool_path, - repository_installed_tool_dependencies, - repository_missing_tool_dependencies, - required_repo_info_dicts, - ) - else: - installed_tool_dependencies = repository_installed_tool_dependencies - missing_tool_dependencies = repository_missing_tool_dependencies - # Handle valid tools. - valid_tools = metadata.get("tools", None) - # Handle Data Managers - valid_data_managers = None - invalid_data_managers = None - data_managers_errors = None - if "data_manager" in metadata: - valid_data_managers = metadata["data_manager"].get("data_managers", None) - invalid_data_managers = metadata["data_manager"].get("invalid_data_managers", None) - data_managers_errors = metadata["data_manager"].get("messages", None) - gucm = GalaxyUtilityContainerManager(self.app) - containers_dict = gucm.build_repository_containers( - repository=repository, - invalid_tools=invalid_tools, - missing_repository_dependencies=missing_repository_dependencies, - missing_tool_dependencies=missing_tool_dependencies, - readme_files_dict=readme_files_dict, - repository_dependencies=installed_repository_dependencies, - tool_dependencies=installed_tool_dependencies, - valid_tools=valid_tools, - valid_data_managers=valid_data_managers, - invalid_data_managers=invalid_data_managers, - data_managers_errors=data_managers_errors, - new_install=False, - reinstalling=reinstalling, - ) - else: - containers_dict = dict( - datatypes=None, - invalid_tools=None, - readme_files_dict=None, - repository_dependencies=None, - tool_dependencies=None, - valid_tools=None, - workflows=None, - ) - return containers_dict - - def populate_tool_dependencies_dicts( - self, - tool_shed_url, - tool_path, - repository_installed_tool_dependencies, - repository_missing_tool_dependencies, - required_repo_info_dicts, - ): - """ - Return the populated installed_tool_dependencies and missing_tool_dependencies dictionaries - for all repositories defined by entries in the received required_repo_info_dicts. - """ - installed_tool_dependencies = None - missing_tool_dependencies = None - if repository_installed_tool_dependencies is None: - repository_installed_tool_dependencies = {} - else: - # Add the install_dir attribute to the tool_dependencies. - repository_installed_tool_dependencies = self.add_installation_directories_to_tool_dependencies( - repository_installed_tool_dependencies - ) - if repository_missing_tool_dependencies is None: - repository_missing_tool_dependencies = {} - else: - # Add the install_dir attribute to the tool_dependencies. - repository_missing_tool_dependencies = self.add_installation_directories_to_tool_dependencies( - repository_missing_tool_dependencies - ) - if required_repo_info_dicts: - # Handle the tool dependencies defined for each of the repository's repository dependencies. - for rid in required_repo_info_dicts: - for name, repo_info_tuple in rid.items(): - ( - description, - repository_clone_url, - changeset_revision, - ctx_rev, - repository_owner, - repository_dependencies, - tool_dependencies, - ) = repository_util.get_repo_info_tuple_contents(repo_info_tuple) - if tool_dependencies: - # Add the install_dir attribute to the tool_dependencies. - tool_dependencies = self.add_installation_directories_to_tool_dependencies(tool_dependencies) - # The required_repository may have been installed with a different changeset revision. - ( - required_repository, - installed_changeset_revision, - ) = repository_util.repository_was_previously_installed( - self.app, tool_shed_url, name, repo_info_tuple, from_tip=False - ) - if required_repository: - ( - required_repository_installed_tool_dependencies, - required_repository_missing_tool_dependencies, - ) = self.get_installed_and_missing_tool_dependencies_for_installed_repository( - required_repository, tool_dependencies - ) - if required_repository_installed_tool_dependencies: - # Add the install_dir attribute to the tool_dependencies. - required_repository_installed_tool_dependencies = ( - self.add_installation_directories_to_tool_dependencies( - required_repository_installed_tool_dependencies - ) - ) - for td_key, td_dict in required_repository_installed_tool_dependencies.items(): - if td_key not in repository_installed_tool_dependencies: - repository_installed_tool_dependencies[td_key] = td_dict - if required_repository_missing_tool_dependencies: - # Add the install_dir attribute to the tool_dependencies. - required_repository_missing_tool_dependencies = ( - self.add_installation_directories_to_tool_dependencies( - required_repository_missing_tool_dependencies - ) - ) - for td_key, td_dict in required_repository_missing_tool_dependencies.items(): - if td_key not in repository_missing_tool_dependencies: - repository_missing_tool_dependencies[td_key] = td_dict - if repository_installed_tool_dependencies: - installed_tool_dependencies = repository_installed_tool_dependencies - if repository_missing_tool_dependencies: - missing_tool_dependencies = repository_missing_tool_dependencies - return installed_tool_dependencies, missing_tool_dependencies - - def tool_dependency_is_orphan(self, type, name, version, tools): - """ - Determine if the combination of the received type, name and version is defined in the - tag for at least one tool in the received list of tools. If not, the tool dependency defined by the - combination is considered an orphan in its repository in the tool shed. - """ - if type == "package": - if name and version: - for tool_dict in tools: - requirements = tool_dict.get("requirements", []) - for requirement_dict in requirements: - req_name = requirement_dict.get("name", None) - req_version = requirement_dict.get("version", None) - req_type = requirement_dict.get("type", None) - if req_name == name and req_version == version and req_type == type: - return False - elif type == "set_environment": - if name: - for tool_dict in tools: - requirements = tool_dict.get("requirements", []) - for requirement_dict in requirements: - req_name = requirement_dict.get("name", None) - req_type = requirement_dict.get("type", None) - if req_name == name and req_type == type: - return False - return True diff --git a/lib/tool_shed/galaxy_install/utility_containers/__init__.py b/lib/tool_shed/galaxy_install/utility_containers/__init__.py deleted file mode 100644 index 223d2bf1bde..00000000000 --- a/lib/tool_shed/galaxy_install/utility_containers/__init__.py +++ /dev/null @@ -1,138 +0,0 @@ -import logging -import threading - -from tool_shed.utility_containers import utility_container_manager - -log = logging.getLogger(__name__) - - -class GalaxyUtilityContainerManager(utility_container_manager.UtilityContainerManager): - def __init__(self, app): - self.app = app - - def build_repository_containers( - self, - repository, - invalid_tools, - missing_repository_dependencies, - missing_tool_dependencies, - readme_files_dict, - repository_dependencies, - tool_dependencies, - valid_tools, - valid_data_managers, - invalid_data_managers, - data_managers_errors, - new_install=False, - reinstalling=False, - ): - """ - Return a dictionary of containers for the received repository's dependencies and readme files for - display during installation to Galaxy. - """ - containers_dict = dict( - datatypes=None, - invalid_tools=None, - missing_tool_dependencies=None, - readme_files=None, - repository_dependencies=None, - missing_repository_dependencies=None, - tool_dependencies=None, - valid_tools=None, - workflows=None, - valid_data_managers=None, - invalid_data_managers=None, - ) - # Some of the tool dependency folders will include links to display tool dependency information, and - # some of these links require the repository id. However we need to be careful because sometimes the - # repository object is None. - if repository: - changeset_revision = repository.changeset_revision - else: - changeset_revision = None - lock = threading.Lock() - lock.acquire(True) - try: - folder_id = 0 - # Invalid tools container. - if invalid_tools: - folder_id, invalid_tools_root_folder = self.build_invalid_tools_folder( - folder_id, invalid_tools, changeset_revision, repository=repository, label="Invalid tools" - ) - containers_dict["invalid_tools"] = invalid_tools_root_folder - # Readme files container. - if readme_files_dict: - folder_id, readme_files_root_folder = self.build_readme_files_folder(folder_id, readme_files_dict) - containers_dict["readme_files"] = readme_files_root_folder - # Installed repository dependencies container. - if repository_dependencies: - if new_install: - label = "Repository dependencies" - else: - label = "Installed repository dependencies" - folder_id, repository_dependencies_root_folder = self.build_repository_dependencies_folder( - folder_id=folder_id, repository_dependencies=repository_dependencies, label=label, installed=True - ) - containers_dict["repository_dependencies"] = repository_dependencies_root_folder - # Missing repository dependencies container. - if missing_repository_dependencies: - folder_id, missing_repository_dependencies_root_folder = self.build_repository_dependencies_folder( - folder_id=folder_id, - repository_dependencies=missing_repository_dependencies, - label="Missing repository dependencies", - installed=False, - ) - containers_dict["missing_repository_dependencies"] = missing_repository_dependencies_root_folder - # Installed tool dependencies container. - if tool_dependencies: - if new_install: - label = "Tool dependencies" - else: - label = "Installed tool dependencies" - # We only want to display the Status column if the tool_dependency is missing. - folder_id, tool_dependencies_root_folder = self.build_tool_dependencies_folder( - folder_id, - tool_dependencies, - label=label, - missing=False, - new_install=new_install, - reinstalling=reinstalling, - ) - containers_dict["tool_dependencies"] = tool_dependencies_root_folder - # Missing tool dependencies container. - if missing_tool_dependencies: - # We only want to display the Status column if the tool_dependency is missing. - folder_id, missing_tool_dependencies_root_folder = self.build_tool_dependencies_folder( - folder_id, - missing_tool_dependencies, - label="Missing tool dependencies", - missing=True, - new_install=new_install, - reinstalling=reinstalling, - ) - containers_dict["missing_tool_dependencies"] = missing_tool_dependencies_root_folder - # Valid tools container. - if valid_tools: - folder_id, valid_tools_root_folder = self.build_tools_folder( - folder_id, valid_tools, repository, changeset_revision, label="Valid tools" - ) - containers_dict["valid_tools"] = valid_tools_root_folder - # Workflows container. - if valid_data_managers: - folder_id, valid_data_managers_root_folder = self.build_data_managers_folder( - folder_id=folder_id, data_managers=valid_data_managers, label="Valid Data Managers" - ) - containers_dict["valid_data_managers"] = valid_data_managers_root_folder - if invalid_data_managers or data_managers_errors: - folder_id, invalid_data_managers_root_folder = self.build_invalid_data_managers_folder( - folder_id=folder_id, - data_managers=invalid_data_managers, - error_messages=data_managers_errors, - label="Invalid Data Managers", - ) - containers_dict["invalid_data_managers"] = invalid_data_managers_root_folder - except Exception as e: - log.debug(f"Exception in build_repository_containers: {str(e)}") - finally: - lock.release() - return containers_dict diff --git a/lib/tool_shed/util/readme_util.py b/lib/tool_shed/util/readme_util.py index e573f4bd6cd..3e5cabaf900 100644 --- a/lib/tool_shed/util/readme_util.py +++ b/lib/tool_shed/util/readme_util.py @@ -4,20 +4,21 @@ import os from mako.template import Template -import tool_shed.util.shed_util_common as suc from galaxy import web +from galaxy.tool_shed.util import basic_util +from galaxy.tool_shed.util.hg_util import ( + get_changectx_for_changeset, + get_file_context_from_ctx, +) +from galaxy.tool_shed.util.repository_util import get_repo_info_tuple_contents +from galaxy.tool_shed.util.shed_util_common import set_image_paths from galaxy.util import ( rst_to_html, unicodify, url_get, ) -from tool_shed.util import ( - basic_util, - common_util, - hg_util, - metadata_util, - repository_util, -) +from galaxy.util.tool_shed.common_util import get_tool_shed_url_from_tool_shed_registry +from tool_shed.util.metadata_util import get_latest_downloadable_changeset_revision log = logging.getLogger(__name__) @@ -31,9 +32,7 @@ def build_readme_files_dict(app, repository, changeset_revision, metadata, tool_ if app.name == "galaxy": can_use_disk_files = True else: - latest_downloadable_changeset_revision = metadata_util.get_latest_downloadable_changeset_revision( - app, repository - ) + latest_downloadable_changeset_revision = get_latest_downloadable_changeset_revision(app, repository) can_use_disk_files = changeset_revision == latest_downloadable_changeset_revision readme_files_dict = {} if metadata: @@ -59,7 +58,7 @@ def build_readme_files_dict(app, repository, changeset_revision, metadata, tool_ if text_of_reasonable_length.find(".. image:: ") >= 0: # Handle image display for README files that are contained in repositories in the tool shed or installed into Galaxy. try: - text_of_reasonable_length = suc.set_image_paths( + text_of_reasonable_length = set_image_paths( app, text_of_reasonable_length, encoded_repository_id=app.security.encode_id(repository.id), @@ -85,9 +84,9 @@ def build_readme_files_dict(app, repository, changeset_revision, metadata, tool_ else: # We must be in the tool shed and have an old changeset_revision, so we need to retrieve the file contents from the repository manifest. repo = repository.hg_repo - ctx = hg_util.get_changectx_for_changeset(repo, changeset_revision) + ctx = get_changectx_for_changeset(repo, changeset_revision) if ctx: - fctx = hg_util.get_file_context_from_ctx(ctx, readme_file_name) + fctx = get_file_context_from_ctx(ctx, readme_file_name) if fctx and fctx not in ["DELETED"]: try: text = unicodify(fctx.data()) @@ -115,9 +114,9 @@ def get_readme_files_dict_for_display(app, tool_shed_url, repo_info_dict): repository_owner, repository_dependencies, installed_td, - ) = repository_util.get_repo_info_tuple_contents(repo_info_tuple) + ) = get_repo_info_tuple_contents(repo_info_tuple) # Handle changing HTTP protocols over time. - tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry(app, tool_shed_url) + tool_shed_url = get_tool_shed_url_from_tool_shed_registry(app, tool_shed_url) params = dict(name=name, owner=repository_owner, changeset_revision=changeset_revision) pathspec = ["repository", "get_readme_files"] raw_text = url_get( diff --git a/lib/tool_shed/utility_containers/__init__.py b/lib/tool_shed/utility_containers/__init__.py index 1058b3169a8..5d0c5b7700c 100644 --- a/lib/tool_shed/utility_containers/__init__.py +++ b/lib/tool_shed/utility_containers/__init__.py @@ -2,12 +2,13 @@ import logging import threading from galaxy import util -from tool_shed.util import ( - common_util, - container_util, - readme_util, +from galaxy.tool_shed.util import utility_container_manager +from galaxy.tool_shed.util.container_util import ( + generate_repository_dependencies_key_for_repository, + STRSEP, ) -from . import utility_container_manager +from galaxy.util.tool_shed.common_util import parse_repository_dependency_tuple +from tool_shed.util.readme_util import build_readme_files_dict log = logging.getLogger(__name__) @@ -166,8 +167,8 @@ class ToolShedUtilityContainerManager(utility_container_manager.UtilityContainer prior_installation_required, only_if_compiling_contained_td, error, - ) = common_util.parse_repository_dependency_tuple(invalid_repository_dependency, contains_error=True) - key = container_util.generate_repository_dependencies_key_for_repository( + ) = parse_repository_dependency_tuple(invalid_repository_dependency, contains_error=True) + key = generate_repository_dependencies_key_for_repository( toolshed, name, owner, @@ -300,9 +301,7 @@ class ToolShedUtilityContainerManager(utility_container_manager.UtilityContainer # Readme files container. if metadata: if "readme_files" not in exclude and "readme_files" in metadata: - readme_files_dict = readme_util.build_readme_files_dict( - self.app, repository, changeset_revision, metadata - ) + readme_files_dict = build_readme_files_dict(self.app, repository, changeset_revision, metadata) folder_id, readme_files_root_folder = self.build_readme_files_folder( folder_id, readme_files_dict ) @@ -363,4 +362,4 @@ class ToolShedUtilityContainerManager(utility_container_manager.UtilityContainer return containers_dict def generate_tool_dependencies_key(self, name, version, type): - return f"{str(name)}{container_util.STRSEP}{str(version)}{container_util.STRSEP}{str(type)}" + return f"{str(name)}{STRSEP}{str(version)}{STRSEP}{str(type)}" diff --git a/lib/tool_shed/webapp/controllers/repository.py b/lib/tool_shed/webapp/controllers/repository.py index bb694d404bf..bcbb6873d1d 100644 --- a/lib/tool_shed/webapp/controllers/repository.py +++ b/lib/tool_shed/webapp/controllers/repository.py @@ -24,6 +24,7 @@ from galaxy import ( util, web, ) +from galaxy.tool_shed.util import dependency_display from galaxy.tools.repositories import ValidationContext from galaxy.web.form_builder import ( CheckboxField, @@ -32,7 +33,6 @@ from galaxy.web.form_builder import ( from galaxy.web.legacy_framework import grids from galaxy.webapps.base.controller import BaseUIController from tool_shed.dependencies.repository import relation_builder -from tool_shed.galaxy_install import dependency_display from tool_shed.metadata import repository_metadata_manager from tool_shed.tools import ( tool_validator, diff --git a/lib/tool_shed/webapp/controllers/upload.py b/lib/tool_shed/webapp/controllers/upload.py index b4ea9b5631d..ef3ef730dd7 100644 --- a/lib/tool_shed/webapp/controllers/upload.py +++ b/lib/tool_shed/webapp/controllers/upload.py @@ -10,10 +10,10 @@ from galaxy import ( util, web, ) +from galaxy.tool_shed.util import dependency_display from galaxy.util import checkers from galaxy.webapps.base.controller import BaseUIController from tool_shed.dependencies import attribute_handlers -from tool_shed.galaxy_install import dependency_display from tool_shed.metadata import repository_metadata_manager from tool_shed.repository_types import util as rt_util from tool_shed.tools.data_table_manager import ShedToolDataTableManager diff --git a/templates/admin/tool_shed_repository/reselect_tool_panel_section.mako b/templates/admin/tool_shed_repository/reselect_tool_panel_section.mako deleted file mode 100644 index 447085d6a32..00000000000 --- a/templates/admin/tool_shed_repository/reselect_tool_panel_section.mako +++ /dev/null @@ -1,101 +0,0 @@ -<%inherit file="/base.mako"/> -<%namespace file="/message.mako" import="render_msg" /> -<%namespace file="/admin/tool_shed_repository/common.mako" import="render_dependencies_section" /> -<%namespace file="/admin/tool_shed_repository/common.mako" import="render_readme_section" /> -<%namespace file="/webapps/tool_shed/repository/common.mako" import="*" /> -<%namespace file="/webapps/tool_shed/common/common.mako" import="*" /> - -<%def name="stylesheets()"> - ${parent.stylesheets()} - ${h.css( "library" )} - - -<%def name="javascripts()"> - ${parent.javascripts()} - ${container_javascripts()} - - -%if message: - ${render_msg( message, status )} -%endif - -
-
-
-
- -
-
- <% readme_files_dict = containers_dict.get( 'readme_files', None ) %> - %if readme_files_dict: -
- - -
Repository README files - may contain important installation or license information
-
- ${render_readme_section( containers_dict )} -
- %endif - %if has_repository_dependencies or includes_tool_dependencies: -
- - -
Confirm dependency installation
-
- ${render_dependencies_section( install_resolver_dependencies_check_box, install_repository_dependencies_check_box, containers_dict, revision_label=None, export=False )} - %endif - %if shed_tool_conf_select_field: -
- - -
Choose the configuration file whose tool_path setting will be used for installing repositories
-
- <% - if len( shed_tool_conf_select_field.options ) == 1: - select_help = "Your Galaxy instance is configured with 1 shed-related tool configuration file, so repositories will be " - select_help += "installed using its tool_path setting." - else: - select_help = "Your Galaxy instance is configured with %d shed-related tool configuration files, " % len( shed_tool_conf_select_field.options ) - select_help += "so select the file whose tool_path setting you want used for installing repositories." - %> -
- - ${render_select(shed_tool_conf_select_field)} -
- ${select_help|h} -
-
-
- %else: - - %endif - %if includes_tools_for_display_in_tool_panel: -
-
- ${render_checkbox(no_changes_check_box)} - -
- Uncheck and select a different tool panel section to load the tools into a different section in the tool panel. -
-
-
- - -
- Add a new tool panel section to contain the installed tools (optional). -
-
-
- - ${render_select(tool_panel_section_select_field)} -
- Choose an existing section in your tool panel to contain the installed tools (optional). -
-
- %endif -
- -
-
-
-
diff --git a/templates/admin/tool_shed_repository/select_tool_panel_section.mako b/templates/admin/tool_shed_repository/select_tool_panel_section.mako deleted file mode 100644 index 4c980982984..00000000000 --- a/templates/admin/tool_shed_repository/select_tool_panel_section.mako +++ /dev/null @@ -1,188 +0,0 @@ -<%inherit file="/base.mako"/> -<%namespace file="/message.mako" import="render_msg" /> -<%namespace file="/admin/tool_shed_repository/common.mako" import="render_dependencies_section" /> -<%namespace file="/admin/tool_shed_repository/common.mako" import="render_readme_section" /> -<%namespace file="/webapps/tool_shed/repository/common.mako" import="*" /> -<%namespace file="/webapps/tool_shed/common/common.mako" import="*" /> - -<%def name="stylesheets()"> - ${parent.stylesheets()} - ${h.css( "library" )} - - -<%def name="javascripts()"> - ${parent.javascripts()} - ${container_javascripts()} - - -<% - # Handle the case where an uninstalled repository encountered errors during the process of being reinstalled. In - # this case, the repository metadata is an empty dictionary, but one or both of has_repository_dependencies - # and includes_tool_dependencies may be True. If either of these are True but we have no metadata, we cannot install - # repository dependencies on this pass. - if has_repository_dependencies: - repository_dependencies = containers_dict[ 'repository_dependencies' ] - missing_repository_dependencies = containers_dict[ 'missing_repository_dependencies' ] - if repository_dependencies or missing_repository_dependencies: - can_display_repository_dependencies = True - else: - can_display_repository_dependencies = False - else: - can_display_repository_dependencies = False - if includes_tool_dependencies: - tool_dependencies = containers_dict[ 'tool_dependencies' ] - missing_tool_dependencies = containers_dict[ 'missing_tool_dependencies' ] - if tool_dependencies or missing_tool_dependencies: - can_display_tool_dependencies = True - else: - can_display_tool_dependencies = False - else: - can_display_tool_dependencies = False - can_display_resolver_installation = install_resolver_dependencies_check_box is not None -%> - -%if message: - ${render_msg( message, status )} -%endif - -
-

- The Galaxy development team does not maintain the contents of many Galaxy Tool Shed repositories. Some - repository tools may include code that produces malicious behavior, so be aware of what you are installing. -

-

- If you discover a repository that causes problems after installation, contact Galaxy support, - sending all necessary information, and appropriate action will be taken. -

-

- Contact the repository owner for - general questions or concerns. -

-
-
-
-
-
- - - - - - - - - - - -
-
- <% readme_files_dict = containers_dict.get( 'readme_files', None ) %> - %if readme_files_dict: -
- - -
Repository README file - may contain important installation or license information
-
- ${render_readme_section( containers_dict )} -
- %endif - <% - if requirements_status and install_resolver_dependencies_check_box or includes_tool_dependencies: - display_dependency_confirmation = True - else: - display_dependency_confirmation = False - %> - %if requirements_status: - %if not install_resolver_dependencies_check_box and not includes_tool_dependencies: -
- - - - -
- - This repository requires dependencies that cannot be installed through the Tool Shed -
-
-
-

This repository defines tool requirements that cannot be installed through the Tool Shed.

-

Please activate Conda dependency resolution, activate Docker dependency resolution, setup Environment Modules -or manually satisfy the dependencies listed below.

-

For details see the dependency resolver documentation.

-
- %endif -
- - -
The following tool dependencies are required by the current repository
-
-
- ${render_tool_dependency_resolver( requirements_status, prepare_for_install=True )} -
-
- %endif - %if can_display_repository_dependencies or display_dependency_confirmation: -
- - -
Confirm dependency installation
-
- ${render_dependencies_section( install_resolver_dependencies_check_box, install_repository_dependencies_check_box, containers_dict, revision_label=None, export=False, requirements_status=requirements_status )} -
- %endif - %if shed_tool_conf_select_field: - %if includes_tools_for_display_in_tool_panel: -
- - -
Choose the tool panel section to contain the installed tools (optional)
-
-
- %endif - <% - if len( shed_tool_conf_select_field.options ) == 1: - select_help = "Your Galaxy instance is configured with 1 shed-related tool configuration file, so repositories will be " - select_help += "installed using its tool_path setting." - else: - select_help = "Your Galaxy instance is configured with %d shed-related tool configuration files, " % len( shed_tool_conf_select_field.options ) - select_help += "so select the file whose tool_path setting you want used for installing repositories." - %> -
- - ${render_select(shed_tool_conf_select_field)} -
- ${select_help|h} -
-
-
-
- %else: - - %endif - %if includes_tools_for_display_in_tool_panel: -
- - -
- Add a new tool panel section to contain the installed tools (optional). -
-
-
- - ${render_select(tool_panel_section_select_field)} -
- Choose an existing section in your tool panel to contain the installed tools (optional). -
-
- %endif -
- -
- %if includes_tools_for_display_in_tool_panel: - Clicking Install without selecting a tool panel section will load the installed tools into the tool panel outside of any sections. - %endif -
-
-
-
-