From 533e3f9be0375dd1e3bde780553e1693f16a120d Mon Sep 17 00:00:00 2001 From: John Chilton Date: Mon, 17 Oct 2022 14:16:03 -0400 Subject: [PATCH 01/19] Remove reselect_tool_panel mako and controller endpoint. --- .../galaxy/controllers/admin_toolshed.py | 170 ---------------- .../reselect_tool_panel_section.mako | 101 ---------- .../select_tool_panel_section.mako | 188 ------------------ 3 files changed, 459 deletions(-) delete mode 100644 templates/admin/tool_shed_repository/reselect_tool_panel_section.mako delete mode 100644 templates/admin/tool_shed_repository/select_tool_panel_section.mako diff --git a/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py b/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py index 4616f761d18..da081c95a64 100644 --- a/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py +++ b/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py @@ -12,7 +12,6 @@ from galaxy.exceptions import ConfigDoesNotAllowException from galaxy.tool_shed.galaxy_install import install_manager from galaxy.tool_shed.galaxy_install.repository_dependencies import repository_dependency_manager from galaxy.tool_shed.galaxy_install.tools import tool_panel_manager -from galaxy.tool_shed.util import tool_util from galaxy.tool_shed.util.repository_util import ( create_or_update_tool_shed_repository, get_absolute_path_to_file_in_repository, @@ -368,172 +367,3 @@ class AdminToolshed(AdminGalaxy): except Exception: log.exception("Problem you know in here somewhere...") raise - - @web.expose - @web.require_admin - @legacy_tool_shed_endpoint - def reselect_tool_panel_section(self, trans, **kwd): - """ - Select or change the tool panel section to contain the tools included in the tool shed repository - being reinstalled. If there are updates available for the repository in the tool shed, the - tool_dependencies and repository_dependencies associated with the updated changeset revision will - have been retrieved from the tool shed and passed in the received kwd. In this case, the stored - tool shed repository metadata from the Galaxy database will not be used since it is outdated. - """ - message = "" - status = "done" - repository_id = kwd.get("id", None) - latest_changeset_revision = kwd.get("latest_changeset_revision", None) - latest_ctx_rev = kwd.get("latest_ctx_rev", None) - tool_shed_repository = get_installed_tool_shed_repository(trans.app, repository_id) - repository_clone_url = common_util.generate_clone_url_for_installed_repository(trans.app, tool_shed_repository) - metadata = tool_shed_repository.metadata_ - tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( - trans.app, str(tool_shed_repository.tool_shed) - ) - tool_path = tool_shed_repository.get_tool_relative_path(trans.app)[0] - if latest_changeset_revision and latest_ctx_rev: - # There are updates available in the tool shed for the repository, so use the received - # dependency information which was retrieved from the tool shed. - encoded_updated_repo_info_dict = kwd.get("updated_repo_info_dict", None) - updated_repo_info_dict = encoding_util.tool_shed_decode(encoded_updated_repo_info_dict) - readme_files_dict = updated_repo_info_dict.get("readme_files_dict", None) - includes_data_managers = updated_repo_info_dict.get("includes_data_managers", False) - includes_tool_dependencies = updated_repo_info_dict.get("includes_tool_dependencies", False) - repo_info_dict = updated_repo_info_dict["repo_info_dict"] - else: - # There are no updates available from the tool shed for the repository, so use its locally stored metadata. - includes_data_managers = False - includes_tool_dependencies = False - readme_files_dict = None - tool_dependencies = None - if metadata: - if "data_manager" in metadata: - includes_data_managers = True - if "tool_dependencies" in metadata: - includes_tool_dependencies = True - # Since we're reinstalling, we need to send a request to the tool shed to get the README files. - params = dict( - name=tool_shed_repository.name, - owner=tool_shed_repository.owner, - changeset_revision=tool_shed_repository.installed_changeset_revision, - ) - pathspec = ["repository", "get_readme_files"] - raw_text = util.url_get( - tool_shed_url, - auth=self.app.tool_shed_registry.url_auth(tool_shed_url), - pathspec=pathspec, - params=params, - ) - readme_files_dict = json.loads(raw_text) - tool_dependencies = metadata.get("tool_dependencies", None) - rdim = repository_dependency_manager.RepositoryDependencyInstallManager(trans.app) - repository_dependencies = rdim.get_repository_dependencies_for_installed_tool_shed_repository( - trans.app, tool_shed_repository - ) - repo_info_dict = create_repo_info_dict( - trans.app, - repository_clone_url=repository_clone_url, - changeset_revision=tool_shed_repository.installed_changeset_revision, - ctx_rev=tool_shed_repository.ctx_rev, - repository_owner=tool_shed_repository.owner, - repository_name=tool_shed_repository.name, - tool_dependencies=tool_dependencies, - repository_dependencies=repository_dependencies, - ) - irm = trans.app.installed_repository_manager - dependencies_for_repository_dict = irm.get_dependencies_for_repository( - tool_shed_url, repo_info_dict, includes_tool_dependencies, updating=True - ) - includes_tool_dependencies = dependencies_for_repository_dict.get("includes_tool_dependencies", False) - includes_tools = dependencies_for_repository_dict.get("includes_tools", False) - includes_tools_for_display_in_tool_panel = dependencies_for_repository_dict.get( - "includes_tools_for_display_in_tool_panel", False - ) - installed_repository_dependencies = dependencies_for_repository_dict.get( - "installed_repository_dependencies", None - ) - installed_tool_dependencies = dependencies_for_repository_dict.get("installed_tool_dependencies", None) - missing_repository_dependencies = dependencies_for_repository_dict.get("missing_repository_dependencies", None) - missing_tool_dependencies = dependencies_for_repository_dict.get("missing_tool_dependencies", None) - if installed_repository_dependencies or missing_repository_dependencies: - has_repository_dependencies = True - else: - has_repository_dependencies = False - if includes_tools_for_display_in_tool_panel: - # Get the location in the tool panel in which the tools were originally loaded. - if "tool_panel_section" in metadata: - tool_panel_dict = metadata["tool_panel_section"] - if tool_panel_dict: - if tool_util.panel_entry_per_tool(tool_panel_dict): - # The following forces everything to be loaded into 1 section (or no section) in the tool panel. - tool_section_dicts = tool_panel_dict[next(iter(tool_panel_dict.keys()))] - tool_section_dict = tool_section_dicts[0] - original_section_name = tool_section_dict["name"] - else: - original_section_name = tool_panel_dict["name"] - else: - original_section_name = "" - else: - original_section_name = "" - tool_panel_section_select_field = tool_util.build_tool_panel_section_select_field(trans.app) - no_changes_check_box = CheckboxField("no_changes", value=True) - if original_section_name: - message += ( - "The tools contained in your %s repository were last loaded into the tool panel section %s. " - % (escape(tool_shed_repository.name), original_section_name) - ) - message += "Uncheck the No changes check box and select a different tool panel section to load the tools in a " - message += "different section in the tool panel. " - status = "warning" - else: - message += f"The tools contained in your {escape(tool_shed_repository.name)} repository were last loaded into the tool panel outside of any sections. " - message += "Uncheck the No changes check box and select a tool panel section to load the tools into that section. " - status = "warning" - else: - no_changes_check_box = None - original_section_name = "" - tool_panel_section_select_field = None - shed_tool_conf_select_field = tool_util.build_shed_tool_conf_select_field(trans.app) - dd = dependency_display.DependencyDisplayer(trans.app) - containers_dict = dd.populate_containers_dict_for_new_install( - tool_shed_url=tool_shed_url, - tool_path=tool_path, - readme_files_dict=readme_files_dict, - installed_repository_dependencies=installed_repository_dependencies, - missing_repository_dependencies=missing_repository_dependencies, - installed_tool_dependencies=installed_tool_dependencies, - missing_tool_dependencies=missing_tool_dependencies, - updating=False, - ) - # Since we're reinstalling we'll merge the list of missing repository dependencies into the list of - # installed repository dependencies since each displayed repository dependency will display a status, - # whether installed or missing. - containers_dict = dd.merge_missing_repository_dependencies_to_installed_container(containers_dict) - # Handle repository dependencies check box. - install_repository_dependencies_check_box = CheckboxField("install_repository_dependencies", value=True) - view = views.DependencyResolversView(self.app) - if view.installable_resolvers: - install_resolver_dependencies_check_box = CheckboxField("install_resolver_dependencies", value=True) - else: - install_resolver_dependencies_check_box = None - return trans.fill_template( - "/admin/tool_shed_repository/reselect_tool_panel_section.mako", - repository=tool_shed_repository, - no_changes_check_box=no_changes_check_box, - original_section_name=original_section_name, - includes_data_managers=includes_data_managers, - includes_tools=includes_tools, - includes_tools_for_display_in_tool_panel=includes_tools_for_display_in_tool_panel, - includes_tool_dependencies=includes_tool_dependencies, - has_repository_dependencies=has_repository_dependencies, - install_repository_dependencies_check_box=install_repository_dependencies_check_box, - install_resolver_dependencies_check_box=install_resolver_dependencies_check_box, - containers_dict=containers_dict, - tool_panel_section_select_field=tool_panel_section_select_field, - shed_tool_conf_select_field=shed_tool_conf_select_field, - encoded_repo_info_dict=encoding_util.tool_shed_encode(repo_info_dict), - repo_info_dict=repo_info_dict, - message=message, - status=status, - ) diff --git a/templates/admin/tool_shed_repository/reselect_tool_panel_section.mako b/templates/admin/tool_shed_repository/reselect_tool_panel_section.mako deleted file mode 100644 index 447085d6a32..00000000000 --- a/templates/admin/tool_shed_repository/reselect_tool_panel_section.mako +++ /dev/null @@ -1,101 +0,0 @@ -<%inherit file="/base.mako"/> -<%namespace file="/message.mako" import="render_msg" /> -<%namespace file="/admin/tool_shed_repository/common.mako" import="render_dependencies_section" /> -<%namespace file="/admin/tool_shed_repository/common.mako" import="render_readme_section" /> -<%namespace file="/webapps/tool_shed/repository/common.mako" import="*" /> -<%namespace file="/webapps/tool_shed/common/common.mako" import="*" /> - -<%def name="stylesheets()"> - ${parent.stylesheets()} - ${h.css( "library" )} - - -<%def name="javascripts()"> - ${parent.javascripts()} - ${container_javascripts()} - - -%if message: - ${render_msg( message, status )} -%endif - -
-
-
-
- -
-
- <% readme_files_dict = containers_dict.get( 'readme_files', None ) %> - %if readme_files_dict: -
- - -
Repository README files - may contain important installation or license information
-
- ${render_readme_section( containers_dict )} -
- %endif - %if has_repository_dependencies or includes_tool_dependencies: -
- - -
Confirm dependency installation
-
- ${render_dependencies_section( install_resolver_dependencies_check_box, install_repository_dependencies_check_box, containers_dict, revision_label=None, export=False )} - %endif - %if shed_tool_conf_select_field: -
- - -
Choose the configuration file whose tool_path setting will be used for installing repositories
-
- <% - if len( shed_tool_conf_select_field.options ) == 1: - select_help = "Your Galaxy instance is configured with 1 shed-related tool configuration file, so repositories will be " - select_help += "installed using its tool_path setting." - else: - select_help = "Your Galaxy instance is configured with %d shed-related tool configuration files, " % len( shed_tool_conf_select_field.options ) - select_help += "so select the file whose tool_path setting you want used for installing repositories." - %> -
- - ${render_select(shed_tool_conf_select_field)} -
- ${select_help|h} -
-
-
- %else: - - %endif - %if includes_tools_for_display_in_tool_panel: -
-
- ${render_checkbox(no_changes_check_box)} - -
- Uncheck and select a different tool panel section to load the tools into a different section in the tool panel. -
-
-
- - -
- Add a new tool panel section to contain the installed tools (optional). -
-
-
- - ${render_select(tool_panel_section_select_field)} -
- Choose an existing section in your tool panel to contain the installed tools (optional). -
-
- %endif -
- -
-
-
-
diff --git a/templates/admin/tool_shed_repository/select_tool_panel_section.mako b/templates/admin/tool_shed_repository/select_tool_panel_section.mako deleted file mode 100644 index 4c980982984..00000000000 --- a/templates/admin/tool_shed_repository/select_tool_panel_section.mako +++ /dev/null @@ -1,188 +0,0 @@ -<%inherit file="/base.mako"/> -<%namespace file="/message.mako" import="render_msg" /> -<%namespace file="/admin/tool_shed_repository/common.mako" import="render_dependencies_section" /> -<%namespace file="/admin/tool_shed_repository/common.mako" import="render_readme_section" /> -<%namespace file="/webapps/tool_shed/repository/common.mako" import="*" /> -<%namespace file="/webapps/tool_shed/common/common.mako" import="*" /> - -<%def name="stylesheets()"> - ${parent.stylesheets()} - ${h.css( "library" )} - - -<%def name="javascripts()"> - ${parent.javascripts()} - ${container_javascripts()} - - -<% - # Handle the case where an uninstalled repository encountered errors during the process of being reinstalled. In - # this case, the repository metadata is an empty dictionary, but one or both of has_repository_dependencies - # and includes_tool_dependencies may be True. If either of these are True but we have no metadata, we cannot install - # repository dependencies on this pass. - if has_repository_dependencies: - repository_dependencies = containers_dict[ 'repository_dependencies' ] - missing_repository_dependencies = containers_dict[ 'missing_repository_dependencies' ] - if repository_dependencies or missing_repository_dependencies: - can_display_repository_dependencies = True - else: - can_display_repository_dependencies = False - else: - can_display_repository_dependencies = False - if includes_tool_dependencies: - tool_dependencies = containers_dict[ 'tool_dependencies' ] - missing_tool_dependencies = containers_dict[ 'missing_tool_dependencies' ] - if tool_dependencies or missing_tool_dependencies: - can_display_tool_dependencies = True - else: - can_display_tool_dependencies = False - else: - can_display_tool_dependencies = False - can_display_resolver_installation = install_resolver_dependencies_check_box is not None -%> - -%if message: - ${render_msg( message, status )} -%endif - -
-

- The Galaxy development team does not maintain the contents of many Galaxy Tool Shed repositories. Some - repository tools may include code that produces malicious behavior, so be aware of what you are installing. -

-

- If you discover a repository that causes problems after installation, contact Galaxy support, - sending all necessary information, and appropriate action will be taken. -

-

- Contact the repository owner for - general questions or concerns. -

-
-
-
-
-
- - - - - - - - - - - -
-
- <% readme_files_dict = containers_dict.get( 'readme_files', None ) %> - %if readme_files_dict: -
- - -
Repository README file - may contain important installation or license information
-
- ${render_readme_section( containers_dict )} -
- %endif - <% - if requirements_status and install_resolver_dependencies_check_box or includes_tool_dependencies: - display_dependency_confirmation = True - else: - display_dependency_confirmation = False - %> - %if requirements_status: - %if not install_resolver_dependencies_check_box and not includes_tool_dependencies: -
- - - - -
- - This repository requires dependencies that cannot be installed through the Tool Shed -
-
-
-

This repository defines tool requirements that cannot be installed through the Tool Shed.

-

Please activate Conda dependency resolution, activate Docker dependency resolution, setup Environment Modules -or manually satisfy the dependencies listed below.

-

For details see the dependency resolver documentation.

-
- %endif -
- - -
The following tool dependencies are required by the current repository
-
-
- ${render_tool_dependency_resolver( requirements_status, prepare_for_install=True )} -
-
- %endif - %if can_display_repository_dependencies or display_dependency_confirmation: -
- - -
Confirm dependency installation
-
- ${render_dependencies_section( install_resolver_dependencies_check_box, install_repository_dependencies_check_box, containers_dict, revision_label=None, export=False, requirements_status=requirements_status )} -
- %endif - %if shed_tool_conf_select_field: - %if includes_tools_for_display_in_tool_panel: -
- - -
Choose the tool panel section to contain the installed tools (optional)
-
-
- %endif - <% - if len( shed_tool_conf_select_field.options ) == 1: - select_help = "Your Galaxy instance is configured with 1 shed-related tool configuration file, so repositories will be " - select_help += "installed using its tool_path setting." - else: - select_help = "Your Galaxy instance is configured with %d shed-related tool configuration files, " % len( shed_tool_conf_select_field.options ) - select_help += "so select the file whose tool_path setting you want used for installing repositories." - %> -
- - ${render_select(shed_tool_conf_select_field)} -
- ${select_help|h} -
-
-
-
- %else: - - %endif - %if includes_tools_for_display_in_tool_panel: -
- - -
- Add a new tool panel section to contain the installed tools (optional). -
-
-
- - ${render_select(tool_panel_section_select_field)} -
- Choose an existing section in your tool panel to contain the installed tools (optional). -
-
- %endif -
- -
- %if includes_tools_for_display_in_tool_panel: - Clicking Install without selecting a tool panel section will load the installed tools into the tool panel outside of any sections. - %endif -
-
-
-
-
From 0d61a37a71e693adbcc9dfbb6aab622a8b2e0ea9 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Mon, 17 Oct 2022 14:23:50 -0400 Subject: [PATCH 02/19] Remove now unused function from dependency_display. --- .../galaxy_install/dependency_display.py | 95 ------------------- 1 file changed, 95 deletions(-) diff --git a/lib/tool_shed/galaxy_install/dependency_display.py b/lib/tool_shed/galaxy_install/dependency_display.py index 5c4c8cf0e56..cc3c06672f5 100644 --- a/lib/tool_shed/galaxy_install/dependency_display.py +++ b/lib/tool_shed/galaxy_install/dependency_display.py @@ -7,12 +7,10 @@ from galaxy import util from tool_shed.galaxy_install.utility_containers import GalaxyUtilityContainerManager from tool_shed.util import ( common_util, - container_util, readme_util, repository_util, tool_dependency_util, ) -from tool_shed.utility_containers import utility_container_manager log = logging.getLogger(__name__) @@ -259,99 +257,6 @@ class DependencyDisplayer: missing_tool_dependencies = None return tool_dependencies, missing_tool_dependencies - def merge_containers_dicts_for_new_install(self, containers_dicts): - """ - When installing one or more tool shed repositories for the first time, the received list of - containers_dicts contains a containers_dict for each repository being installed. Since the - repositories are being installed for the first time, all entries are None except the repository - dependencies and tool dependencies. The entries for missing dependencies are all None since - they have previously been merged into the installed dependencies. This method will merge the - dependencies entries into a single container and return it for display. - """ - new_containers_dict = dict( - readme_files=None, - datatypes=None, - missing_repository_dependencies=None, - repository_dependencies=None, - missing_tool_dependencies=None, - tool_dependencies=None, - invalid_tools=None, - valid_tools=None, - workflows=None, - ) - if containers_dicts: - lock = threading.Lock() - lock.acquire(True) - try: - repository_dependencies_root_folder = None - tool_dependencies_root_folder = None - # Use a unique folder id (hopefully the following is). - folder_id = 867 - for old_container_dict in containers_dicts: - # Merge repository_dependencies. - old_container_repository_dependencies_root = old_container_dict["repository_dependencies"] - if old_container_repository_dependencies_root: - if repository_dependencies_root_folder is None: - repository_dependencies_root_folder = utility_container_manager.Folder( - id=folder_id, key="root", label="root", parent=None - ) - folder_id += 1 - repository_dependencies_folder = utility_container_manager.Folder( - id=folder_id, - key="merged", - label="Repository dependencies", - parent=repository_dependencies_root_folder, - ) - folder_id += 1 - # The old_container_repository_dependencies_root will be a root folder containing a single sub_folder. - old_container_repository_dependencies_folder = ( - old_container_repository_dependencies_root.folders[0] - ) - # Change the folder id so it won't confict with others being merged. - old_container_repository_dependencies_folder.id = folder_id - folder_id += 1 - repository_components_tuple = container_util.get_components_from_key( - old_container_repository_dependencies_folder.key - ) - components_list = repository_util.extract_components_from_tuple(repository_components_tuple) - name = components_list[1] - # Generate the label by retrieving the repository name. - old_container_repository_dependencies_folder.label = str(name) - repository_dependencies_folder.folders.append(old_container_repository_dependencies_folder) - # Merge tool_dependencies. - old_container_tool_dependencies_root = old_container_dict["tool_dependencies"] - if old_container_tool_dependencies_root: - if tool_dependencies_root_folder is None: - tool_dependencies_root_folder = utility_container_manager.Folder( - id=folder_id, key="root", label="root", parent=None - ) - folder_id += 1 - tool_dependencies_folder = utility_container_manager.Folder( - id=folder_id, - key="merged", - label="Tool dependencies", - parent=tool_dependencies_root_folder, - ) - folder_id += 1 - else: - td_list = [td.listify for td in tool_dependencies_folder.tool_dependencies] - # The old_container_tool_dependencies_root will be a root folder containing a single sub_folder. - old_container_tool_dependencies_folder = old_container_tool_dependencies_root.folders[0] - for td in old_container_tool_dependencies_folder.tool_dependencies: - if td.listify not in td_list: - tool_dependencies_folder.tool_dependencies.append(td) - if repository_dependencies_root_folder: - repository_dependencies_root_folder.folders.append(repository_dependencies_folder) - new_containers_dict["repository_dependencies"] = repository_dependencies_root_folder - if tool_dependencies_root_folder: - tool_dependencies_root_folder.folders.append(tool_dependencies_folder) - new_containers_dict["tool_dependencies"] = tool_dependencies_root_folder - except Exception as e: - log.debug(f"Exception in merge_containers_dicts_for_new_install: {e}") - finally: - lock.release() - return new_containers_dict - def merge_missing_repository_dependencies_to_installed_container(self, containers_dict): """ Merge the list of missing repository dependencies into the list of installed From 03f3f0ee0a85fc3019e4b6d53b2ac6f7152549f9 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Mon, 17 Oct 2022 14:37:41 -0400 Subject: [PATCH 03/19] Improved dependency imports in readme_util. --- lib/tool_shed/util/readme_util.py | 31 +++++++++++++++---------------- 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/lib/tool_shed/util/readme_util.py b/lib/tool_shed/util/readme_util.py index e573f4bd6cd..3e5cabaf900 100644 --- a/lib/tool_shed/util/readme_util.py +++ b/lib/tool_shed/util/readme_util.py @@ -4,20 +4,21 @@ import os from mako.template import Template -import tool_shed.util.shed_util_common as suc from galaxy import web +from galaxy.tool_shed.util import basic_util +from galaxy.tool_shed.util.hg_util import ( + get_changectx_for_changeset, + get_file_context_from_ctx, +) +from galaxy.tool_shed.util.repository_util import get_repo_info_tuple_contents +from galaxy.tool_shed.util.shed_util_common import set_image_paths from galaxy.util import ( rst_to_html, unicodify, url_get, ) -from tool_shed.util import ( - basic_util, - common_util, - hg_util, - metadata_util, - repository_util, -) +from galaxy.util.tool_shed.common_util import get_tool_shed_url_from_tool_shed_registry +from tool_shed.util.metadata_util import get_latest_downloadable_changeset_revision log = logging.getLogger(__name__) @@ -31,9 +32,7 @@ def build_readme_files_dict(app, repository, changeset_revision, metadata, tool_ if app.name == "galaxy": can_use_disk_files = True else: - latest_downloadable_changeset_revision = metadata_util.get_latest_downloadable_changeset_revision( - app, repository - ) + latest_downloadable_changeset_revision = get_latest_downloadable_changeset_revision(app, repository) can_use_disk_files = changeset_revision == latest_downloadable_changeset_revision readme_files_dict = {} if metadata: @@ -59,7 +58,7 @@ def build_readme_files_dict(app, repository, changeset_revision, metadata, tool_ if text_of_reasonable_length.find(".. image:: ") >= 0: # Handle image display for README files that are contained in repositories in the tool shed or installed into Galaxy. try: - text_of_reasonable_length = suc.set_image_paths( + text_of_reasonable_length = set_image_paths( app, text_of_reasonable_length, encoded_repository_id=app.security.encode_id(repository.id), @@ -85,9 +84,9 @@ def build_readme_files_dict(app, repository, changeset_revision, metadata, tool_ else: # We must be in the tool shed and have an old changeset_revision, so we need to retrieve the file contents from the repository manifest. repo = repository.hg_repo - ctx = hg_util.get_changectx_for_changeset(repo, changeset_revision) + ctx = get_changectx_for_changeset(repo, changeset_revision) if ctx: - fctx = hg_util.get_file_context_from_ctx(ctx, readme_file_name) + fctx = get_file_context_from_ctx(ctx, readme_file_name) if fctx and fctx not in ["DELETED"]: try: text = unicodify(fctx.data()) @@ -115,9 +114,9 @@ def get_readme_files_dict_for_display(app, tool_shed_url, repo_info_dict): repository_owner, repository_dependencies, installed_td, - ) = repository_util.get_repo_info_tuple_contents(repo_info_tuple) + ) = get_repo_info_tuple_contents(repo_info_tuple) # Handle changing HTTP protocols over time. - tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry(app, tool_shed_url) + tool_shed_url = get_tool_shed_url_from_tool_shed_registry(app, tool_shed_url) params = dict(name=name, owner=repository_owner, changeset_revision=changeset_revision) pathspec = ["repository", "get_readme_files"] raw_text = url_get( From 6915c5c5561dfbe169b7b1dc074ea4269e1cab25 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Mon, 17 Oct 2022 14:43:49 -0400 Subject: [PATCH 04/19] Cleanup up imports in dependency_display.py. --- .../galaxy_install/dependency_display.py | 36 +++++++++++-------- 1 file changed, 21 insertions(+), 15 deletions(-) diff --git a/lib/tool_shed/galaxy_install/dependency_display.py b/lib/tool_shed/galaxy_install/dependency_display.py index cc3c06672f5..bde25e80488 100644 --- a/lib/tool_shed/galaxy_install/dependency_display.py +++ b/lib/tool_shed/galaxy_install/dependency_display.py @@ -1,16 +1,22 @@ import json import logging import os -import threading from galaxy import util -from tool_shed.galaxy_install.utility_containers import GalaxyUtilityContainerManager -from tool_shed.util import ( - common_util, - readme_util, - repository_util, - tool_dependency_util, +from galaxy.tool_shed.util.common_util import ( + get_tool_shed_url_from_tool_shed_registry, + parse_repository_dependency_tuple, ) +from galaxy.tool_shed.util.repository_util import ( + get_repo_info_tuple_contents, + repository_was_previously_installed, +) +from galaxy.tool_shed.util.tool_dependency_util import ( + get_tool_dependency_by_name_type_repository, + get_tool_dependency_by_name_version_type_repository, +) +from tool_shed.galaxy_install.utility_containers import GalaxyUtilityContainerManager +from tool_shed.util.readme_util import build_readme_files_dict log = logging.getLogger(__name__) @@ -70,7 +76,7 @@ class DependencyDisplayer: prior_installation_required, only_if_compiling_contained_td, error, - ) = common_util.parse_repository_dependency_tuple( + ) = parse_repository_dependency_tuple( repository_dependency_tup, contains_error=True ) if error: @@ -103,7 +109,7 @@ class DependencyDisplayer: changeset_revision, pir, oicct, - ) = common_util.parse_repository_dependency_tuple(rd_tup) + ) = parse_repository_dependency_tuple(rd_tup) if util.asbool(pir): pir_str = "True" else: @@ -211,7 +217,7 @@ class DependencyDisplayer: name = td_info_dict["name"] version = None type = td_info_dict["type"] - tool_dependency = tool_dependency_util.get_tool_dependency_by_name_type_repository( + tool_dependency = get_tool_dependency_by_name_type_repository( self.app, repository, name, type ) if tool_dependency: @@ -234,7 +240,7 @@ class DependencyDisplayer: name = val["name"] version = val["version"] type = val["type"] - tool_dependency = tool_dependency_util.get_tool_dependency_by_name_version_type_repository( + tool_dependency = get_tool_dependency_by_name_version_type_repository( self.app, repository, name, version, type ) if tool_dependency: @@ -391,7 +397,7 @@ class DependencyDisplayer: self.app.install_model.ToolShedRepository.installation_status.INSTALLED, ]: # Since we're reinstalling, we need to send a request to the tool shed to get the README files. - tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry(self.app, tool_shed_url) + tool_shed_url = get_tool_shed_url_from_tool_shed_registry(self.app, tool_shed_url) params = dict( name=str(repository.name), owner=str(repository.owner), @@ -406,7 +412,7 @@ class DependencyDisplayer: ) readme_files_dict = json.loads(raw_text) else: - readme_files_dict = readme_util.build_readme_files_dict( + readme_files_dict = build_readme_files_dict( self.app, repository, repository.changeset_revision, repository.metadata_, tool_path ) else: @@ -519,7 +525,7 @@ class DependencyDisplayer: repository_owner, repository_dependencies, tool_dependencies, - ) = repository_util.get_repo_info_tuple_contents(repo_info_tuple) + ) = get_repo_info_tuple_contents(repo_info_tuple) if tool_dependencies: # Add the install_dir attribute to the tool_dependencies. tool_dependencies = self.add_installation_directories_to_tool_dependencies(tool_dependencies) @@ -527,7 +533,7 @@ class DependencyDisplayer: ( required_repository, installed_changeset_revision, - ) = repository_util.repository_was_previously_installed( + ) = repository_was_previously_installed( self.app, tool_shed_url, name, repo_info_tuple, from_tip=False ) if required_repository: From 2dd9065926ab4e01ed7e165bf51d1bcd4a49a2ee Mon Sep 17 00:00:00 2001 From: John Chilton Date: Mon, 17 Oct 2022 14:56:34 -0400 Subject: [PATCH 05/19] Mark dependency_display internal-only methods as such. --- .../galaxy_install/dependency_display.py | 38 +++++++++---------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/lib/tool_shed/galaxy_install/dependency_display.py b/lib/tool_shed/galaxy_install/dependency_display.py index bde25e80488..15a77f33880 100644 --- a/lib/tool_shed/galaxy_install/dependency_display.py +++ b/lib/tool_shed/galaxy_install/dependency_display.py @@ -25,7 +25,7 @@ class DependencyDisplayer: def __init__(self, app): self.app = app - def add_installation_directories_to_tool_dependencies(self, tool_dependencies): + def _add_installation_directories_to_tool_dependencies(self, tool_dependencies): """ Determine the path to the installation directory for each of the received tool dependencies. This path will be displayed within the tool dependencies @@ -172,7 +172,7 @@ class DependencyDisplayer: for env_requirements_dict in requirements_dict: name = env_requirements_dict["name"] type = env_requirements_dict["type"] - if self.tool_dependency_is_orphan(type, name, None, tools): + if self._tool_dependency_is_orphan(type, name, None, tools): if not has_orphan_set_environment_dependencies: has_orphan_set_environment_dependencies = True set_environment_orphans_str += f"* name: {name}, type: {type}
" @@ -181,7 +181,7 @@ class DependencyDisplayer: name = requirements_dict["name"] type = requirements_dict["type"] version = requirements_dict["version"] - if self.tool_dependency_is_orphan(type, name, version, tools): + if self._tool_dependency_is_orphan(type, name, version, tools): if not has_orphan_package_dependencies: has_orphan_package_dependencies = True package_orphans_str += ( @@ -203,7 +203,7 @@ class DependencyDisplayer: message += set_environment_orphans_str return message - def get_installed_and_missing_tool_dependencies_for_installed_repository(self, repository, all_tool_dependencies): + def _get_installed_and_missing_tool_dependencies_for_installed_repository(self, repository, all_tool_dependencies): """ Return the lists of installed tool dependencies and missing tool dependencies for a Tool Shed repository that has been installed into Galaxy. @@ -263,7 +263,7 @@ class DependencyDisplayer: missing_tool_dependencies = None return tool_dependencies, missing_tool_dependencies - def merge_missing_repository_dependencies_to_installed_container(self, containers_dict): + def _merge_missing_repository_dependencies_to_installed_container(self, containers_dict): """ Merge the list of missing repository dependencies into the list of installed repository dependencies. @@ -295,7 +295,7 @@ class DependencyDisplayer: containers_dict["missing_repository_dependencies"] = None return containers_dict - def merge_missing_tool_dependencies_to_installed_container(self, containers_dict): + def _merge_missing_tool_dependencies_to_installed_container(self, containers_dict): """ Merge the list of missing tool dependencies into the list of installed tool dependencies. @@ -343,7 +343,7 @@ class DependencyDisplayer: or for an installed repository that is being updated and the updates include newly defined repository (and possibly tool) dependencies. """ - installed_tool_dependencies, missing_tool_dependencies = self.populate_tool_dependencies_dicts( + installed_tool_dependencies, missing_tool_dependencies = self._populate_tool_dependencies_dicts( tool_shed_url=tool_shed_url, tool_path=tool_path, repository_installed_tool_dependencies=installed_tool_dependencies, @@ -372,9 +372,9 @@ class DependencyDisplayer: # the missing_repository_dependencies container contents to the installed_repository_dependencies # container. When updating an installed repository, merging will result in losing newly defined # dependencies included in the updates. - containers_dict = self.merge_missing_repository_dependencies_to_installed_container(containers_dict) + containers_dict = self._merge_missing_repository_dependencies_to_installed_container(containers_dict) # Merge the missing_tool_dependencies container contents to the installed_tool_dependencies container. - containers_dict = self.merge_missing_tool_dependencies_to_installed_container(containers_dict) + containers_dict = self._merge_missing_tool_dependencies_to_installed_container(containers_dict) return containers_dict def populate_containers_dict_from_repository_metadata( @@ -433,11 +433,11 @@ class DependencyDisplayer: ( repository_installed_tool_dependencies, repository_missing_tool_dependencies, - ) = self.get_installed_and_missing_tool_dependencies_for_installed_repository( + ) = self._get_installed_and_missing_tool_dependencies_for_installed_repository( repository, repository_tool_dependencies ) if reinstalling: - installed_tool_dependencies, missing_tool_dependencies = self.populate_tool_dependencies_dicts( + installed_tool_dependencies, missing_tool_dependencies = self._populate_tool_dependencies_dicts( tool_shed_url, tool_path, repository_installed_tool_dependencies, @@ -485,7 +485,7 @@ class DependencyDisplayer: ) return containers_dict - def populate_tool_dependencies_dicts( + def _populate_tool_dependencies_dicts( self, tool_shed_url, tool_path, @@ -503,14 +503,14 @@ class DependencyDisplayer: repository_installed_tool_dependencies = {} else: # Add the install_dir attribute to the tool_dependencies. - repository_installed_tool_dependencies = self.add_installation_directories_to_tool_dependencies( + repository_installed_tool_dependencies = self._add_installation_directories_to_tool_dependencies( repository_installed_tool_dependencies ) if repository_missing_tool_dependencies is None: repository_missing_tool_dependencies = {} else: # Add the install_dir attribute to the tool_dependencies. - repository_missing_tool_dependencies = self.add_installation_directories_to_tool_dependencies( + repository_missing_tool_dependencies = self._add_installation_directories_to_tool_dependencies( repository_missing_tool_dependencies ) if required_repo_info_dicts: @@ -528,7 +528,7 @@ class DependencyDisplayer: ) = get_repo_info_tuple_contents(repo_info_tuple) if tool_dependencies: # Add the install_dir attribute to the tool_dependencies. - tool_dependencies = self.add_installation_directories_to_tool_dependencies(tool_dependencies) + tool_dependencies = self._add_installation_directories_to_tool_dependencies(tool_dependencies) # The required_repository may have been installed with a different changeset revision. ( required_repository, @@ -540,13 +540,13 @@ class DependencyDisplayer: ( required_repository_installed_tool_dependencies, required_repository_missing_tool_dependencies, - ) = self.get_installed_and_missing_tool_dependencies_for_installed_repository( + ) = self._get_installed_and_missing_tool_dependencies_for_installed_repository( required_repository, tool_dependencies ) if required_repository_installed_tool_dependencies: # Add the install_dir attribute to the tool_dependencies. required_repository_installed_tool_dependencies = ( - self.add_installation_directories_to_tool_dependencies( + self._add_installation_directories_to_tool_dependencies( required_repository_installed_tool_dependencies ) ) @@ -556,7 +556,7 @@ class DependencyDisplayer: if required_repository_missing_tool_dependencies: # Add the install_dir attribute to the tool_dependencies. required_repository_missing_tool_dependencies = ( - self.add_installation_directories_to_tool_dependencies( + self._add_installation_directories_to_tool_dependencies( required_repository_missing_tool_dependencies ) ) @@ -569,7 +569,7 @@ class DependencyDisplayer: missing_tool_dependencies = repository_missing_tool_dependencies return installed_tool_dependencies, missing_tool_dependencies - def tool_dependency_is_orphan(self, type, name, version, tools): + def _tool_dependency_is_orphan(self, type, name, version, tools): """ Determine if the combination of the received type, name and version is defined in the tag for at least one tool in the received list of tools. If not, the tool dependency defined by the From fcc58d54111fa178247a4c540ae55f1bf7f400ac Mon Sep 17 00:00:00 2001 From: John Chilton Date: Mon, 17 Oct 2022 15:10:17 -0400 Subject: [PATCH 06/19] Improved dependency imports for toolshed. --- lib/tool_shed/utility_containers/__init__.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/lib/tool_shed/utility_containers/__init__.py b/lib/tool_shed/utility_containers/__init__.py index 1058b3169a8..a4e244ead7a 100644 --- a/lib/tool_shed/utility_containers/__init__.py +++ b/lib/tool_shed/utility_containers/__init__.py @@ -2,11 +2,12 @@ import logging import threading from galaxy import util -from tool_shed.util import ( - common_util, - container_util, - readme_util, +from galaxy.tool_shed.util.common_util import parse_repository_dependency_tuple +from galaxy.tool_shed.util.container_util import ( + generate_repository_dependencies_key_for_repository, + STRSEP, ) +from tool_shed.util.readme_util import build_readme_files_dict from . import utility_container_manager log = logging.getLogger(__name__) @@ -166,8 +167,8 @@ class ToolShedUtilityContainerManager(utility_container_manager.UtilityContainer prior_installation_required, only_if_compiling_contained_td, error, - ) = common_util.parse_repository_dependency_tuple(invalid_repository_dependency, contains_error=True) - key = container_util.generate_repository_dependencies_key_for_repository( + ) = parse_repository_dependency_tuple(invalid_repository_dependency, contains_error=True) + key = generate_repository_dependencies_key_for_repository( toolshed, name, owner, @@ -300,7 +301,7 @@ class ToolShedUtilityContainerManager(utility_container_manager.UtilityContainer # Readme files container. if metadata: if "readme_files" not in exclude and "readme_files" in metadata: - readme_files_dict = readme_util.build_readme_files_dict( + readme_files_dict = build_readme_files_dict( self.app, repository, changeset_revision, metadata ) folder_id, readme_files_root_folder = self.build_readme_files_folder( @@ -363,4 +364,4 @@ class ToolShedUtilityContainerManager(utility_container_manager.UtilityContainer return containers_dict def generate_tool_dependencies_key(self, name, version, type): - return f"{str(name)}{container_util.STRSEP}{str(version)}{container_util.STRSEP}{str(type)}" + return f"{str(name)}{STRSEP}{str(version)}{STRSEP}{str(type)}" From 1326745b65a255a2f5b62b4ccd2b648c58346a59 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Mon, 17 Oct 2022 15:11:42 -0400 Subject: [PATCH 07/19] Remove unused methods in dependency_display.py. --- .../galaxy_install/dependency_display.py | 198 ------------------ 1 file changed, 198 deletions(-) diff --git a/lib/tool_shed/galaxy_install/dependency_display.py b/lib/tool_shed/galaxy_install/dependency_display.py index 15a77f33880..15100bafe73 100644 --- a/lib/tool_shed/galaxy_install/dependency_display.py +++ b/lib/tool_shed/galaxy_install/dependency_display.py @@ -263,120 +263,6 @@ class DependencyDisplayer: missing_tool_dependencies = None return tool_dependencies, missing_tool_dependencies - def _merge_missing_repository_dependencies_to_installed_container(self, containers_dict): - """ - Merge the list of missing repository dependencies into the list of installed - repository dependencies. - """ - missing_rd_container_root = containers_dict.get("missing_repository_dependencies", None) - if missing_rd_container_root: - # The missing_rd_container_root will be a root folder containing a single sub_folder. - missing_rd_container = missing_rd_container_root.folders[0] - installed_rd_container_root = containers_dict.get("repository_dependencies", None) - # The installed_rd_container_root will be a root folder containing a single sub_folder. - if installed_rd_container_root: - installed_rd_container = installed_rd_container_root.folders[0] - installed_rd_container.label = "Repository dependencies" - for index, rd in enumerate(missing_rd_container.repository_dependencies): - # Skip the header row. - if index == 0: - continue - installed_rd_container.repository_dependencies.append(rd) - installed_rd_container_root.folders = [installed_rd_container] - containers_dict["repository_dependencies"] = installed_rd_container_root - else: - # Change the folder label from 'Missing repository dependencies' to be - # 'Repository dependencies' for display. - root_container = containers_dict["missing_repository_dependencies"] - for sub_container in root_container.folders: - # There should only be 1 sub-folder. - sub_container.label = "Repository dependencies" - containers_dict["repository_dependencies"] = root_container - containers_dict["missing_repository_dependencies"] = None - return containers_dict - - def _merge_missing_tool_dependencies_to_installed_container(self, containers_dict): - """ - Merge the list of missing tool dependencies into the list of installed tool - dependencies. - """ - missing_td_container_root = containers_dict.get("missing_tool_dependencies", None) - if missing_td_container_root: - # The missing_td_container_root will be a root folder containing a single sub_folder. - missing_td_container = missing_td_container_root.folders[0] - installed_td_container_root = containers_dict.get("tool_dependencies", None) - # The installed_td_container_root will be a root folder containing a single sub_folder. - if installed_td_container_root: - installed_td_container = installed_td_container_root.folders[0] - installed_td_container.label = "Tool dependencies" - for index, td in enumerate(missing_td_container.tool_dependencies): - # Skip the header row. - if index == 0: - continue - installed_td_container.tool_dependencies.append(td) - installed_td_container_root.folders = [installed_td_container] - containers_dict["tool_dependencies"] = installed_td_container_root - else: - # Change the folder label from 'Missing tool dependencies' to be - # 'Tool dependencies' for display. - root_container = containers_dict["missing_tool_dependencies"] - for sub_container in root_container.folders: - # There should only be 1 subfolder. - sub_container.label = "Tool dependencies" - containers_dict["tool_dependencies"] = root_container - containers_dict["missing_tool_dependencies"] = None - return containers_dict - - def populate_containers_dict_for_new_install( - self, - tool_shed_url, - tool_path, - readme_files_dict, - installed_repository_dependencies, - missing_repository_dependencies, - installed_tool_dependencies, - missing_tool_dependencies, - updating=False, - ): - """ - Return the populated containers for a repository being installed for the first time - or for an installed repository that is being updated and the updates include newly - defined repository (and possibly tool) dependencies. - """ - installed_tool_dependencies, missing_tool_dependencies = self._populate_tool_dependencies_dicts( - tool_shed_url=tool_shed_url, - tool_path=tool_path, - repository_installed_tool_dependencies=installed_tool_dependencies, - repository_missing_tool_dependencies=missing_tool_dependencies, - required_repo_info_dicts=None, - ) - # Most of the repository contents are set to None since we don't yet know what they are. - gucm = GalaxyUtilityContainerManager(self.app) - containers_dict = gucm.build_repository_containers( - repository=None, - invalid_tools=None, - missing_repository_dependencies=missing_repository_dependencies, - missing_tool_dependencies=missing_tool_dependencies, - readme_files_dict=readme_files_dict, - repository_dependencies=installed_repository_dependencies, - tool_dependencies=installed_tool_dependencies, - valid_tools=None, - valid_data_managers=None, - invalid_data_managers=None, - data_managers_errors=None, - new_install=True, - reinstalling=False, - ) - if not updating: - # If we installing a new repository and not updaing an installed repository, we can merge - # the missing_repository_dependencies container contents to the installed_repository_dependencies - # container. When updating an installed repository, merging will result in losing newly defined - # dependencies included in the updates. - containers_dict = self._merge_missing_repository_dependencies_to_installed_container(containers_dict) - # Merge the missing_tool_dependencies container contents to the installed_tool_dependencies container. - containers_dict = self._merge_missing_tool_dependencies_to_installed_container(containers_dict) - return containers_dict - def populate_containers_dict_from_repository_metadata( self, tool_shed_url, tool_path, repository, reinstalling=False, required_repo_info_dicts=None ): @@ -485,90 +371,6 @@ class DependencyDisplayer: ) return containers_dict - def _populate_tool_dependencies_dicts( - self, - tool_shed_url, - tool_path, - repository_installed_tool_dependencies, - repository_missing_tool_dependencies, - required_repo_info_dicts, - ): - """ - Return the populated installed_tool_dependencies and missing_tool_dependencies dictionaries - for all repositories defined by entries in the received required_repo_info_dicts. - """ - installed_tool_dependencies = None - missing_tool_dependencies = None - if repository_installed_tool_dependencies is None: - repository_installed_tool_dependencies = {} - else: - # Add the install_dir attribute to the tool_dependencies. - repository_installed_tool_dependencies = self._add_installation_directories_to_tool_dependencies( - repository_installed_tool_dependencies - ) - if repository_missing_tool_dependencies is None: - repository_missing_tool_dependencies = {} - else: - # Add the install_dir attribute to the tool_dependencies. - repository_missing_tool_dependencies = self._add_installation_directories_to_tool_dependencies( - repository_missing_tool_dependencies - ) - if required_repo_info_dicts: - # Handle the tool dependencies defined for each of the repository's repository dependencies. - for rid in required_repo_info_dicts: - for name, repo_info_tuple in rid.items(): - ( - description, - repository_clone_url, - changeset_revision, - ctx_rev, - repository_owner, - repository_dependencies, - tool_dependencies, - ) = get_repo_info_tuple_contents(repo_info_tuple) - if tool_dependencies: - # Add the install_dir attribute to the tool_dependencies. - tool_dependencies = self._add_installation_directories_to_tool_dependencies(tool_dependencies) - # The required_repository may have been installed with a different changeset revision. - ( - required_repository, - installed_changeset_revision, - ) = repository_was_previously_installed( - self.app, tool_shed_url, name, repo_info_tuple, from_tip=False - ) - if required_repository: - ( - required_repository_installed_tool_dependencies, - required_repository_missing_tool_dependencies, - ) = self._get_installed_and_missing_tool_dependencies_for_installed_repository( - required_repository, tool_dependencies - ) - if required_repository_installed_tool_dependencies: - # Add the install_dir attribute to the tool_dependencies. - required_repository_installed_tool_dependencies = ( - self._add_installation_directories_to_tool_dependencies( - required_repository_installed_tool_dependencies - ) - ) - for td_key, td_dict in required_repository_installed_tool_dependencies.items(): - if td_key not in repository_installed_tool_dependencies: - repository_installed_tool_dependencies[td_key] = td_dict - if required_repository_missing_tool_dependencies: - # Add the install_dir attribute to the tool_dependencies. - required_repository_missing_tool_dependencies = ( - self._add_installation_directories_to_tool_dependencies( - required_repository_missing_tool_dependencies - ) - ) - for td_key, td_dict in required_repository_missing_tool_dependencies.items(): - if td_key not in repository_missing_tool_dependencies: - repository_missing_tool_dependencies[td_key] = td_dict - if repository_installed_tool_dependencies: - installed_tool_dependencies = repository_installed_tool_dependencies - if repository_missing_tool_dependencies: - missing_tool_dependencies = repository_missing_tool_dependencies - return installed_tool_dependencies, missing_tool_dependencies - def _tool_dependency_is_orphan(self, type, name, version, tools): """ Determine if the combination of the received type, name and version is defined in the From 401328a6dd7e5f80b89d3ea19b354fee696ee874 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Mon, 17 Oct 2022 15:24:47 -0400 Subject: [PATCH 08/19] Remove no-op Lock in utility containers. --- lib/tool_shed/galaxy_install/utility_containers/__init__.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/lib/tool_shed/galaxy_install/utility_containers/__init__.py b/lib/tool_shed/galaxy_install/utility_containers/__init__.py index 223d2bf1bde..14f2d5c46c6 100644 --- a/lib/tool_shed/galaxy_install/utility_containers/__init__.py +++ b/lib/tool_shed/galaxy_install/utility_containers/__init__.py @@ -1,5 +1,4 @@ import logging -import threading from tool_shed.utility_containers import utility_container_manager @@ -50,8 +49,6 @@ class GalaxyUtilityContainerManager(utility_container_manager.UtilityContainerMa changeset_revision = repository.changeset_revision else: changeset_revision = None - lock = threading.Lock() - lock.acquire(True) try: folder_id = 0 # Invalid tools container. @@ -133,6 +130,4 @@ class GalaxyUtilityContainerManager(utility_container_manager.UtilityContainerMa containers_dict["invalid_data_managers"] = invalid_data_managers_root_folder except Exception as e: log.debug(f"Exception in build_repository_containers: {str(e)}") - finally: - lock.release() return containers_dict From 6dfb59f9ce4cf4849906c1d4e04a25626ac9e98d Mon Sep 17 00:00:00 2001 From: John Chilton Date: Mon, 17 Oct 2022 15:25:46 -0400 Subject: [PATCH 09/19] Simplify dependency_display now that it is used for a lot less. --- .../galaxy/controllers/admin_toolshed.py | 4 -- .../galaxy_install/dependency_display.py | 62 ++----------------- .../utility_containers/__init__.py | 9 --- 3 files changed, 6 insertions(+), 69 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py b/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py index da081c95a64..b96866a37f2 100644 --- a/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py +++ b/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py @@ -170,11 +170,7 @@ class AdminToolshed(AdminGalaxy): message = "The repository information has been updated." dd = dependency_display.DependencyDisplayer(trans.app) containers_dict = dd.populate_containers_dict_from_repository_metadata( - tool_shed_url=tool_shed_url, - tool_path=tool_path, repository=repository, - reinstalling=False, - required_repo_info_dicts=None, ) view = views.DependencyResolversView(self.app) tool_requirements_d = suc.get_requirements_from_repository(repository) diff --git a/lib/tool_shed/galaxy_install/dependency_display.py b/lib/tool_shed/galaxy_install/dependency_display.py index 15100bafe73..2cb03b3c82d 100644 --- a/lib/tool_shed/galaxy_install/dependency_display.py +++ b/lib/tool_shed/galaxy_install/dependency_display.py @@ -3,20 +3,12 @@ import logging import os from galaxy import util -from galaxy.tool_shed.util.common_util import ( - get_tool_shed_url_from_tool_shed_registry, - parse_repository_dependency_tuple, -) -from galaxy.tool_shed.util.repository_util import ( - get_repo_info_tuple_contents, - repository_was_previously_installed, -) +from galaxy.tool_shed.util.common_util import parse_repository_dependency_tuple from galaxy.tool_shed.util.tool_dependency_util import ( get_tool_dependency_by_name_type_repository, get_tool_dependency_by_name_version_type_repository, ) from tool_shed.galaxy_install.utility_containers import GalaxyUtilityContainerManager -from tool_shed.util.readme_util import build_readme_files_dict log = logging.getLogger(__name__) @@ -264,7 +256,7 @@ class DependencyDisplayer: return tool_dependencies, missing_tool_dependencies def populate_containers_dict_from_repository_metadata( - self, tool_shed_url, tool_path, repository, reinstalling=False, required_repo_info_dicts=None + self, repository ): """ Retrieve necessary information from the received repository's metadata to populate the @@ -276,33 +268,6 @@ class DependencyDisplayer: if metadata: # Handle invalid tools. invalid_tools = metadata.get("invalid_tools", None) - # Handle README files. - if repository.has_readme_files: - if reinstalling or repository.status not in [ - self.app.install_model.ToolShedRepository.installation_status.DEACTIVATED, - self.app.install_model.ToolShedRepository.installation_status.INSTALLED, - ]: - # Since we're reinstalling, we need to send a request to the tool shed to get the README files. - tool_shed_url = get_tool_shed_url_from_tool_shed_registry(self.app, tool_shed_url) - params = dict( - name=str(repository.name), - owner=str(repository.owner), - changeset_revision=str(repository.installed_changeset_revision), - ) - pathspec = ["repository", "get_readme_files"] - raw_text = util.url_get( - tool_shed_url, - auth=self.app.tool_shed_registry.url_auth(tool_shed_url), - pathspec=pathspec, - params=params, - ) - readme_files_dict = json.loads(raw_text) - else: - readme_files_dict = build_readme_files_dict( - self.app, repository, repository.changeset_revision, repository.metadata_, tool_path - ) - else: - readme_files_dict = None # Handle repository dependencies. ( installed_repository_dependencies, @@ -322,17 +287,8 @@ class DependencyDisplayer: ) = self._get_installed_and_missing_tool_dependencies_for_installed_repository( repository, repository_tool_dependencies ) - if reinstalling: - installed_tool_dependencies, missing_tool_dependencies = self._populate_tool_dependencies_dicts( - tool_shed_url, - tool_path, - repository_installed_tool_dependencies, - repository_missing_tool_dependencies, - required_repo_info_dicts, - ) - else: - installed_tool_dependencies = repository_installed_tool_dependencies - missing_tool_dependencies = repository_missing_tool_dependencies + installed_tool_dependencies = repository_installed_tool_dependencies + missing_tool_dependencies = repository_missing_tool_dependencies # Handle valid tools. valid_tools = metadata.get("tools", None) # Handle Data Managers @@ -349,7 +305,7 @@ class DependencyDisplayer: invalid_tools=invalid_tools, missing_repository_dependencies=missing_repository_dependencies, missing_tool_dependencies=missing_tool_dependencies, - readme_files_dict=readme_files_dict, + readme_files_dict=None, repository_dependencies=installed_repository_dependencies, tool_dependencies=installed_tool_dependencies, valid_tools=valid_tools, @@ -357,17 +313,11 @@ class DependencyDisplayer: invalid_data_managers=invalid_data_managers, data_managers_errors=data_managers_errors, new_install=False, - reinstalling=reinstalling, + reinstalling=False, ) else: containers_dict = dict( - datatypes=None, - invalid_tools=None, - readme_files_dict=None, repository_dependencies=None, - tool_dependencies=None, - valid_tools=None, - workflows=None, ) return containers_dict diff --git a/lib/tool_shed/galaxy_install/utility_containers/__init__.py b/lib/tool_shed/galaxy_install/utility_containers/__init__.py index 14f2d5c46c6..d9ad93bd7e1 100644 --- a/lib/tool_shed/galaxy_install/utility_containers/__init__.py +++ b/lib/tool_shed/galaxy_install/utility_containers/__init__.py @@ -30,17 +30,8 @@ class GalaxyUtilityContainerManager(utility_container_manager.UtilityContainerMa display during installation to Galaxy. """ containers_dict = dict( - datatypes=None, - invalid_tools=None, - missing_tool_dependencies=None, - readme_files=None, repository_dependencies=None, missing_repository_dependencies=None, - tool_dependencies=None, - valid_tools=None, - workflows=None, - valid_data_managers=None, - invalid_data_managers=None, ) # Some of the tool dependency folders will include links to display tool dependency information, and # some of these links require the repository id. However we need to be careful because sometimes the From e6519265ad234276de81523564e4e54c403866d3 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Mon, 17 Oct 2022 15:27:03 -0400 Subject: [PATCH 10/19] Remove unused container invalid_tools. --- lib/tool_shed/galaxy_install/dependency_display.py | 3 --- .../galaxy_install/utility_containers/__init__.py | 7 ------- 2 files changed, 10 deletions(-) diff --git a/lib/tool_shed/galaxy_install/dependency_display.py b/lib/tool_shed/galaxy_install/dependency_display.py index 2cb03b3c82d..0dee318c917 100644 --- a/lib/tool_shed/galaxy_install/dependency_display.py +++ b/lib/tool_shed/galaxy_install/dependency_display.py @@ -266,8 +266,6 @@ class DependencyDisplayer: """ metadata = repository.metadata_ if metadata: - # Handle invalid tools. - invalid_tools = metadata.get("invalid_tools", None) # Handle repository dependencies. ( installed_repository_dependencies, @@ -302,7 +300,6 @@ class DependencyDisplayer: gucm = GalaxyUtilityContainerManager(self.app) containers_dict = gucm.build_repository_containers( repository=repository, - invalid_tools=invalid_tools, missing_repository_dependencies=missing_repository_dependencies, missing_tool_dependencies=missing_tool_dependencies, readme_files_dict=None, diff --git a/lib/tool_shed/galaxy_install/utility_containers/__init__.py b/lib/tool_shed/galaxy_install/utility_containers/__init__.py index d9ad93bd7e1..56c23303345 100644 --- a/lib/tool_shed/galaxy_install/utility_containers/__init__.py +++ b/lib/tool_shed/galaxy_install/utility_containers/__init__.py @@ -12,7 +12,6 @@ class GalaxyUtilityContainerManager(utility_container_manager.UtilityContainerMa def build_repository_containers( self, repository, - invalid_tools, missing_repository_dependencies, missing_tool_dependencies, readme_files_dict, @@ -42,12 +41,6 @@ class GalaxyUtilityContainerManager(utility_container_manager.UtilityContainerMa changeset_revision = None try: folder_id = 0 - # Invalid tools container. - if invalid_tools: - folder_id, invalid_tools_root_folder = self.build_invalid_tools_folder( - folder_id, invalid_tools, changeset_revision, repository=repository, label="Invalid tools" - ) - containers_dict["invalid_tools"] = invalid_tools_root_folder # Readme files container. if readme_files_dict: folder_id, readme_files_root_folder = self.build_readme_files_folder(folder_id, readme_files_dict) From 8e6fc9dcc78a76e0dca49f51e08b047be1e36bbc Mon Sep 17 00:00:00 2001 From: John Chilton Date: Mon, 17 Oct 2022 15:27:55 -0400 Subject: [PATCH 11/19] Remove unused container readme files. --- lib/tool_shed/galaxy_install/dependency_display.py | 1 - lib/tool_shed/galaxy_install/utility_containers/__init__.py | 5 ----- 2 files changed, 6 deletions(-) diff --git a/lib/tool_shed/galaxy_install/dependency_display.py b/lib/tool_shed/galaxy_install/dependency_display.py index 0dee318c917..0fe28bb3006 100644 --- a/lib/tool_shed/galaxy_install/dependency_display.py +++ b/lib/tool_shed/galaxy_install/dependency_display.py @@ -302,7 +302,6 @@ class DependencyDisplayer: repository=repository, missing_repository_dependencies=missing_repository_dependencies, missing_tool_dependencies=missing_tool_dependencies, - readme_files_dict=None, repository_dependencies=installed_repository_dependencies, tool_dependencies=installed_tool_dependencies, valid_tools=valid_tools, diff --git a/lib/tool_shed/galaxy_install/utility_containers/__init__.py b/lib/tool_shed/galaxy_install/utility_containers/__init__.py index 56c23303345..0e9dab9bc8b 100644 --- a/lib/tool_shed/galaxy_install/utility_containers/__init__.py +++ b/lib/tool_shed/galaxy_install/utility_containers/__init__.py @@ -14,7 +14,6 @@ class GalaxyUtilityContainerManager(utility_container_manager.UtilityContainerMa repository, missing_repository_dependencies, missing_tool_dependencies, - readme_files_dict, repository_dependencies, tool_dependencies, valid_tools, @@ -41,10 +40,6 @@ class GalaxyUtilityContainerManager(utility_container_manager.UtilityContainerMa changeset_revision = None try: folder_id = 0 - # Readme files container. - if readme_files_dict: - folder_id, readme_files_root_folder = self.build_readme_files_folder(folder_id, readme_files_dict) - containers_dict["readme_files"] = readme_files_root_folder # Installed repository dependencies container. if repository_dependencies: if new_install: From 25e830fa8128ff1ec08961f2bed0cf36dcb97b69 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Mon, 17 Oct 2022 15:29:47 -0400 Subject: [PATCH 12/19] Remove more unused containers. --- .../galaxy_install/dependency_display.py | 14 ---------- .../utility_containers/__init__.py | 28 ------------------- 2 files changed, 42 deletions(-) diff --git a/lib/tool_shed/galaxy_install/dependency_display.py b/lib/tool_shed/galaxy_install/dependency_display.py index 0fe28bb3006..949b3fc168b 100644 --- a/lib/tool_shed/galaxy_install/dependency_display.py +++ b/lib/tool_shed/galaxy_install/dependency_display.py @@ -287,16 +287,6 @@ class DependencyDisplayer: ) installed_tool_dependencies = repository_installed_tool_dependencies missing_tool_dependencies = repository_missing_tool_dependencies - # Handle valid tools. - valid_tools = metadata.get("tools", None) - # Handle Data Managers - valid_data_managers = None - invalid_data_managers = None - data_managers_errors = None - if "data_manager" in metadata: - valid_data_managers = metadata["data_manager"].get("data_managers", None) - invalid_data_managers = metadata["data_manager"].get("invalid_data_managers", None) - data_managers_errors = metadata["data_manager"].get("messages", None) gucm = GalaxyUtilityContainerManager(self.app) containers_dict = gucm.build_repository_containers( repository=repository, @@ -304,10 +294,6 @@ class DependencyDisplayer: missing_tool_dependencies=missing_tool_dependencies, repository_dependencies=installed_repository_dependencies, tool_dependencies=installed_tool_dependencies, - valid_tools=valid_tools, - valid_data_managers=valid_data_managers, - invalid_data_managers=invalid_data_managers, - data_managers_errors=data_managers_errors, new_install=False, reinstalling=False, ) diff --git a/lib/tool_shed/galaxy_install/utility_containers/__init__.py b/lib/tool_shed/galaxy_install/utility_containers/__init__.py index 0e9dab9bc8b..670fec127a5 100644 --- a/lib/tool_shed/galaxy_install/utility_containers/__init__.py +++ b/lib/tool_shed/galaxy_install/utility_containers/__init__.py @@ -16,10 +16,6 @@ class GalaxyUtilityContainerManager(utility_container_manager.UtilityContainerMa missing_tool_dependencies, repository_dependencies, tool_dependencies, - valid_tools, - valid_data_managers, - invalid_data_managers, - data_managers_errors, new_install=False, reinstalling=False, ): @@ -34,10 +30,6 @@ class GalaxyUtilityContainerManager(utility_container_manager.UtilityContainerMa # Some of the tool dependency folders will include links to display tool dependency information, and # some of these links require the repository id. However we need to be careful because sometimes the # repository object is None. - if repository: - changeset_revision = repository.changeset_revision - else: - changeset_revision = None try: folder_id = 0 # Installed repository dependencies container. @@ -87,26 +79,6 @@ class GalaxyUtilityContainerManager(utility_container_manager.UtilityContainerMa reinstalling=reinstalling, ) containers_dict["missing_tool_dependencies"] = missing_tool_dependencies_root_folder - # Valid tools container. - if valid_tools: - folder_id, valid_tools_root_folder = self.build_tools_folder( - folder_id, valid_tools, repository, changeset_revision, label="Valid tools" - ) - containers_dict["valid_tools"] = valid_tools_root_folder - # Workflows container. - if valid_data_managers: - folder_id, valid_data_managers_root_folder = self.build_data_managers_folder( - folder_id=folder_id, data_managers=valid_data_managers, label="Valid Data Managers" - ) - containers_dict["valid_data_managers"] = valid_data_managers_root_folder - if invalid_data_managers or data_managers_errors: - folder_id, invalid_data_managers_root_folder = self.build_invalid_data_managers_folder( - folder_id=folder_id, - data_managers=invalid_data_managers, - error_messages=data_managers_errors, - label="Invalid Data Managers", - ) - containers_dict["invalid_data_managers"] = invalid_data_managers_root_folder except Exception as e: log.debug(f"Exception in build_repository_containers: {str(e)}") return containers_dict From dea1d0e12f40c3f12f7be2ff1c36593d9131c1dd Mon Sep 17 00:00:00 2001 From: John Chilton Date: Mon, 17 Oct 2022 15:31:05 -0400 Subject: [PATCH 13/19] Remove more unused parameters to GalaxyUtilityContainerManager --- .../galaxy_install/dependency_display.py | 3 --- .../utility_containers/__init__.py | 21 ++++++------------- 2 files changed, 6 insertions(+), 18 deletions(-) diff --git a/lib/tool_shed/galaxy_install/dependency_display.py b/lib/tool_shed/galaxy_install/dependency_display.py index 949b3fc168b..5a05818da74 100644 --- a/lib/tool_shed/galaxy_install/dependency_display.py +++ b/lib/tool_shed/galaxy_install/dependency_display.py @@ -289,13 +289,10 @@ class DependencyDisplayer: missing_tool_dependencies = repository_missing_tool_dependencies gucm = GalaxyUtilityContainerManager(self.app) containers_dict = gucm.build_repository_containers( - repository=repository, missing_repository_dependencies=missing_repository_dependencies, missing_tool_dependencies=missing_tool_dependencies, repository_dependencies=installed_repository_dependencies, tool_dependencies=installed_tool_dependencies, - new_install=False, - reinstalling=False, ) else: containers_dict = dict( diff --git a/lib/tool_shed/galaxy_install/utility_containers/__init__.py b/lib/tool_shed/galaxy_install/utility_containers/__init__.py index 670fec127a5..02b92be217a 100644 --- a/lib/tool_shed/galaxy_install/utility_containers/__init__.py +++ b/lib/tool_shed/galaxy_install/utility_containers/__init__.py @@ -11,13 +11,10 @@ class GalaxyUtilityContainerManager(utility_container_manager.UtilityContainerMa def build_repository_containers( self, - repository, missing_repository_dependencies, missing_tool_dependencies, repository_dependencies, tool_dependencies, - new_install=False, - reinstalling=False, ): """ Return a dictionary of containers for the received repository's dependencies and readme files for @@ -34,10 +31,7 @@ class GalaxyUtilityContainerManager(utility_container_manager.UtilityContainerMa folder_id = 0 # Installed repository dependencies container. if repository_dependencies: - if new_install: - label = "Repository dependencies" - else: - label = "Installed repository dependencies" + label = "Installed repository dependencies" folder_id, repository_dependencies_root_folder = self.build_repository_dependencies_folder( folder_id=folder_id, repository_dependencies=repository_dependencies, label=label, installed=True ) @@ -53,18 +47,15 @@ class GalaxyUtilityContainerManager(utility_container_manager.UtilityContainerMa containers_dict["missing_repository_dependencies"] = missing_repository_dependencies_root_folder # Installed tool dependencies container. if tool_dependencies: - if new_install: - label = "Tool dependencies" - else: - label = "Installed tool dependencies" + label = "Installed tool dependencies" # We only want to display the Status column if the tool_dependency is missing. folder_id, tool_dependencies_root_folder = self.build_tool_dependencies_folder( folder_id, tool_dependencies, label=label, missing=False, - new_install=new_install, - reinstalling=reinstalling, + new_install=False, + reinstalling=False, ) containers_dict["tool_dependencies"] = tool_dependencies_root_folder # Missing tool dependencies container. @@ -75,8 +66,8 @@ class GalaxyUtilityContainerManager(utility_container_manager.UtilityContainerMa missing_tool_dependencies, label="Missing tool dependencies", missing=True, - new_install=new_install, - reinstalling=reinstalling, + new_install=False, + reinstalling=False, ) containers_dict["missing_tool_dependencies"] = missing_tool_dependencies_root_folder except Exception as e: From f734fb81595fab4eb757d4c1aa92b70ef09e37f0 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Mon, 17 Oct 2022 15:33:57 -0400 Subject: [PATCH 14/19] Remove tool dependency related folders in container dict. --- .../galaxy_install/dependency_display.py | 75 ------------------- .../utility_containers/__init__.py | 27 ------- 2 files changed, 102 deletions(-) diff --git a/lib/tool_shed/galaxy_install/dependency_display.py b/lib/tool_shed/galaxy_install/dependency_display.py index 5a05818da74..46b0b34be61 100644 --- a/lib/tool_shed/galaxy_install/dependency_display.py +++ b/lib/tool_shed/galaxy_install/dependency_display.py @@ -1,13 +1,8 @@ -import json import logging import os from galaxy import util from galaxy.tool_shed.util.common_util import parse_repository_dependency_tuple -from galaxy.tool_shed.util.tool_dependency_util import ( - get_tool_dependency_by_name_type_repository, - get_tool_dependency_by_name_version_type_repository, -) from tool_shed.galaxy_install.utility_containers import GalaxyUtilityContainerManager log = logging.getLogger(__name__) @@ -195,66 +190,6 @@ class DependencyDisplayer: message += set_environment_orphans_str return message - def _get_installed_and_missing_tool_dependencies_for_installed_repository(self, repository, all_tool_dependencies): - """ - Return the lists of installed tool dependencies and missing tool dependencies for a Tool Shed - repository that has been installed into Galaxy. - """ - if all_tool_dependencies: - tool_dependencies = {} - missing_tool_dependencies = {} - for td_key, val in all_tool_dependencies.items(): - if td_key in ["set_environment"]: - for index, td_info_dict in enumerate(val): - name = td_info_dict["name"] - version = None - type = td_info_dict["type"] - tool_dependency = get_tool_dependency_by_name_type_repository( - self.app, repository, name, type - ) - if tool_dependency: - td_info_dict["repository_id"] = repository.id - td_info_dict["tool_dependency_id"] = tool_dependency.id - if tool_dependency.status: - tool_dependency_status = str(tool_dependency.status) - else: - tool_dependency_status = "Never installed" - td_info_dict["status"] = tool_dependency_status - val[index] = td_info_dict - if ( - tool_dependency.status - == self.app.install_model.ToolDependency.installation_status.INSTALLED - ): - tool_dependencies[td_key] = val - else: - missing_tool_dependencies[td_key] = val - else: - name = val["name"] - version = val["version"] - type = val["type"] - tool_dependency = get_tool_dependency_by_name_version_type_repository( - self.app, repository, name, version, type - ) - if tool_dependency: - val["repository_id"] = repository.id - val["tool_dependency_id"] = tool_dependency.id - if tool_dependency.status: - tool_dependency_status = str(tool_dependency.status) - else: - tool_dependency_status = "Never installed" - val["status"] = tool_dependency_status - if ( - tool_dependency.status - == self.app.install_model.ToolDependency.installation_status.INSTALLED - ): - tool_dependencies[td_key] = val - else: - missing_tool_dependencies[td_key] = val - else: - tool_dependencies = None - missing_tool_dependencies = None - return tool_dependencies, missing_tool_dependencies - def populate_containers_dict_from_repository_metadata( self, repository ): @@ -279,20 +214,10 @@ class DependencyDisplayer: if repository_tool_dependencies is None: repository_tool_dependencies = {} repository_tool_dependencies.update(repository_invalid_tool_dependencies) - ( - repository_installed_tool_dependencies, - repository_missing_tool_dependencies, - ) = self._get_installed_and_missing_tool_dependencies_for_installed_repository( - repository, repository_tool_dependencies - ) - installed_tool_dependencies = repository_installed_tool_dependencies - missing_tool_dependencies = repository_missing_tool_dependencies gucm = GalaxyUtilityContainerManager(self.app) containers_dict = gucm.build_repository_containers( missing_repository_dependencies=missing_repository_dependencies, - missing_tool_dependencies=missing_tool_dependencies, repository_dependencies=installed_repository_dependencies, - tool_dependencies=installed_tool_dependencies, ) else: containers_dict = dict( diff --git a/lib/tool_shed/galaxy_install/utility_containers/__init__.py b/lib/tool_shed/galaxy_install/utility_containers/__init__.py index 02b92be217a..260c7a7bbc5 100644 --- a/lib/tool_shed/galaxy_install/utility_containers/__init__.py +++ b/lib/tool_shed/galaxy_install/utility_containers/__init__.py @@ -12,9 +12,7 @@ class GalaxyUtilityContainerManager(utility_container_manager.UtilityContainerMa def build_repository_containers( self, missing_repository_dependencies, - missing_tool_dependencies, repository_dependencies, - tool_dependencies, ): """ Return a dictionary of containers for the received repository's dependencies and readme files for @@ -45,31 +43,6 @@ class GalaxyUtilityContainerManager(utility_container_manager.UtilityContainerMa installed=False, ) containers_dict["missing_repository_dependencies"] = missing_repository_dependencies_root_folder - # Installed tool dependencies container. - if tool_dependencies: - label = "Installed tool dependencies" - # We only want to display the Status column if the tool_dependency is missing. - folder_id, tool_dependencies_root_folder = self.build_tool_dependencies_folder( - folder_id, - tool_dependencies, - label=label, - missing=False, - new_install=False, - reinstalling=False, - ) - containers_dict["tool_dependencies"] = tool_dependencies_root_folder - # Missing tool dependencies container. - if missing_tool_dependencies: - # We only want to display the Status column if the tool_dependency is missing. - folder_id, missing_tool_dependencies_root_folder = self.build_tool_dependencies_folder( - folder_id, - missing_tool_dependencies, - label="Missing tool dependencies", - missing=True, - new_install=False, - reinstalling=False, - ) - containers_dict["missing_tool_dependencies"] = missing_tool_dependencies_root_folder except Exception as e: log.debug(f"Exception in build_repository_containers: {str(e)}") return containers_dict From 4f603d68f7a9057539d8da833953143da3855b0c Mon Sep 17 00:00:00 2001 From: John Chilton Date: Mon, 17 Oct 2022 15:41:45 -0400 Subject: [PATCH 15/19] Refactor utility container stuff to avoid galaxy -> tool shed imports. --- .../tool_shed/util}/utility_container_manager.py | 0 lib/tool_shed/galaxy_install/utility_containers/__init__.py | 2 +- lib/tool_shed/utility_containers/__init__.py | 2 +- 3 files changed, 2 insertions(+), 2 deletions(-) rename lib/{tool_shed/utility_containers => galaxy/tool_shed/util}/utility_container_manager.py (100%) diff --git a/lib/tool_shed/utility_containers/utility_container_manager.py b/lib/galaxy/tool_shed/util/utility_container_manager.py similarity index 100% rename from lib/tool_shed/utility_containers/utility_container_manager.py rename to lib/galaxy/tool_shed/util/utility_container_manager.py diff --git a/lib/tool_shed/galaxy_install/utility_containers/__init__.py b/lib/tool_shed/galaxy_install/utility_containers/__init__.py index 260c7a7bbc5..7561ab21820 100644 --- a/lib/tool_shed/galaxy_install/utility_containers/__init__.py +++ b/lib/tool_shed/galaxy_install/utility_containers/__init__.py @@ -1,6 +1,6 @@ import logging -from tool_shed.utility_containers import utility_container_manager +from galaxy.tool_shed.util import utility_container_manager log = logging.getLogger(__name__) diff --git a/lib/tool_shed/utility_containers/__init__.py b/lib/tool_shed/utility_containers/__init__.py index a4e244ead7a..37d99394e3b 100644 --- a/lib/tool_shed/utility_containers/__init__.py +++ b/lib/tool_shed/utility_containers/__init__.py @@ -2,13 +2,13 @@ import logging import threading from galaxy import util +from galaxy.tool_shed.util import utility_container_manager from galaxy.tool_shed.util.common_util import parse_repository_dependency_tuple from galaxy.tool_shed.util.container_util import ( generate_repository_dependencies_key_for_repository, STRSEP, ) from tool_shed.util.readme_util import build_readme_files_dict -from . import utility_container_manager log = logging.getLogger(__name__) From be49c8edd1c6fd2098571bef4d4bf07b4d741442 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Mon, 17 Oct 2022 15:44:22 -0400 Subject: [PATCH 16/19] Rework GalaxyUtilityContainerManager to avoid a Galaxy-> Tool Shed import. --- .../galaxy_install/dependency_display.py | 45 ++++++++++++++++- .../utility_containers/__init__.py | 48 ------------------- lib/tool_shed/utility_containers/__init__.py | 6 +-- 3 files changed, 46 insertions(+), 53 deletions(-) delete mode 100644 lib/tool_shed/galaxy_install/utility_containers/__init__.py diff --git a/lib/tool_shed/galaxy_install/dependency_display.py b/lib/tool_shed/galaxy_install/dependency_display.py index 46b0b34be61..f51824037f6 100644 --- a/lib/tool_shed/galaxy_install/dependency_display.py +++ b/lib/tool_shed/galaxy_install/dependency_display.py @@ -3,7 +3,7 @@ import os from galaxy import util from galaxy.tool_shed.util.common_util import parse_repository_dependency_tuple -from tool_shed.galaxy_install.utility_containers import GalaxyUtilityContainerManager +from galaxy.tool_shed.util import utility_container_manager log = logging.getLogger(__name__) @@ -251,3 +251,46 @@ class DependencyDisplayer: if req_name == name and req_type == type: return False return True + + +class GalaxyUtilityContainerManager(utility_container_manager.UtilityContainerManager): + def __init__(self, app): + self.app = app + + def build_repository_containers( + self, + missing_repository_dependencies, + repository_dependencies, + ): + """ + Return a dictionary of containers for the received repository's dependencies and readme files for + display during installation to Galaxy. + """ + containers_dict = dict( + repository_dependencies=None, + missing_repository_dependencies=None, + ) + # Some of the tool dependency folders will include links to display tool dependency information, and + # some of these links require the repository id. However we need to be careful because sometimes the + # repository object is None. + try: + folder_id = 0 + # Installed repository dependencies container. + if repository_dependencies: + label = "Installed repository dependencies" + folder_id, repository_dependencies_root_folder = self.build_repository_dependencies_folder( + folder_id=folder_id, repository_dependencies=repository_dependencies, label=label, installed=True + ) + containers_dict["repository_dependencies"] = repository_dependencies_root_folder + # Missing repository dependencies container. + if missing_repository_dependencies: + folder_id, missing_repository_dependencies_root_folder = self.build_repository_dependencies_folder( + folder_id=folder_id, + repository_dependencies=missing_repository_dependencies, + label="Missing repository dependencies", + installed=False, + ) + containers_dict["missing_repository_dependencies"] = missing_repository_dependencies_root_folder + except Exception as e: + log.debug(f"Exception in build_repository_containers: {str(e)}") + return containers_dict diff --git a/lib/tool_shed/galaxy_install/utility_containers/__init__.py b/lib/tool_shed/galaxy_install/utility_containers/__init__.py deleted file mode 100644 index 7561ab21820..00000000000 --- a/lib/tool_shed/galaxy_install/utility_containers/__init__.py +++ /dev/null @@ -1,48 +0,0 @@ -import logging - -from galaxy.tool_shed.util import utility_container_manager - -log = logging.getLogger(__name__) - - -class GalaxyUtilityContainerManager(utility_container_manager.UtilityContainerManager): - def __init__(self, app): - self.app = app - - def build_repository_containers( - self, - missing_repository_dependencies, - repository_dependencies, - ): - """ - Return a dictionary of containers for the received repository's dependencies and readme files for - display during installation to Galaxy. - """ - containers_dict = dict( - repository_dependencies=None, - missing_repository_dependencies=None, - ) - # Some of the tool dependency folders will include links to display tool dependency information, and - # some of these links require the repository id. However we need to be careful because sometimes the - # repository object is None. - try: - folder_id = 0 - # Installed repository dependencies container. - if repository_dependencies: - label = "Installed repository dependencies" - folder_id, repository_dependencies_root_folder = self.build_repository_dependencies_folder( - folder_id=folder_id, repository_dependencies=repository_dependencies, label=label, installed=True - ) - containers_dict["repository_dependencies"] = repository_dependencies_root_folder - # Missing repository dependencies container. - if missing_repository_dependencies: - folder_id, missing_repository_dependencies_root_folder = self.build_repository_dependencies_folder( - folder_id=folder_id, - repository_dependencies=missing_repository_dependencies, - label="Missing repository dependencies", - installed=False, - ) - containers_dict["missing_repository_dependencies"] = missing_repository_dependencies_root_folder - except Exception as e: - log.debug(f"Exception in build_repository_containers: {str(e)}") - return containers_dict diff --git a/lib/tool_shed/utility_containers/__init__.py b/lib/tool_shed/utility_containers/__init__.py index 37d99394e3b..5d0c5b7700c 100644 --- a/lib/tool_shed/utility_containers/__init__.py +++ b/lib/tool_shed/utility_containers/__init__.py @@ -3,11 +3,11 @@ import threading from galaxy import util from galaxy.tool_shed.util import utility_container_manager -from galaxy.tool_shed.util.common_util import parse_repository_dependency_tuple from galaxy.tool_shed.util.container_util import ( generate_repository_dependencies_key_for_repository, STRSEP, ) +from galaxy.util.tool_shed.common_util import parse_repository_dependency_tuple from tool_shed.util.readme_util import build_readme_files_dict log = logging.getLogger(__name__) @@ -301,9 +301,7 @@ class ToolShedUtilityContainerManager(utility_container_manager.UtilityContainer # Readme files container. if metadata: if "readme_files" not in exclude and "readme_files" in metadata: - readme_files_dict = build_readme_files_dict( - self.app, repository, changeset_revision, metadata - ) + readme_files_dict = build_readme_files_dict(self.app, repository, changeset_revision, metadata) folder_id, readme_files_root_folder = self.build_readme_files_folder( folder_id, readme_files_dict ) From ead32bcfc5e6f6fe5feaaaa6c416b55ff7e2be75 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Mon, 17 Oct 2022 15:46:19 -0400 Subject: [PATCH 17/19] Move dependency_display to avoid Galaxy -> Tool Shed import. --- .../tool_shed/util}/dependency_display.py | 10 +++------- .../webapps/galaxy/controllers/admin_toolshed.py | 2 +- lib/tool_shed/webapp/controllers/repository.py | 2 +- lib/tool_shed/webapp/controllers/upload.py | 2 +- 4 files changed, 6 insertions(+), 10 deletions(-) rename lib/{tool_shed/galaxy_install => galaxy/tool_shed/util}/dependency_display.py (98%) diff --git a/lib/tool_shed/galaxy_install/dependency_display.py b/lib/galaxy/tool_shed/util/dependency_display.py similarity index 98% rename from lib/tool_shed/galaxy_install/dependency_display.py rename to lib/galaxy/tool_shed/util/dependency_display.py index f51824037f6..1d27431e495 100644 --- a/lib/tool_shed/galaxy_install/dependency_display.py +++ b/lib/galaxy/tool_shed/util/dependency_display.py @@ -2,8 +2,8 @@ import logging import os from galaxy import util -from galaxy.tool_shed.util.common_util import parse_repository_dependency_tuple from galaxy.tool_shed.util import utility_container_manager +from galaxy.util.tool_shed.common_util import parse_repository_dependency_tuple log = logging.getLogger(__name__) @@ -63,9 +63,7 @@ class DependencyDisplayer: prior_installation_required, only_if_compiling_contained_td, error, - ) = parse_repository_dependency_tuple( - repository_dependency_tup, contains_error=True - ) + ) = parse_repository_dependency_tuple(repository_dependency_tup, contains_error=True) if error: message += f"{error} " else: @@ -190,9 +188,7 @@ class DependencyDisplayer: message += set_environment_orphans_str return message - def populate_containers_dict_from_repository_metadata( - self, repository - ): + def populate_containers_dict_from_repository_metadata(self, repository): """ Retrieve necessary information from the received repository's metadata to populate the containers_dict for display. This method is called only from Galaxy (not the tool shed) diff --git a/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py b/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py index b96866a37f2..a5e2d32550a 100644 --- a/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py +++ b/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py @@ -25,7 +25,7 @@ from galaxy.util.tool_shed import ( encoding_util, ) from galaxy.web.form_builder import CheckboxField -from tool_shed.galaxy_install import dependency_display +from galaxy.tool_shed.util import dependency_display from tool_shed.util import shed_util_common as suc from tool_shed.util.repository_util import create_repo_info_dict from tool_shed.util.web_util import escape diff --git a/lib/tool_shed/webapp/controllers/repository.py b/lib/tool_shed/webapp/controllers/repository.py index bb694d404bf..bcbb6873d1d 100644 --- a/lib/tool_shed/webapp/controllers/repository.py +++ b/lib/tool_shed/webapp/controllers/repository.py @@ -24,6 +24,7 @@ from galaxy import ( util, web, ) +from galaxy.tool_shed.util import dependency_display from galaxy.tools.repositories import ValidationContext from galaxy.web.form_builder import ( CheckboxField, @@ -32,7 +33,6 @@ from galaxy.web.form_builder import ( from galaxy.web.legacy_framework import grids from galaxy.webapps.base.controller import BaseUIController from tool_shed.dependencies.repository import relation_builder -from tool_shed.galaxy_install import dependency_display from tool_shed.metadata import repository_metadata_manager from tool_shed.tools import ( tool_validator, diff --git a/lib/tool_shed/webapp/controllers/upload.py b/lib/tool_shed/webapp/controllers/upload.py index b4ea9b5631d..ef3ef730dd7 100644 --- a/lib/tool_shed/webapp/controllers/upload.py +++ b/lib/tool_shed/webapp/controllers/upload.py @@ -10,10 +10,10 @@ from galaxy import ( util, web, ) +from galaxy.tool_shed.util import dependency_display from galaxy.util import checkers from galaxy.webapps.base.controller import BaseUIController from tool_shed.dependencies import attribute_handlers -from tool_shed.galaxy_install import dependency_display from tool_shed.metadata import repository_metadata_manager from tool_shed.repository_types import util as rt_util from tool_shed.tools.data_table_manager import ShedToolDataTableManager From 217a21b4f6a028439140f6069d849319de65b0f4 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Mon, 17 Oct 2022 15:52:09 -0400 Subject: [PATCH 18/19] Another attempt to cleanup imports of admin_toolshed. --- .../galaxy/controllers/admin_toolshed.py | 30 ++++++++++++------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py b/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py index a5e2d32550a..a8d1c5e4b3f 100644 --- a/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py +++ b/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py @@ -12,12 +12,19 @@ from galaxy.exceptions import ConfigDoesNotAllowException from galaxy.tool_shed.galaxy_install import install_manager from galaxy.tool_shed.galaxy_install.repository_dependencies import repository_dependency_manager from galaxy.tool_shed.galaxy_install.tools import tool_panel_manager +from galaxy.tool_shed.util import dependency_display from galaxy.tool_shed.util.repository_util import ( create_or_update_tool_shed_repository, get_absolute_path_to_file_in_repository, get_installed_tool_shed_repository, get_tool_shed_repository_by_id, ) +from galaxy.tool_shed.util.shed_util_common import ( + clean_dependency_relationships, + get_ctx_rev, + get_tool_panel_config_tool_path_install_dir, + have_shed_tool_conf_for_install, +) from galaxy.tool_util.deps import views from galaxy.util import unicodify from galaxy.util.tool_shed import ( @@ -25,9 +32,12 @@ from galaxy.util.tool_shed import ( encoding_util, ) from galaxy.web.form_builder import CheckboxField -from galaxy.tool_shed.util import dependency_display -from tool_shed.util import shed_util_common as suc from tool_shed.util.repository_util import create_repo_info_dict +from tool_shed.util.shed_util_common import ( + get_repository_type_from_tool_shed, + get_requirements_from_repository, + get_tool_dependency_definition_metadata_from_tool_shed, +) from tool_shed.util.web_util import escape from .admin import AdminGalaxy @@ -153,7 +163,7 @@ class AdminToolshed(AdminGalaxy): tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry(trans.app, str(repository.tool_shed)) description = kwd.get("description", repository.description) status = "ok" - _, tool_path, _ = suc.get_tool_panel_config_tool_path_install_dir(trans.app, repository) + _, tool_path, _ = get_tool_panel_config_tool_path_install_dir(trans.app, repository) if repository.in_error_state: message = "This repository is not installed correctly (see the Repository installation error below). Choose " message += "Reset to install from the Repository Actions menu, correct problems if necessary and try " @@ -173,7 +183,7 @@ class AdminToolshed(AdminGalaxy): repository=repository, ) view = views.DependencyResolversView(self.app) - tool_requirements_d = suc.get_requirements_from_repository(repository) + tool_requirements_d = get_requirements_from_repository(repository) requirements_status = view.get_requirements_status(tool_requirements_d, repository.installed_tool_dependencies) management_dict = { "status": status, @@ -206,9 +216,9 @@ class AdminToolshed(AdminGalaxy): install_repository_dependencies = CheckboxField.is_checked(kwd.get("install_repository_dependencies", "")) install_tool_dependencies = CheckboxField.is_checked(kwd.get("install_tool_dependencies", "")) install_resolver_dependencies = CheckboxField.is_checked(kwd.get("install_resolver_dependencies", "")) - if not suc.have_shed_tool_conf_for_install(trans.app): + if not have_shed_tool_conf_for_install(trans.app): raise Exception("No valid shed tool configuration file available, please configure one") - shed_tool_conf, tool_path, relative_install_dir = suc.get_tool_panel_config_tool_path_install_dir( + shed_tool_conf, tool_path, relative_install_dir = get_tool_panel_config_tool_path_install_dir( trans.app, tool_shed_repository ) repository_clone_url = common_util.generate_clone_url_for_installed_repository( @@ -240,19 +250,19 @@ class AdminToolshed(AdminGalaxy): # Just in case the tool_section.id differs from tool_panel_section_id, which it shouldn't... tool_panel_section_id = str(tool_section.id) if tool_shed_repository.status == trans.install_model.ToolShedRepository.installation_status.UNINSTALLED: - repository_type = suc.get_repository_type_from_tool_shed( + repository_type = get_repository_type_from_tool_shed( trans.app, tool_shed_url, tool_shed_repository.name, tool_shed_repository.owner ) if repository_type == rt_util.TOOL_DEPENDENCY_DEFINITION: # Repositories of type tool_dependency_definition must get the latest # metadata from the Tool Shed since they have only a single installable # revision. - raw_text = suc.get_tool_dependency_definition_metadata_from_tool_shed( + raw_text = get_tool_dependency_definition_metadata_from_tool_shed( trans.app, tool_shed_url, tool_shed_repository.name, tool_shed_repository.owner ) new_meta = json.loads(raw_text) # Clean up old repository dependency and tool dependency relationships. - suc.clean_dependency_relationships(trans, new_meta, tool_shed_repository, tool_shed_url) + clean_dependency_relationships(trans, new_meta, tool_shed_repository, tool_shed_url) # The repository's status must be updated from 'Uninstalled' to 'New' when initiating reinstall # so the repository_installation_updater will function. tool_shed_repository = create_or_update_tool_shed_repository( @@ -268,7 +278,7 @@ class AdminToolshed(AdminGalaxy): owner=tool_shed_repository.owner, dist_to_shed=tool_shed_repository.dist_to_shed, ) - ctx_rev = suc.get_ctx_rev( + ctx_rev = get_ctx_rev( trans.app, tool_shed_url, tool_shed_repository.name, From 6547e421f8d3796ef8d7657ea35bfa194ac5c6e2 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Mon, 17 Oct 2022 17:01:28 -0400 Subject: [PATCH 19/19] linting fixes for admin_toolshed.py. --- lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py b/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py index a8d1c5e4b3f..ea077071571 100644 --- a/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py +++ b/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py @@ -160,10 +160,8 @@ class AdminToolshed(AdminGalaxy): repository = get_installed_tool_shed_repository(trans.app, repository_id) if repository is None: return trans.show_error_message("Invalid repository specified.") - tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry(trans.app, str(repository.tool_shed)) description = kwd.get("description", repository.description) status = "ok" - _, tool_path, _ = get_tool_panel_config_tool_path_install_dir(trans.app, repository) if repository.in_error_state: message = "This repository is not installed correctly (see the Repository installation error below). Choose " message += "Reset to install from the Repository Actions menu, correct problems if necessary and try "