From 525eec2cb6fa98df2d58c31404251c99734e86d1 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Wed, 21 Sep 2022 10:57:05 -0400 Subject: [PATCH 1/6] tool_util: Rework copy_sample_file with typing and less app dependence. --- .../tool_shed/galaxy_install/install_manager.py | 5 +++-- .../tool_shed/metadata/metadata_generator.py | 2 +- lib/galaxy/tool_shed/util/tool_util.py | 16 ++++++++++++---- lib/tool_shed/tools/tool_validator.py | 2 +- 4 files changed, 17 insertions(+), 8 deletions(-) diff --git a/lib/galaxy/tool_shed/galaxy_install/install_manager.py b/lib/galaxy/tool_shed/galaxy_install/install_manager.py index deca45a3ea5..563de8f02e5 100644 --- a/lib/galaxy/tool_shed/galaxy_install/install_manager.py +++ b/lib/galaxy/tool_shed/galaxy_install/install_manager.py @@ -162,7 +162,8 @@ class InstallRepositoryManager: ) sample_files = irmm_metadata_dict.get("sample_files", []) tool_index_sample_files = stdtm.get_tool_index_sample_files(sample_files) - tool_util.copy_sample_files(self.app, tool_index_sample_files, tool_path=tool_path) + tool_data_path = self.app.config.tool_data_path + tool_util.copy_sample_files(tool_data_path, tool_index_sample_files, tool_path=tool_path) sample_files_copied = [str(s) for s in tool_index_sample_files] repository_tools_tups = irmm.get_repository_tools_tups() if repository_tools_tups: @@ -177,7 +178,7 @@ class InstallRepositoryManager: # Copy remaining sample files included in the repository to the ~/tool-data directory of the # local Galaxy instance. tool_util.copy_sample_files( - self.app, sample_files, tool_path=tool_path, sample_files_copied=sample_files_copied + tool_data_path, sample_files, tool_path=tool_path, sample_files_copied=sample_files_copied ) self.tpm.add_to_tool_panel( repository_name=tool_shed_repository.name, diff --git a/lib/galaxy/tool_shed/metadata/metadata_generator.py b/lib/galaxy/tool_shed/metadata/metadata_generator.py index b831d19c9b1..c483bf3c324 100644 --- a/lib/galaxy/tool_shed/metadata/metadata_generator.py +++ b/lib/galaxy/tool_shed/metadata/metadata_generator.py @@ -314,7 +314,7 @@ class MetadataGenerator: # can load tools that depend on them. data_table_conf_xml_sample_files = [] for sample_file in sample_file_copy_paths: - tool_util.copy_sample_file(self.app, sample_file, dest_path=work_dir) + tool_util.copy_sample_file(self.app.config.tool_data_path, sample_file, dest_path=work_dir) # If the list of sample files includes a tool_data_table_conf.xml.sample file, load # its table elements into memory. relative_path, filename = os.path.split(sample_file) diff --git a/lib/galaxy/tool_shed/util/tool_util.py b/lib/galaxy/tool_shed/util/tool_util.py index f93370d4860..eeebc0066dd 100644 --- a/lib/galaxy/tool_shed/util/tool_util.py +++ b/lib/galaxy/tool_shed/util/tool_util.py @@ -1,6 +1,7 @@ import logging import os import shutil +from typing import Optional import galaxy.tools from galaxy import util @@ -39,14 +40,15 @@ def build_tool_panel_section_select_field(app): return select_field -def copy_sample_file(app, filename, dest_path=None): +def copy_sample_file(tool_data_path: str, filename: str, dest_path: Optional[str] = None) -> str: """ Copies a sample file at `filename` to `the dest_path` directory and strips the '.sample' extensions from `filename`. Returns the path to the copied file (with the .sample extension). """ if dest_path is None: - dest_path = os.path.abspath(app.config.tool_data_path) + dest_path = tool_data_path + assert dest_path sample_file_name = basic_util.strip_path(filename) copied_file = sample_file_name.rsplit(".sample", 1)[0] full_source_path = os.path.abspath(filename) @@ -63,7 +65,13 @@ def copy_sample_file(app, filename, dest_path=None): return non_sample_path -def copy_sample_files(app, sample_files, tool_path=None, sample_files_copied=None, dest_path=None): +def copy_sample_files( + tool_data_path: str, + sample_files, + tool_path: Optional[str] = None, + sample_files_copied=None, + dest_path: Optional[str] = None, +) -> None: """ Copy all appropriate files to dest_path in the local Galaxy environment that have not already been copied. Those that have been copied are contained in sample_files_copied. @@ -80,7 +88,7 @@ def copy_sample_files(app, sample_files, tool_path=None, sample_files_copied=Non filename = os.path.join(tool_path, filename) # Attempt to ensure we're copying an appropriate file. if is_data_index_sample_file(filename): - copy_sample_file(app, filename, dest_path=dest_path) + copy_sample_file(tool_data_path, filename, dest_path=dest_path) def generate_message_for_invalid_tools( diff --git a/lib/tool_shed/tools/tool_validator.py b/lib/tool_shed/tools/tool_validator.py index 0c5ec6a6f02..2a99c7eb2bd 100644 --- a/lib/tool_shed/tools/tool_validator.py +++ b/lib/tool_shed/tools/tool_validator.py @@ -65,7 +65,7 @@ class ToolValidator(GalaxyToolValidator): for name in files: if name.endswith(".sample"): relative_path = os.path.join(root, name) - tool_util.copy_sample_file(self.app, relative_path, dest_path=dest_path) + tool_util.copy_sample_file(self.app.config.tool_data_path, relative_path, dest_path=dest_path) sample_files.append(name) return sample_files From 81a29392735375f7b98d92e870c9620a515f06cb Mon Sep 17 00:00:00 2001 From: John Chilton Date: Wed, 21 Sep 2022 11:03:16 -0400 Subject: [PATCH 2/6] tool_util: Some more typing for tool_util. --- lib/galaxy/tool_shed/util/tool_util.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/tool_shed/util/tool_util.py b/lib/galaxy/tool_shed/util/tool_util.py index eeebc0066dd..b9e9c89dece 100644 --- a/lib/galaxy/tool_shed/util/tool_util.py +++ b/lib/galaxy/tool_shed/util/tool_util.py @@ -92,8 +92,13 @@ def copy_sample_files( def generate_message_for_invalid_tools( - app, invalid_file_tups, repository, metadata_dict, as_html=True, displaying_invalid_tool=False -): + app, + invalid_file_tups: list, + repository, + metadata_dict: Optional[dict], + as_html: bool = True, + displaying_invalid_tool: bool = False, +) -> str: if as_html: new_line = "
" bold_start = "" From 26b6962a1c6205e7dea1caebd58ff726e3f07f73 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Wed, 21 Sep 2022 11:10:48 -0400 Subject: [PATCH 3/6] tool_util: Do not re-export tool_util methods to tool shed that aren't used by the tool shed. --- lib/tool_shed/util/tool_util.py | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/lib/tool_shed/util/tool_util.py b/lib/tool_shed/util/tool_util.py index 63b85802f6a..d4df72e993c 100644 --- a/lib/tool_shed/util/tool_util.py +++ b/lib/tool_shed/util/tool_util.py @@ -1,25 +1,13 @@ from galaxy.tool_shed.util.tool_util import ( - build_shed_tool_conf_select_field, - build_tool_panel_section_select_field, copy_sample_file, copy_sample_files, generate_message_for_invalid_tools, - get_tool_path_install_dir, - handle_missing_index_file, - is_data_index_sample_file, new_state, - panel_entry_per_tool, ) __all__ = ( - "build_shed_tool_conf_select_field", - "build_tool_panel_section_select_field", "copy_sample_file", "copy_sample_files", "generate_message_for_invalid_tools", - "get_tool_path_install_dir", - "handle_missing_index_file", - "is_data_index_sample_file", "new_state", - "panel_entry_per_tool", ) From c8b59e8f10b467701ade54d6d2930aa83c656b2c Mon Sep 17 00:00:00 2001 From: John Chilton Date: Wed, 21 Sep 2022 11:12:08 -0400 Subject: [PATCH 4/6] tool_util: Remove unused method from tool_util. --- lib/galaxy/tool_shed/util/tool_util.py | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/lib/galaxy/tool_shed/util/tool_util.py b/lib/galaxy/tool_shed/util/tool_util.py index b9e9c89dece..ed094113f1a 100644 --- a/lib/galaxy/tool_shed/util/tool_util.py +++ b/lib/galaxy/tool_shed/util/tool_util.py @@ -146,23 +146,6 @@ def generate_message_for_invalid_tools( return message -def get_tool_path_install_dir(partial_install_dir, shed_tool_conf_dict, tool_dict, config_elems): - for elem in config_elems: - if elem.tag == "tool": - if elem.get("guid") == tool_dict["guid"]: - tool_path = shed_tool_conf_dict["tool_path"] - relative_install_dir = os.path.join(tool_path, partial_install_dir) - return tool_path, relative_install_dir - elif elem.tag == "section": - for section_elem in elem: - if section_elem.tag == "tool": - if section_elem.get("guid") == tool_dict["guid"]: - tool_path = shed_tool_conf_dict["tool_path"] - relative_install_dir = os.path.join(tool_path, partial_install_dir) - return tool_path, relative_install_dir - return None, None - - def handle_missing_index_file(app, tool_path, sample_files, repository_tools_tups, sample_files_copied): """ Inspect each tool to see if it has any input parameters that are dynamically @@ -267,7 +250,6 @@ __all__ = ( "copy_sample_file", "copy_sample_files", "generate_message_for_invalid_tools", - "get_tool_path_install_dir", "handle_missing_index_file", "is_data_index_sample_file", "new_state", From b6fbd05522d676b8673c820b7f5fb5ef6647e378 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Wed, 21 Sep 2022 11:13:52 -0400 Subject: [PATCH 5/6] tool_util: do not export method only used in this file --- lib/galaxy/tool_shed/util/tool_util.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/galaxy/tool_shed/util/tool_util.py b/lib/galaxy/tool_shed/util/tool_util.py index ed094113f1a..59c484bd665 100644 --- a/lib/galaxy/tool_shed/util/tool_util.py +++ b/lib/galaxy/tool_shed/util/tool_util.py @@ -87,7 +87,7 @@ def copy_sample_files( if tool_path: filename = os.path.join(tool_path, filename) # Attempt to ensure we're copying an appropriate file. - if is_data_index_sample_file(filename): + if _is_data_index_sample_file(filename): copy_sample_file(tool_data_path, filename, dest_path=dest_path) @@ -171,7 +171,7 @@ def handle_missing_index_file(app, tool_path, sample_files, repository_tools_tup return repository_tools_tups, sample_files_copied -def is_data_index_sample_file(file_path): +def _is_data_index_sample_file(file_path): """ Attempt to determine if a .sample file is appropriate for copying to ~/tool-data when a tool shed repository is being installed into a Galaxy instance. @@ -251,7 +251,6 @@ __all__ = ( "copy_sample_files", "generate_message_for_invalid_tools", "handle_missing_index_file", - "is_data_index_sample_file", "new_state", "panel_entry_per_tool", ) From 23e78c542568480869dc18b82c3ea572bfd912ce Mon Sep 17 00:00:00 2001 From: John Chilton Date: Wed, 21 Sep 2022 11:38:57 -0400 Subject: [PATCH 6/6] Import galaxy version of tool shed utils in admin_toolshed. --- .../webapps/galaxy/controllers/admin_toolshed.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py b/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py index eb4a84e54aa..e9cc2e8bb7a 100644 --- a/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py +++ b/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py @@ -14,20 +14,24 @@ from galaxy.exceptions import ConfigDoesNotAllowException from galaxy.tool_shed.galaxy_install import install_manager from galaxy.tool_shed.galaxy_install.repository_dependencies import repository_dependency_manager from galaxy.tool_shed.galaxy_install.tools import tool_panel_manager +from galaxy.tool_shed.util import ( + hg_util, + tool_dependency_util, + tool_util, +) from galaxy.tool_util.deps import views from galaxy.util import unicodify +from galaxy.util.tool_shed import ( + common_util, + encoding_util, +) from galaxy.web.form_builder import CheckboxField from tool_shed.galaxy_install import dependency_display from tool_shed.galaxy_install.grids import admin_toolshed_grids from tool_shed.util import ( - common_util, - encoding_util, - hg_util, readme_util, repository_util, shed_util_common as suc, - tool_dependency_util, - tool_util, ) from tool_shed.util.web_util import escape from .admin import AdminGalaxy