From 90858d3137eeaa0a44ea74eb3ea2adf421e636f5 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Tue, 13 Sep 2022 17:45:57 -0400 Subject: [PATCH 1/5] Move create_element into the only place it is used. --- lib/galaxy/util/tool_shed/xml_util.py | 51 ++-------------- .../dependencies/attribute_handlers.py | 61 ++++++++++++++++++- lib/tool_shed/util/xml_util.py | 2 - 3 files changed, 64 insertions(+), 50 deletions(-) diff --git a/lib/galaxy/util/tool_shed/xml_util.py b/lib/galaxy/util/tool_shed/xml_util.py index 3b0bf1ec73f..c68b27123f4 100644 --- a/lib/galaxy/util/tool_shed/xml_util.py +++ b/lib/galaxy/util/tool_shed/xml_util.py @@ -1,6 +1,10 @@ import logging import os import tempfile +from typing import ( + Optional, + Tuple, +) from galaxy.util import ( etree, @@ -21,51 +25,7 @@ def create_and_write_tmp_file(elem): return tmp_filename -def create_element(tag, attributes=None, sub_elements=None): - """ - Create a new element whose tag is the value of the received tag, and whose attributes are all - key / value pairs in the received attributes and sub_elements. - """ - if tag: - elem = etree.Element(tag) - if attributes: - # The received attributes is an odict to preserve ordering. - for k, v in attributes.items(): - elem.set(k, v) - if sub_elements: - # The received attributes is an odict. These handle information that tends to be - # long text including paragraphs (e.g., description and long_description. - for k, v in sub_elements.items(): - # Don't include fields that are blank. - if v: - if k == "packages": - # The received sub_elements is an odict whose key is 'packages' and whose - # value is a list of ( name, version ) tuples. - for v_tuple in v: - sub_elem = etree.SubElement(elem, "package") - sub_elem_name, sub_elem_version = v_tuple - sub_elem.set("name", sub_elem_name) - sub_elem.set("version", sub_elem_version) - elif isinstance(v, list): - sub_elem = etree.SubElement(elem, k) - # If v is a list, then it must be a list of tuples where the first - # item is the tag and the second item is the text value. - for v_tuple in v: - if len(v_tuple) == 2: - v_tag = v_tuple[0] - v_text = v_tuple[1] - # Don't include fields that are blank. - if v_text: - v_elem = etree.SubElement(sub_elem, v_tag) - v_elem.text = v_text - else: - sub_elem = etree.SubElement(elem, k) - sub_elem.text = v - return elem - return None - - -def parse_xml(file_name, check_exists=True): +def parse_xml(file_name, check_exists=True) -> Tuple[Optional[etree.ElementTree], str]: """Returns a parsed xml tree with comments intact.""" error_message = "" if check_exists and not os.path.exists(file_name): @@ -83,6 +43,5 @@ def parse_xml(file_name, check_exists=True): __all__ = ( "create_and_write_tmp_file", - "create_element", "parse_xml", ) diff --git a/lib/tool_shed/dependencies/attribute_handlers.py b/lib/tool_shed/dependencies/attribute_handlers.py index 726ab53c5a9..57d312a832e 100644 --- a/lib/tool_shed/dependencies/attribute_handlers.py +++ b/lib/tool_shed/dependencies/attribute_handlers.py @@ -1,7 +1,16 @@ import copy import logging +from typing import ( + Dict, + List, + Optional, + Tuple, +) -from galaxy.util import asbool +from galaxy.util import ( + asbool, + etree, +) from galaxy.web import url_for from tool_shed.dependencies.tool import tag_attribute_handler from tool_shed.repository_types.util import ( @@ -96,7 +105,7 @@ class RepositoryDependencyAttributeHandler: prior_installation_required = elem.get("prior_installation_required") if asbool(prior_installation_required): attributes["prior_installation_required"] = "True" - new_elem = xml_util.create_element("repository", attributes=attributes, sub_elements=sub_elements) + new_elem = _create_element("repository", attributes=attributes, sub_elements=sub_elements) altered = True return altered, new_elem, error_message # From here on we're populating the toolshed and changeset_revision attributes if necessary. @@ -200,3 +209,51 @@ class ToolDependencyAttributeHandler: root = tree.getroot() altered, new_root, error_message = tah.process_config(root, skip_actions_tags=False) return altered, new_root, error_message + + +def _create_element( + tag: str, + attributes: Optional[Dict[str, str]] = None, + sub_elements: Optional[Dict[str, List[Tuple[str, str]]]] = None, +) -> Optional[etree.Element]: + """ + Create a new element whose tag is the value of the received tag, and whose attributes are all + key / value pairs in the received attributes and sub_elements. + """ + if tag: + elem = etree.Element(tag) + if attributes: + # The received attributes is an odict to preserve ordering. + for k, attribute_value in attributes.items(): + elem.set(k, attribute_value) + if sub_elements: + # The received attributes is an odict. These handle information that tends to be + # long text including paragraphs (e.g., description and long_description. + for k, v in sub_elements.items(): + # Don't include fields that are blank. + if v: + if k == "packages": + # The received sub_elements is an odict whose key is 'packages' and whose + # value is a list of ( name, version ) tuples. + for v_tuple in v: + sub_elem = etree.SubElement(elem, "package") + sub_elem_name, sub_elem_version = v_tuple + sub_elem.set("name", sub_elem_name) + sub_elem.set("version", sub_elem_version) + elif isinstance(v, list): + sub_elem = etree.SubElement(elem, k) + # If v is a list, then it must be a list of tuples where the first + # item is the tag and the second item is the text value. + for v_tuple in v: + if len(v_tuple) == 2: + v_tag = v_tuple[0] + v_text = v_tuple[1] + # Don't include fields that are blank. + if v_text: + v_elem = etree.SubElement(sub_elem, v_tag) + v_elem.text = v_text + else: + sub_elem = etree.SubElement(elem, k) + sub_elem.text = v + return elem + return None diff --git a/lib/tool_shed/util/xml_util.py b/lib/tool_shed/util/xml_util.py index 1944d9eae02..7705439478b 100644 --- a/lib/tool_shed/util/xml_util.py +++ b/lib/tool_shed/util/xml_util.py @@ -1,11 +1,9 @@ from galaxy.util.tool_shed.xml_util import ( create_and_write_tmp_file, - create_element, parse_xml, ) __all__ = ( "create_and_write_tmp_file", - "create_element", "parse_xml", ) From d26cade072225b5f595e667282d964e584f3dc2a Mon Sep 17 00:00:00 2001 From: John Chilton Date: Tue, 13 Sep 2022 17:50:16 -0400 Subject: [PATCH 2/5] Remove more unused tool shed code. --- .../installed_repository_manager.py | 34 ------------------- 1 file changed, 34 deletions(-) diff --git a/lib/galaxy/tool_shed/galaxy_install/installed_repository_manager.py b/lib/galaxy/tool_shed/galaxy_install/installed_repository_manager.py index 22ac9abe823..9587a418deb 100644 --- a/lib/galaxy/tool_shed/galaxy_install/installed_repository_manager.py +++ b/lib/galaxy/tool_shed/galaxy_install/installed_repository_manager.py @@ -594,40 +594,6 @@ class InstalledRepositoryManager: return relative_path return None - def get_runtime_dependent_tool_dependency_tuples(self, tool_dependency, status=None): - """ - Return the list of tool dependency objects that require the received tool dependency at run time. The returned - list will be filtered by the received status if it is not None. This method is called only from Galaxy. - """ - runtime_dependent_tool_dependency_tups = [] - required_env_shell_file_path = tool_dependency.get_env_shell_file_path(self.app) - if required_env_shell_file_path: - required_env_shell_file_path = os.path.abspath(required_env_shell_file_path) - if required_env_shell_file_path is not None: - for td in self.app.install_model.context.query(self.app.install_model.ToolDependency): - if status is None or td.status == status: - env_shell_file_path = td.get_env_shell_file_path(self.app) - if env_shell_file_path is not None: - try: - contents = open(env_shell_file_path).read() - except Exception as e: - contents = None - log.debug( - "Error reading file %s, so cannot determine if package %s requires package %s at run time: %s" - % (str(env_shell_file_path), str(td.name), str(tool_dependency.name), str(e)) - ) - if contents is not None and contents.find(required_env_shell_file_path) >= 0: - td_tuple = self.get_tool_dependency_tuple_for_installed_repository_manager(td) - runtime_dependent_tool_dependency_tups.append(td_tuple) - return runtime_dependent_tool_dependency_tups - - def get_tool_dependency_tuple_for_installed_repository_manager(self, tool_dependency): - if tool_dependency.type is None: - type = None - else: - type = str(tool_dependency.type) - return (tool_dependency.tool_shed_repository_id, str(tool_dependency.name), str(tool_dependency.version), type) - def handle_existing_tool_dependencies_that_changed_in_update( self, repository, original_dependency_dict, new_dependency_dict ): From 9351be5c96b2e03c7e0a4dd4f683e6fd80d57d07 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Mon, 12 Sep 2022 16:05:30 -0400 Subject: [PATCH 3/5] Refactor method name to make clear it is private. --- lib/galaxy/tool_shed/galaxy_install/install_manager.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/galaxy/tool_shed/galaxy_install/install_manager.py b/lib/galaxy/tool_shed/galaxy_install/install_manager.py index 153ed17e8a2..07464fee24f 100644 --- a/lib/galaxy/tool_shed/galaxy_install/install_manager.py +++ b/lib/galaxy/tool_shed/galaxy_install/install_manager.py @@ -42,7 +42,7 @@ class InstallRepositoryManager: else: self.tpm = tpm - def get_repository_components_for_installation( + def _get_repository_components_for_installation( self, encoded_tsr_id, encoded_tsr_ids, repo_info_dicts, tool_panel_section_keys ): """ @@ -468,7 +468,7 @@ class InstallRepositoryManager: self.install_model.ToolShedRepository.installation_status.UNINSTALLED, ]: repositories_for_installation.append(repository) - repo_info_dict, tool_panel_section_key = self.get_repository_components_for_installation( + repo_info_dict, tool_panel_section_key = self._get_repository_components_for_installation( tsr_id, ordered_tsr_ids, ordered_repo_info_dicts, ordered_tool_panel_section_keys ) filtered_repo_info_dicts.append(repo_info_dict) @@ -797,7 +797,7 @@ class InstallRepositoryManager: ( prior_repo_info_dict, prior_tool_panel_section_key, - ) = self.get_repository_components_for_installation( + ) = self._get_repository_components_for_installation( prior_install_required_id, tsr_ids, repo_info_dicts, @@ -806,7 +806,7 @@ class InstallRepositoryManager: ordered_tsr_ids.append(prior_install_required_id) ordered_repo_info_dicts.append(prior_repo_info_dict) ordered_tool_panel_section_keys.append(prior_tool_panel_section_key) - repo_info_dict, tool_panel_section_key = self.get_repository_components_for_installation( + repo_info_dict, tool_panel_section_key = self._get_repository_components_for_installation( tsr_id, tsr_ids, repo_info_dicts, tool_panel_section_keys=tool_panel_section_keys ) if tsr_id not in ordered_tsr_ids: From 0869bfd17f0709ce863da33688f1b4c6338261ba Mon Sep 17 00:00:00 2001 From: John Chilton Date: Mon, 12 Sep 2022 16:20:30 -0400 Subject: [PATCH 4/5] Rename method to indicate an internal method. --- lib/galaxy/tool_shed/metadata/metadata_generator.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/tool_shed/metadata/metadata_generator.py b/lib/galaxy/tool_shed/metadata/metadata_generator.py index 69a54c4c407..b831d19c9b1 100644 --- a/lib/galaxy/tool_shed/metadata/metadata_generator.py +++ b/lib/galaxy/tool_shed/metadata/metadata_generator.py @@ -120,7 +120,7 @@ class MetadataGenerator: suc.REPOSITORY_DATA_MANAGER_CONFIG_FILENAME, ] - def generate_data_manager_metadata( + def _generate_data_manager_metadata( self, repo_dir, data_manager_config_filename, metadata_dict, shed_config_dict=None ): """ @@ -415,7 +415,7 @@ class MetadataGenerator: ) # Handle any data manager entries data_manager_config = get_config_from_disk(suc.REPOSITORY_DATA_MANAGER_CONFIG_FILENAME, files_dir) - metadata_dict = self.generate_data_manager_metadata( + metadata_dict = self._generate_data_manager_metadata( files_dir, data_manager_config, metadata_dict, shed_config_dict=self.shed_config_dict ) From b2a4cdeb82335b64e1e6c526c85f54d6999241cd Mon Sep 17 00:00:00 2001 From: John Chilton Date: Mon, 12 Sep 2022 16:36:30 -0400 Subject: [PATCH 5/5] Remove now unused function. --- lib/galaxy/model/tool_shed_install/__init__.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/lib/galaxy/model/tool_shed_install/__init__.py b/lib/galaxy/model/tool_shed_install/__init__.py index 2cb34b7338a..4f9995ebcb9 100644 --- a/lib/galaxy/model/tool_shed_install/__init__.py +++ b/lib/galaxy/model/tool_shed_install/__init__.py @@ -727,13 +727,6 @@ class ToolDependency(Base, _HasTable): self.installation_status.UNINSTALLED, ] - def get_env_shell_file_path(self, app): - installation_directory = self.installation_directory(app) - file_path = os.path.join(installation_directory, "env.sh") - if os.path.exists(file_path): - return file_path - return None - @property def in_error_state(self): return self.status == self.installation_status.ERROR