From ad42060dff17a7dd39bea89cc2dcf0599dc6b9e6 Mon Sep 17 00:00:00 2001 From: selten Date: Thu, 12 Sep 2019 20:32:36 +0200 Subject: [PATCH 1/5] Fix a bug that occurs while installing a data manager. Upon loading of the tool_data_table_conf.xml.sample file, it would immediately parse this. However further down the line, the example files present in a tool shed repository are required to properly install the data manager. --- lib/tool_shed/metadata/metadata_generator.py | 22 ++++++++++++-------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/lib/tool_shed/metadata/metadata_generator.py b/lib/tool_shed/metadata/metadata_generator.py index a37ddac7b0c..d2bed64636e 100644 --- a/lib/tool_shed/metadata/metadata_generator.py +++ b/lib/tool_shed/metadata/metadata_generator.py @@ -359,21 +359,25 @@ class MetadataGenerator(object): metadata_dict['sample_files'] = sample_file_metadata_paths # Copy all sample files included in the repository to a single directory location so we # can load tools that depend on them. + datatableconfxmlsample_found = False for sample_file in sample_file_copy_paths: tool_util.copy_sample_file(self.app, sample_file, dest_path=work_dir) # If the list of sample files includes a tool_data_table_conf.xml.sample file, load # its table elements into memory. relative_path, filename = os.path.split(sample_file) if filename == 'tool_data_table_conf.xml.sample': - # We create a new ToolDataTableManager to avoid adding entries to the app-wide - # tool data tables. This is only used for checking that the data table is valid. - new_table_elems, error_message = \ - validation_context.tool_data_tables.add_new_entries_from_config_file(config_filename=sample_file, - tool_data_path=work_dir, - shed_tool_data_table_config=work_dir, - persist=False) - if error_message: - self.invalid_file_tups.append((filename, error_message)) + datatableconfxmlsample_found = True + + if datatableconfxmlsample_found: + # We create a new ToolDataTableManager to avoid adding entries to the app-wide + # tool data tables. This is only used for checking that the data table is valid. + new_table_elems, error_message = \ + validation_context.tool_data_tables.add_new_entries_from_config_file(config_filename=sample_file, + tool_data_path=work_dir, + shed_tool_data_table_config=work_dir, + persist=False) + if error_message: + self.invalid_file_tups.append((filename, error_message)) for root, dirs, files in os.walk(files_dir): if root.find('.hg') < 0 and root.find('hgrc') < 0: if '.hg' in dirs: From a02c701171541b743926fec7a3c4405bd436e642 Mon Sep 17 00:00:00 2001 From: selten Date: Thu, 12 Sep 2019 20:42:48 +0200 Subject: [PATCH 2/5] Use file instead of boolean --- lib/tool_shed/metadata/metadata_generator.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/tool_shed/metadata/metadata_generator.py b/lib/tool_shed/metadata/metadata_generator.py index d2bed64636e..51bfe1775ce 100644 --- a/lib/tool_shed/metadata/metadata_generator.py +++ b/lib/tool_shed/metadata/metadata_generator.py @@ -359,20 +359,20 @@ class MetadataGenerator(object): metadata_dict['sample_files'] = sample_file_metadata_paths # Copy all sample files included in the repository to a single directory location so we # can load tools that depend on them. - datatableconfxmlsample_found = False + data_table_conf_xml_sample_file = None for sample_file in sample_file_copy_paths: tool_util.copy_sample_file(self.app, sample_file, dest_path=work_dir) # If the list of sample files includes a tool_data_table_conf.xml.sample file, load # its table elements into memory. relative_path, filename = os.path.split(sample_file) if filename == 'tool_data_table_conf.xml.sample': - datatableconfxmlsample_found = True + data_table_conf_xml_sample_file = sample_file - if datatableconfxmlsample_found: + if data_table_conf_xml_sample_file is not None: # We create a new ToolDataTableManager to avoid adding entries to the app-wide # tool data tables. This is only used for checking that the data table is valid. new_table_elems, error_message = \ - validation_context.tool_data_tables.add_new_entries_from_config_file(config_filename=sample_file, + validation_context.tool_data_tables.add_new_entries_from_config_file(config_filename=data_table_conf_xml_sample_file, tool_data_path=work_dir, shed_tool_data_table_config=work_dir, persist=False) From aa5da468a49add1dfb0c0092ab88c3223bce8526 Mon Sep 17 00:00:00 2001 From: selten Date: Fri, 13 Sep 2019 11:21:52 +0200 Subject: [PATCH 3/5] Support multiple `data_table_conf.xml.sample` files in a repository --- lib/tool_shed/metadata/metadata_generator.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/tool_shed/metadata/metadata_generator.py b/lib/tool_shed/metadata/metadata_generator.py index 51bfe1775ce..30026ab8c46 100644 --- a/lib/tool_shed/metadata/metadata_generator.py +++ b/lib/tool_shed/metadata/metadata_generator.py @@ -359,16 +359,16 @@ class MetadataGenerator(object): metadata_dict['sample_files'] = sample_file_metadata_paths # Copy all sample files included in the repository to a single directory location so we # can load tools that depend on them. - data_table_conf_xml_sample_file = None + data_table_conf_xml_sample_files = [] for sample_file in sample_file_copy_paths: tool_util.copy_sample_file(self.app, sample_file, dest_path=work_dir) # If the list of sample files includes a tool_data_table_conf.xml.sample file, load # its table elements into memory. relative_path, filename = os.path.split(sample_file) if filename == 'tool_data_table_conf.xml.sample': - data_table_conf_xml_sample_file = sample_file + data_table_conf_xml_sample_files.append(sample_file) - if data_table_conf_xml_sample_file is not None: + for data_table_conf_xml_sample_file in data_table_conf_xml_sample_files: # We create a new ToolDataTableManager to avoid adding entries to the app-wide # tool data tables. This is only used for checking that the data table is valid. new_table_elems, error_message = \ From a084686fc56c2be63c96f16076418bfe6d8fb8b9 Mon Sep 17 00:00:00 2001 From: selten Date: Fri, 13 Sep 2019 13:10:18 +0200 Subject: [PATCH 4/5] Add traceback to loading of data managers so diagnosing issues with data managers XML files becomes easier. --- lib/galaxy/tools/data/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/galaxy/tools/data/__init__.py b/lib/galaxy/tools/data/__init__.py index 0149a444b1c..7694bbb0ddf 100644 --- a/lib/galaxy/tools/data/__init__.py +++ b/lib/galaxy/tools/data/__init__.py @@ -20,6 +20,7 @@ from tempfile import NamedTemporaryFile from xml.etree import ElementTree import requests +import traceback from galaxy import util from galaxy.util.dictifiable import Dictifiable @@ -158,6 +159,7 @@ class ToolDataTableManager(object): except Exception as e: error_message = 'Error attempting to parse file %s: %s' % (str(os.path.split(config_filename)[1]), util.unicodify(e)) log.debug(error_message) + log.debug(traceback.format_exc()) table_elems = [] if persist: # Persist Galaxy's version of the changed tool_data_table_conf.xml file. From 46155acfeb4d494b86446c3404a38f41e832f60f Mon Sep 17 00:00:00 2001 From: selten Date: Fri, 13 Sep 2019 13:28:42 +0200 Subject: [PATCH 5/5] Change to suggested method for error handling --- lib/galaxy/tools/data/__init__.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/lib/galaxy/tools/data/__init__.py b/lib/galaxy/tools/data/__init__.py index 7694bbb0ddf..f302e1441d4 100644 --- a/lib/galaxy/tools/data/__init__.py +++ b/lib/galaxy/tools/data/__init__.py @@ -20,7 +20,6 @@ from tempfile import NamedTemporaryFile from xml.etree import ElementTree import requests -import traceback from galaxy import util from galaxy.util.dictifiable import Dictifiable @@ -158,8 +157,7 @@ class ToolDataTableManager(object): from_shed_config=True) except Exception as e: error_message = 'Error attempting to parse file %s: %s' % (str(os.path.split(config_filename)[1]), util.unicodify(e)) - log.debug(error_message) - log.debug(traceback.format_exc()) + log.debug(error_message, exc_info=True) table_elems = [] if persist: # Persist Galaxy's version of the changed tool_data_table_conf.xml file.