From 52bb33c51943138f643160bb6f9a2263af3fb167 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Thu, 1 Apr 2021 13:18:52 -0400 Subject: [PATCH 1/6] Fix auto-pair naming in list paired creator. Fixes (or at least improves) https://github.com/galaxyproject/galaxy/issues/11744. --- .../PairedListCollectionCreator.vue | 23 ++++++++----------- 1 file changed, 9 insertions(+), 14 deletions(-) diff --git a/client/src/components/Collections/PairedListCollectionCreator.vue b/client/src/components/Collections/PairedListCollectionCreator.vue index cd867e44cd9..1246c598d28 100644 --- a/client/src/components/Collections/PairedListCollectionCreator.vue +++ b/client/src/components/Collections/PairedListCollectionCreator.vue @@ -693,27 +693,22 @@ export default { /** try to find a good pair name for the given fwd and rev datasets */ _guessNameForPair: function (fwd, rev, removeExtensions) { removeExtensions = removeExtensions ? removeExtensions : this.removeExtensions; - var fwdName = fwd.name; - var revName = rev.name; - - var lcs = this._naiveStartingAndEndingLCS( - fwdName.replace(new RegExp(this.filters[0]), ""), - revName.replace(new RegExp(this.filters[1]), "") - ); - + let fwdName = fwd.name; + let revName = rev.name; + const fwdNameFilter = fwdName.replace(new RegExp(this.forwardFilter), ""); + const revNameFilter = revName.replace(new RegExp(this.reverseFilter), ""); + let lcs = this._naiveStartingAndEndingLCS(fwdNameFilter, revNameFilter); /** remove url prefix if files were uploaded by url */ - var lastSlashIndex = lcs.lastIndexOf("/"); + const lastSlashIndex = lcs.lastIndexOf("/"); if (lastSlashIndex > 0) { - var urlprefix = lcs.slice(0, lastSlashIndex + 1); + const urlprefix = lcs.slice(0, lastSlashIndex + 1); lcs = lcs.replace(urlprefix, ""); - fwdName = fwdName.replace(extension, ""); - revName = revName.replace(extension, ""); } if (removeExtensions) { - var lastDotIndex = lcs.lastIndexOf("."); + const lastDotIndex = lcs.lastIndexOf("."); if (lastDotIndex > 0) { - var extension = lcs.slice(lastDotIndex, lcs.length); + const extension = lcs.slice(lastDotIndex, lcs.length); lcs = lcs.replace(extension, ""); fwdName = fwdName.replace(extension, ""); revName = revName.replace(extension, ""); From ccf6bca065d42ba5df3680aecde2a863804dc7ac Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Tue, 22 Jun 2021 20:10:03 +0200 Subject: [PATCH 2/6] Allow ``decompress="true"`` in test comparison methods This means you can provide a short fragment of expected content (compressed or uncompressed) in your test data and compare it with the produced output. This was only enabled for the default `files_diff` method previously. --- lib/galaxy/tool_util/verify/__init__.py | 44 +++++++++++++------------ test/unit/test_verify.py | 10 ++++-- 2 files changed, 30 insertions(+), 24 deletions(-) diff --git a/lib/galaxy/tool_util/verify/__init__.py b/lib/galaxy/tool_util/verify/__init__.py index 5414d31ac75..6e2f5c9abdb 100644 --- a/lib/galaxy/tool_util/verify/__init__.py +++ b/lib/galaxy/tool_util/verify/__init__.py @@ -224,8 +224,17 @@ def files_delta(file1, file2, attributes=None): raise AssertionError('Files %s=%db but %s=%db - compare by size (delta_frac=%s) failed' % (file1, s1, file2, s2, delta_frac)) +def get_compressed_formats(attributes): + attributes = attributes or {} + decompress = attributes.get("decompress") + # None means all compressed formats are allowed + return None if decompress else [] + + def files_diff(file1, file2, attributes=None): """Check the contents of 2 files for differences.""" + attributes = attributes or {} + def get_lines_diff(diff): count = 0 for line in diff: @@ -234,14 +243,7 @@ def files_diff(file1, file2, attributes=None): return count if not filecmp.cmp(file1, file2, shallow=False): - if attributes is None: - attributes = {} - decompress = attributes.get("decompress", None) - if decompress: - # None means all compressed formats are allowed - compressed_formats = None - else: - compressed_formats = [] + compressed_formats = get_compressed_formats(attributes) is_pdf = False try: with get_fileobj(file2, compressed_formats=compressed_formats) as fh: @@ -309,12 +311,14 @@ def files_diff(file1, file2, attributes=None): def files_re_match(file1, file2, attributes=None): """Check the contents of 2 files for differences using re.match.""" + attributes = attributes or {} join_char = '' to_strip = os.linesep + compressed_formats = get_compressed_formats(attributes) try: - with open(file2, encoding='utf-8') as fh: + with get_fileobj(file2, compressed_formats=compressed_formats) as fh: history_data = fh.readlines() - with open(file1, encoding='utf-8') as fh: + with get_fileobj(file1, compressed_formats=compressed_formats) as fh: local_file = fh.readlines() except UnicodeDecodeError: join_char = b'' @@ -324,8 +328,6 @@ def files_re_match(file1, file2, attributes=None): with open(file1, 'rb') as fh: local_file = fh.readlines() assert len(local_file) == len(history_data), 'Data File and Regular Expression File contain a different number of lines (%d != %d)\nHistory Data (first 40 lines):\n%s' % (len(local_file), len(history_data), join_char.join(history_data[:40])) - if attributes is None: - attributes = {} if attributes.get('sort', False): history_data.sort() lines_diff = int(attributes.get('lines_diff', 0)) @@ -343,11 +345,13 @@ def files_re_match(file1, file2, attributes=None): def files_re_match_multiline(file1, file2, attributes=None): """Check the contents of 2 files for differences using re.match in multiline mode.""" + attributes = attributes or {} join_char = '' + compressed_formats = get_compressed_formats(attributes) try: - with open(file2, encoding='utf-8') as fh: + with get_fileobj(file2, compressed_formats=compressed_formats) as fh: history_data = fh.readlines() - with open(file1, encoding='utf-8') as fh: + with get_fileobj(file1, compressed_formats=compressed_formats) as fh: local_file = fh.read() except UnicodeDecodeError: join_char = b'' @@ -355,8 +359,6 @@ def files_re_match_multiline(file1, file2, attributes=None): history_data = fh.readlines() with open(file1, 'rb') as fh: local_file = fh.read() - if attributes is None: - attributes = {} if attributes.get('sort', False): history_data.sort() history_data = join_char.join(history_data) @@ -367,11 +369,13 @@ def files_re_match_multiline(file1, file2, attributes=None): def files_contains(file1, file2, attributes=None): """Check the contents of file2 for substrings found in file1, on a per-line basis.""" # TODO: allow forcing ordering of contains + attributes = attributes or {} to_strip = os.linesep + compressed_formats = get_compressed_formats(attributes) try: - with open(file2, encoding='utf-8') as fh: + with get_fileobj(file2, compressed_formats=compressed_formats) as fh: history_data = fh.read() - with open(file1, encoding='utf-8') as fh: + with get_fileobj(file1, compressed_formats=compressed_formats) as fh: local_file = fh.readlines() except UnicodeDecodeError: to_strip = os.linesep.encode('utf-8') @@ -379,8 +383,6 @@ def files_contains(file1, file2, attributes=None): history_data = fh.read() with open(file1, 'rb') as fh: local_file = fh.readlines() - if attributes is None: - attributes = {} lines_diff = int(attributes.get('lines_diff', 0)) line_diff_count = 0 for contains in local_file: @@ -388,4 +390,4 @@ def files_contains(file1, file2, attributes=None): if contains not in history_data: line_diff_count += 1 if line_diff_count > lines_diff: - raise AssertionError("Failed to find '%s' in history data. (lines_diff=%i)" % (contains, lines_diff)) + raise AssertionError(f"Failed to find '{contains}' in history data. (lines_diff={lines_diff}).") diff --git a/test/unit/test_verify.py b/test/unit/test_verify.py index e51011ca604..bd4dccc30d6 100644 --- a/test/unit/test_verify.py +++ b/test/unit/test_verify.py @@ -1,4 +1,5 @@ import collections +import gzip import tempfile import pytest @@ -22,15 +23,17 @@ TestFile = collections.namedtuple('TestFile', 'value path') def test_file_list(): files = [] - for b, ext in [(F1, '.txt'), (F2, '.txt'), (F3, '.pdf'), (F4, '.txt'), (MULTILINE_MATCH, '.txt')]: + for b, ext in [(F1, '.txt'), (F2, '.txt'), (F3, '.pdf'), (F4, '.txt'), (MULTILINE_MATCH, '.txt'), (F1, '.txt.gz')]: with tempfile.NamedTemporaryFile(mode='wb', suffix=ext, delete=False) as out: + if ext == '.txt.gz': + b = gzip.compress(b) out.write(b) files.append(TestFile(b, out.name)) return files def generate_tests(multiline=False): - f1, f2, f3, f4, multiline_match = test_file_list() + f1, f2, f3, f4, multiline_match, f5 = test_file_list() if multiline: tests = [(multiline_match, f1, {'lines_diff': 0, 'sort': True}, None)] else: @@ -39,12 +42,13 @@ def generate_tests(multiline=False): (f1, f2, {'lines_diff': 0, 'sort': True}, AssertionError), (f1, f3, None, AssertionError), (f1, f4, None, None), + (f1, f5, {'decompress': True}, None), ]) return tests def generate_tests_sim_size(): - f1, f2, f3, f4, multiline_match = test_file_list() + f1, f2, f3, f4, multiline_match, f5 = test_file_list() # tests for equal files tests = [(f1, f1, None, None), # pass default values (f1, f1, {'delta': 0}, None), # pass for values that should always pass From 735b9afb5cf8c3d350f4f7aeeff510e4e7f92e03 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Tue, 22 Jun 2021 20:14:30 +0200 Subject: [PATCH 3/6] Don't requires attributes when creating TestCollectionOutputDef So we can drop workarounds like https://github.com/galaxyproject/iwc/blob/main/workflows/sars-cov-2-variant-calling/sars-cov-2-pe-illumina-wgs-variant-calling/pe-wgs-variation-test.yml#L22 --- lib/galaxy/tool_util/parser/interface.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/tool_util/parser/interface.py b/lib/galaxy/tool_util/parser/interface.py index 96c974dd2d2..9d4740ce865 100644 --- a/lib/galaxy/tool_util/parser/interface.py +++ b/lib/galaxy/tool_util/parser/interface.py @@ -465,7 +465,7 @@ class TestCollectionOutputDef: def from_dict(as_dict): return TestCollectionOutputDef( name=as_dict["name"], - attrib=as_dict["attributes"], + attrib=as_dict.get("attributes", {}), element_tests=as_dict["element_tests"], ) From 7543a983da3fba5c57e056ca73d03b3bab5c60e4 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Wed, 23 Jun 2021 10:05:26 -0400 Subject: [PATCH 4/6] Version 21.1.2 of tool-util (tag galaxy-tool-util-21.1.2). --- packages/tool_util/HISTORY.rst | 2 +- packages/tool_util/galaxy/project_galaxy_tool_util.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/tool_util/HISTORY.rst b/packages/tool_util/HISTORY.rst index 053ba4ab90e..8f1b214e3a6 100644 --- a/packages/tool_util/HISTORY.rst +++ b/packages/tool_util/HISTORY.rst @@ -6,7 +6,7 @@ History .. to_doc --------------------- -21.1.2.dev0 +21.1.2 (2021-06-23) --------------------- diff --git a/packages/tool_util/galaxy/project_galaxy_tool_util.py b/packages/tool_util/galaxy/project_galaxy_tool_util.py index fb12eda4e1b..22f1c39abc4 100644 --- a/packages/tool_util/galaxy/project_galaxy_tool_util.py +++ b/packages/tool_util/galaxy/project_galaxy_tool_util.py @@ -1,4 +1,4 @@ -__version__ = '21.1.2.dev0' +__version__ = '21.1.2' PROJECT_NAME = "galaxy-tool-util" PROJECT_OWNER = PROJECT_USERAME = "galaxyproject" From e2ac10eb3b8890155ad23445d982da1f9b64532a Mon Sep 17 00:00:00 2001 From: John Chilton Date: Wed, 23 Jun 2021 10:05:45 -0400 Subject: [PATCH 5/6] Starting work on tool-util 21.1.3 --- packages/tool_util/HISTORY.rst | 6 ++++++ packages/tool_util/galaxy/project_galaxy_tool_util.py | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/packages/tool_util/HISTORY.rst b/packages/tool_util/HISTORY.rst index 8f1b214e3a6..8123bb92be7 100644 --- a/packages/tool_util/HISTORY.rst +++ b/packages/tool_util/HISTORY.rst @@ -5,6 +5,12 @@ History .. to_doc +--------------------- +21.1.3.dev0 +--------------------- + + + --------------------- 21.1.2 (2021-06-23) --------------------- diff --git a/packages/tool_util/galaxy/project_galaxy_tool_util.py b/packages/tool_util/galaxy/project_galaxy_tool_util.py index 22f1c39abc4..a1c3f940732 100644 --- a/packages/tool_util/galaxy/project_galaxy_tool_util.py +++ b/packages/tool_util/galaxy/project_galaxy_tool_util.py @@ -1,4 +1,4 @@ -__version__ = '21.1.2' +__version__ = '21.1.3.dev0' PROJECT_NAME = "galaxy-tool-util" PROJECT_OWNER = PROJECT_USERAME = "galaxyproject" From d8e94e716585017042bef6acc7239c6863c8dcaf Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Wed, 30 Jun 2021 18:46:17 +0200 Subject: [PATCH 6/6] Fix running metadata tool on LDDAs The library tests were producing datasets with failed_metadata states, but we never asserted the state, so we never noticed this. --- lib/galaxy/tools/parameters/basic.py | 15 ++++---- lib/galaxy_test/api/test_libraries.py | 54 +++++++++++++-------------- lib/galaxy_test/base/populators.py | 2 +- 3 files changed, 34 insertions(+), 37 deletions(-) diff --git a/lib/galaxy/tools/parameters/basic.py b/lib/galaxy/tools/parameters/basic.py index 337fcc4b54e..52a2b59674b 100644 --- a/lib/galaxy/tools/parameters/basic.py +++ b/lib/galaxy/tools/parameters/basic.py @@ -1894,13 +1894,12 @@ class DataToolParameter(BaseDataToolParameter): rval.append(trans.sa_session.query(trans.app.model.LibraryDatasetDatasetAssociation).get(decoded_id)) else: raise ValueError("Unknown input source %s passed to job submission API." % single_value['src']) - elif isinstance(single_value, trans.app.model.HistoryDatasetCollectionAssociation): - rval.append(single_value) - elif isinstance(single_value, trans.app.model.DatasetCollectionElement): - rval.append(single_value) - elif isinstance(single_value, trans.app.model.HistoryDatasetAssociation): - rval.append(single_value) - elif isinstance(single_value, trans.app.model.LibraryDatasetDatasetAssociation): + elif isinstance(single_value, ( + trans.app.model.HistoryDatasetCollectionAssociation, + trans.app.model.DatasetCollectionElement, + trans.app.model.HistoryDatasetAssociation, + trans.app.model.LibraryDatasetDatasetAssociation + )): rval.append(single_value) else: if len(str(single_value)) == 16: @@ -1913,7 +1912,7 @@ class DataToolParameter(BaseDataToolParameter): for val in rval: if not isinstance(val, trans.app.model.HistoryDatasetCollectionAssociation): raise ParameterValueError("if collections are supplied to multiple data input parameter, only collections may be used", self.name) - elif isinstance(value, trans.app.model.HistoryDatasetAssociation): + elif isinstance(value, (trans.app.model.HistoryDatasetAssociation, trans.app.model.LibraryDatasetDatasetAssociation)): rval = value elif isinstance(value, dict) and 'src' in value and 'id' in value: if value['src'] == 'hda': diff --git a/lib/galaxy_test/api/test_libraries.py b/lib/galaxy_test/api/test_libraries.py index 08f752d7d97..62a4068052a 100644 --- a/lib/galaxy_test/api/test_libraries.py +++ b/lib/galaxy_test/api/test_libraries.py @@ -1,5 +1,4 @@ import json -import time import unittest from galaxy_test.base.populators import ( @@ -284,44 +283,43 @@ class LibrariesApiTestCase(ApiTestCase, TestsDatasets): container_update_time = container_fetch_response.json()['folder_contents'][0]['update_time'] assert dataset_update_time == container_update_time, container_fetch_response - def test_update_dataset_in_folder(self): - ld = self._create_dataset_in_folder_in_library("ForUpdateDataset") - data = {'name': 'updated_name', 'file_ext': 'fastq', 'misc_info': 'updated_info', 'genome_build': 'updated_genome_build'} - create_response = self._patch("libraries/datasets/%s" % ld.json()["id"], data=data) + def _patch_library_dataset(self, library_dataset_id, data): + create_response = self._patch("libraries/datasets/{}".format(library_dataset_id), data=data) self._assert_status_code_is(create_response, 200) - self._assert_has_keys(create_response.json(), "name", "file_ext", "misc_info", "genome_build") + ld = create_response.json() + library_id = ld['parent_library_id'] + return self.library_populator.wait_on_library_dataset(library={'id': library_id}, dataset=ld) + + def test_update_dataset_in_folder(self): + ld = self._create_dataset_in_folder_in_library("ForUpdateDataset", content=">seq\nATGC", wait=True).json() + data = {'name': 'updated_name', 'file_ext': 'fasta', 'misc_info': 'updated_info', 'genome_build': 'updated_genome_build'} + ld_updated = self._patch_library_dataset(ld['id'], data) + self._assert_has_keys(ld_updated, "name", "file_ext", "misc_info", "genome_build") def test_update_dataset_tags(self): - ld = self._create_dataset_in_folder_in_library("ForTagtestDataset") + ld = self._create_dataset_in_folder_in_library("ForTagtestDataset", wait=True).json() data = {"tags": ["#Lancelot", "name:Holy Grail", "blue"]} - create_response = self._patch("libraries/datasets/%s" % ld.json()["id"], data=data) - self._assert_status_code_is(create_response, 200) - self._assert_has_keys(create_response.json(), "tags") - assert create_response.json()["tags"] == "name:Lancelot, name:HolyGrail, blue" + ld_updated = self._patch_library_dataset(ld['id'], data) + self._assert_has_keys(ld_updated, "tags") + assert ld_updated["tags"] == "name:Lancelot, name:HolyGrail, blue" def test_invalid_update_dataset_in_folder(self): - ld = self._create_dataset_in_folder_in_library("ForInvalidUpdateDataset") + ld = self._create_dataset_in_folder_in_library("ForInvalidUpdateDataset", wait=True).json() data = {'file_ext': 'nonexisting_type'} - create_response = self._patch("libraries/datasets/%s" % ld.json()["id"], data=data) + create_response = self._patch("libraries/datasets/{}".format(ld["id"]), data=data) self._assert_status_code_is(create_response, 400) assert 'This Galaxy does not recognize the datatype of:' in create_response.json()['err_msg'] def test_detect_datatype_of_dataset_in_folder(self): - ld = self._create_dataset_in_folder_in_library("ForDetectDataset") - # Wait for metadata job to finish. - time.sleep(2) + ld = self._create_dataset_in_folder_in_library("ForDetectDataset", wait=True).json() data = {'file_ext': 'data'} - create_response = self._patch("libraries/datasets/%s" % ld.json()["id"], data=data) - self._assert_status_code_is(create_response, 200) - self._assert_has_keys(create_response.json(), "file_ext") - assert create_response.json()["file_ext"] == "data" - # Wait for metadata job to finish. - time.sleep(2) + ld_updated = self._patch_library_dataset(ld['id'], data) + self._assert_has_keys(ld_updated, "file_ext") + assert ld_updated["file_ext"] == "data" data = {'file_ext': 'auto'} - create_response = self._patch("libraries/datasets/%s" % ld.json()["id"], data=data) - self._assert_status_code_is(create_response, 200) - self._assert_has_keys(create_response.json(), "file_ext") - assert create_response.json()["file_ext"] == "txt" + ld_updated = self._patch_library_dataset(ld['id'], data) + self._assert_has_keys(ld_updated, "file_ext") + assert ld_updated["file_ext"] == "txt" def test_ldda_collection_import_to_history(self): self._import_to_history(visible=True) @@ -426,13 +424,13 @@ class LibrariesApiTestCase(ApiTestCase, TestsDatasets): ) return self._post("folders/%s" % containing_folder_id, data=create_data) - def _create_dataset_in_folder_in_library(self, library_name): + def _create_dataset_in_folder_in_library(self, library_name, content="1 2 3", wait=False): library = self.library_populator.new_private_library(library_name) folder_response = self._create_folder(library) self._assert_status_code_is(folder_response, 200) folder_id = folder_response.json()[0]['id'] history_id = self.dataset_populator.new_history() - hda_id = self.dataset_populator.new_dataset(history_id, content="1 2 3")['id'] + hda_id = self.dataset_populator.new_dataset(history_id, content=content, wait=wait)['id'] payload = {'from_hda_id': hda_id, 'create_type': 'file', 'folder_id': folder_id} ld = self._post("libraries/%s/contents" % folder_id, payload) return ld diff --git a/lib/galaxy_test/base/populators.py b/lib/galaxy_test/base/populators.py index dcfc2dee805..818d6bcc14e 100644 --- a/lib/galaxy_test/base/populators.py +++ b/lib/galaxy_test/base/populators.py @@ -1570,7 +1570,7 @@ def wait_on_state(state_func, desc="state", skip_states=None, ok_states=None, as return state if skip_states is None: - skip_states = ["running", "queued", "new", "ready", "stop", "stopped"] + skip_states = ["running", "queued", "new", "ready", "stop", "stopped", "setting_metadata"] if ok_states is None: ok_states = ["ok", "scheduled"] try: