From a776836dbabcb7d43a982b9cfde1a371a09e3503 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Sat, 17 May 2025 09:32:46 -0400 Subject: [PATCH 01/19] Update rule builder to allow list:paired_or_unpaired creation. --- .../RuleBuilder/rule-definitions.js | 8 ++ .../src/components/RuleCollectionBuilder.vue | 48 +++++++++- lib/galaxy/managers/collections.py | 25 ++++- lib/galaxy/util/rules_dsl.py | 7 ++ lib/galaxy_test/api/test_tools.py | 3 + lib/galaxy_test/base/rules_test_data.py | 94 +++++++++++++++++++ lib/galaxy_test/selenium/test_tool_form.py | 6 ++ 7 files changed, 185 insertions(+), 6 deletions(-) diff --git a/client/src/components/RuleBuilder/rule-definitions.js b/client/src/components/RuleBuilder/rule-definitions.js index aec823b5a9b..d0f0d0a0586 100644 --- a/client/src/components/RuleBuilder/rule-definitions.js +++ b/client/src/components/RuleBuilder/rule-definitions.js @@ -814,6 +814,14 @@ const MAPPING_TARGETS = { ), importType: "collections", }, + paired_or_unpaired_identifier: { + label: _l("Optional Paired-end Indicator (Advanced)"), + columnHeader: _l("Optional Paired Indicator"), + help: _l( + "This should be set to '1', 'R1', 'forward', 'f', or 'F' to indicate forward reads, and '2', 'r', 'reverse', 'R2', 'R', or 'R2' to indicate reverse reads. Unmatched '1' for 'forward' elements will be 'unpaired' in the resulting list, alternatively this column can be set to 'u' or 'unpaired' to force the element to be unpaired." + ), + importType: "collections", + }, collection_name: { label: _l("Collection Name"), help: _l( diff --git a/client/src/components/RuleCollectionBuilder.vue b/client/src/components/RuleCollectionBuilder.vue index 11e2aaf166c..f64e92a814c 100644 --- a/client/src/components/RuleCollectionBuilder.vue +++ b/client/src/components/RuleCollectionBuilder.vue @@ -1058,6 +1058,13 @@ export default { collectionType = "paired"; } } + if (this.mappingAsDict.paired_or_unpaired_identifier) { + if (collectionType) { + collectionType += ":paired_or_unpaired"; + } else { + collectionType = "paired_or_unpaired"; + } + } return collectionType; }, validName() { @@ -1557,6 +1564,9 @@ export default { if (this.mappingAsDict.paired_identifier) { identifierColumns.push(this.mappingAsDict.paired_identifier.columns[0]); } + if (this.mappingAsDict.paired_or_unpaired_identifier) { + identifierColumns.push(this.mappingAsDict.paired_or_unpaired_identifier.columns[0]); + } return identifierColumns; }, buildRequestElements(createDatasetDescription, createSubcollectionDescription, subElementProp) { @@ -1611,15 +1621,24 @@ export default { let identifier = String(rowData[identifierColumns[identifierColumnIndex]]); if (identifierColumnIndex + 1 == numIdentifierColumns) { // At correct final position in nested structure for this dataset. - if (collectionTypeAtDepth === "paired") { + if (["paired", "paired_or_unpaired"].indexOf(collectionTypeAtDepth) > -1) { if (["f", "1", "r1", "forward"].indexOf(identifier.toLowerCase()) > -1) { identifier = "forward"; } else if (["r", "2", "r2", "reverse"].indexOf(identifier.toLowerCase()) > -1) { identifier = "reverse"; + } else if ( + collectionTypeAtDepth == "paired_or_unpaired" && + ["unpaired", "u"].indexOf(identifier.toLowerCase()) > -1 + ) { + // assert collectionTypeAtDepth == paired_or_unpaired + identifier = "unpaired"; } else { this.state = "error"; - this.errorMessage = - "Unknown indicator of paired status encountered - only values of F, R, 1, 2, R1, R2, forward, or reverse are allowed."; + const allowedIndicators = ["F", "R", "1", "2", "R1", "R2", "forward", "reverse"]; + if (collectionTypeAtDepth == "paired_or_unpaired") { + allowedIndicators.push("unpaired", "u"); + } + this.errorMessage = `Unknown indicator (${identifier}) of paired status encountered - only values of (${allowedIndicators}) are allowed.`; return; } } @@ -1658,6 +1677,29 @@ export default { } } + // Recursively descend elements to handle "paired_or_unpaired" collections + const updateUnpairedIdentifiers = (elements) => { + for (const value of Object.values(elements)) { + if (typeof value !== "object" || value === null) { + continue; + } + if (value.src === "new_collection" && value.collection_type === "paired_or_unpaired") { + const subElements = value.elements; + if (subElements["forward"] && !subElements["reverse"]) { + subElements["unpaired"] = subElements["forward"]; + delete subElements["forward"]; + } + } + if (value.elements) { + updateUnpairedIdentifiers(value.elements); + } + } + }; + + if (collectionType.endsWith("paired_or_unpaired")) { + updateUnpairedIdentifiers(elements); + } + elementsByName[collectionName] = elements; } diff --git a/lib/galaxy/managers/collections.py b/lib/galaxy/managers/collections.py index cec4e6fa31b..1b1fbd942ec 100644 --- a/lib/galaxy/managers/collections.py +++ b/lib/galaxy/managers/collections.py @@ -748,14 +748,19 @@ class DatasetCollectionManager: if i + 1 == len(identifier_columns): # At correct final position in nested structure for this dataset. - if collection_type_at_depth.collection_type == "paired": + if collection_type_at_depth.collection_type in ["paired", "paired_or_unpaired"]: if identifier.lower() in ["f", "1", "r1", "forward"]: identifier = "forward" elif identifier.lower() in ["r", "2", "r2", "reverse"]: identifier = "reverse" + elif identifier.lower() in ["u", "unpaired"]: + identifier = "unpaired" else: + allow_identifiers = ["F", "R", "1", "2", "R1", "R2", "forward", "reverse"] + if collection_type_at_depth == "paired_or_unpaired": + allow_identifiers.extend(["unpaired", "u"]) raise Exception( - "Unknown indicator of paired status encountered - only values of F, R, 1, 2, R1, R2, forward, or reverse are allowed." + f"Unknown indicator of paired status encountered ({identifier}) - only values from ({allow_identifiers}) are allowed." ) tags = [] @@ -773,7 +778,6 @@ class DatasetCollectionManager: effective_dataset = handle_dataset(sources[data_index]["dataset"], tags) elements_at_depth[identifier] = effective_dataset - # log.info("Handling dataset [%s] with sources [%s], need to add tags [%s]" % (effective_dataset, sources, tags)) else: collection_type_at_depth = collection_type_at_depth.child_collection_type_description() found = False @@ -792,6 +796,21 @@ class DatasetCollectionManager: # Subsequent loop fills elements of newly created collection elements_at_depth = sub_collection["elements"] + # Recursively descend elements to handle "paired_or_unpaired" collections + def update_unpaired_identifiers(elements): + for value in elements.values(): + if not isinstance(value, dict): + continue + if value.get("src") == "new_collection" and value.get("collection_type") == "paired_or_unpaired": + sub_elements = value["elements"] + if "forward" in sub_elements and "reverse" not in sub_elements: + sub_elements["unpaired"] = sub_elements.pop("forward") + if "elements" in value: + update_unpaired_identifiers(value["elements"]) + + if collection_type_description.collection_type.endswith("paired_or_unpaired"): + update_unpaired_identifiers(elements) + return elements def __init_rule_data(self, elements, collection_type_description, parent_identifiers=None, parent_indices=None): diff --git a/lib/galaxy/util/rules_dsl.py b/lib/galaxy/util/rules_dsl.py index 7657eebebc3..516ece43c94 100644 --- a/lib/galaxy/util/rules_dsl.py +++ b/lib/galaxy/util/rules_dsl.py @@ -596,6 +596,8 @@ class RuleSet: identifier_columns.extend(mapping_as_dict["list_identifiers"]["columns"]) if "paired_identifier" in mapping_as_dict: identifier_columns.append(mapping_as_dict["paired_identifier"]["columns"][0]) + if "paired_or_unpaired_identifier" in mapping_as_dict: + identifier_columns.append(mapping_as_dict["paired_or_unpaired_identifier"]["columns"][0]) return identifier_columns @@ -609,6 +611,11 @@ class RuleSet: collection_type += ":paired" else: collection_type = "paired" + if "paired_or_unpaired_identifier" in mapping_as_dict: + if collection_type: + collection_type += ":paired_or_unpaired" + else: + collection_type = "paired_or_unpaired" return collection_type @property diff --git a/lib/galaxy_test/api/test_tools.py b/lib/galaxy_test/api/test_tools.py index 4adf6d7bdc5..ca5556951a0 100644 --- a/lib/galaxy_test/api/test_tools.py +++ b/lib/galaxy_test/api/test_tools.py @@ -897,6 +897,9 @@ class TestToolsApi(ApiTestCase, TestsTools): def test_apply_rules_6(self): self._apply_rules_and_check(rules_test_data.EXAMPLE_6) + def test_apply_rules_create_paired_or_unpaired_list(self): + self._apply_rules_and_check(rules_test_data.EXAMPLE_CREATE_PAIRED_OR_UNPAIRED_COLLECTION) + def test_apply_rules_flatten_with_indices(self): self._apply_rules_and_check(rules_test_data.EXAMPLE_FLATTEN_USING_INDICES) diff --git a/lib/galaxy_test/base/rules_test_data.py b/lib/galaxy_test/base/rules_test_data.py index 3eb7cea1f09..5f2c445980a 100644 --- a/lib/galaxy_test/base/rules_test_data.py +++ b/lib/galaxy_test/base/rules_test_data.py @@ -348,6 +348,100 @@ EXAMPLE_FLATTEN_USING_INDICES = { } +def check_example_create_paired_or_unpaired_with_unmatched_forward_becoming_unpaired(hdca, dataset_populator): + assert hdca["collection_type"] == "list:paired_or_unpaired" + assert hdca["element_count"] == 4 + sample1_el = hdca["elements"][0] + assert "object" in sample1_el, hdca + assert "element_identifier" in sample1_el + assert sample1_el["element_identifier"] == "sample1", hdca + child_collection_level = sample1_el["object"] + assert child_collection_level["collection_type"] == "paired_or_unpaired" + assert child_collection_level["elements"][0]["element_identifier"] == "forward", hdca + + sample2_el = hdca["elements"][1] + assert "object" in sample2_el, hdca + assert "element_identifier" in sample2_el + assert sample2_el["element_identifier"] == "sample2", hdca + child_collection_level = sample2_el["object"] + assert child_collection_level["collection_type"] == "paired_or_unpaired" + assert child_collection_level["elements"][0]["element_identifier"] == "unpaired", hdca + + sample3_el = hdca["elements"][2] + assert "object" in sample3_el, hdca + assert "element_identifier" in sample3_el + assert sample3_el["element_identifier"] == "sample3", hdca + child_collection_level = sample3_el["object"] + assert child_collection_level["collection_type"] == "paired_or_unpaired" + assert child_collection_level["elements"][0]["element_identifier"] == "unpaired", hdca + + sample4_el = hdca["elements"][2] + assert "object" in sample4_el, hdca + assert "element_identifier" in sample4_el + assert sample4_el["element_identifier"] == "sample3", hdca + child_collection_level = sample4_el["object"] + assert child_collection_level["collection_type"] == "paired_or_unpaired" + assert child_collection_level["elements"][0]["element_identifier"] == "unpaired", hdca + + +EXAMPLE_CREATE_PAIRED_OR_UNPAIRED_COLLECTION = { + "rules": { + "rules": [ + { + "type": "add_column_metadata", + "value": "identifier0", + }, + { + "type": "add_column_metadata", + "value": "identifier1", + }, + ], + "mapping": [ + { + "type": "list_identifiers", + "columns": [0], + }, + { + "type": "paired_or_unpaired_identifier", + "columns": [1], + }, + ], + }, + "test_data": { + "type": "list:list", + "elements": [ + { + "identifier": "sample1", + "elements": [ + {"identifier": "forward", "class": "File", "contents": "TestData123forward"}, + {"identifier": "reverse", "class": "File", "contents": "TestData123reverse"}, + ], + }, + { + "identifier": "sample2", + "elements": [ + {"identifier": "unpaired", "class": "File", "contents": "TestData123unpaired"}, + ], + }, + { + "identifier": "sample3", + "elements": [ + {"identifier": "u", "class": "File", "contents": "TestData123unpaired-2"}, + ], + }, + { + "identifier": "sample4", + "elements": [ + {"identifier": "forward", "class": "File", "contents": "TestData123unpaired-3"}, + ], + }, + ], + }, + "check": check_example_create_paired_or_unpaired_with_unmatched_forward_becoming_unpaired, + "output_hid": 12, +} + + def check_example_flatten_paired_or_unpaired(hdca, dataset_populator): assert hdca["collection_type"] == "list" assert hdca["element_count"] == 3 diff --git a/lib/galaxy_test/selenium/test_tool_form.py b/lib/galaxy_test/selenium/test_tool_form.py index 79a7f0b3cae..4d49d09523f 100644 --- a/lib/galaxy_test/selenium/test_tool_form.py +++ b/lib/galaxy_test/selenium/test_tool_form.py @@ -392,6 +392,12 @@ class TestLoggedInToolForm(SeleniumTestCase): self._apply_rules_and_check(rules_test_data.EXAMPLE_FLATTEN_PAIRED_OR_UNPAIRED) self.screenshot("tool_apply_rules_example_flatten_paired_unpaired_final") + @selenium_test + @managed_history + def test_run_apply_rules_create_paired_or_unpaired_list(self): + self._apply_rules_and_check(rules_test_data.EXAMPLE_CREATE_PAIRED_OR_UNPAIRED_COLLECTION) + self.screenshot("tool_apply_rules_example_flatten_paired_unpaired_final") + @selenium_test def test_run_apply_rules_flatten_with_indices(self): self._apply_rules_and_check(rules_test_data.EXAMPLE_FLATTEN_USING_INDICES) From 05ecc12d96bcdf2367dd16e05fb1105d225afbd8 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Sat, 17 May 2025 12:58:23 -0400 Subject: [PATCH 02/19] Test case for rule builder backend errors. --- lib/galaxy_test/api/test_tools.py | 48 +++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/lib/galaxy_test/api/test_tools.py b/lib/galaxy_test/api/test_tools.py index ca5556951a0..f5f2199a553 100644 --- a/lib/galaxy_test/api/test_tools.py +++ b/lib/galaxy_test/api/test_tools.py @@ -876,6 +876,54 @@ class TestToolsApi(ApiTestCase, TestsTools): output_hdca = self.dataset_populator.get_history_collection_details(history_id, hid=output_hid, wait=False) example["check"](output_hdca, self.dataset_populator) + def test_apply_rules_with_error_in_mapping(self): + # this would produce a list:paired but the child identifiers are incorrectly spelled + example_with_mapping_error = { + "rules": { + "rules": [ + { + "type": "add_column_metadata", + "value": "identifier0", + }, + { + "type": "add_column_metadata", + "value": "identifier1", + }, + ], + "mapping": [ + { + "type": "list_identifiers", + "columns": [0], + }, + { + "type": "paired_identifier", + "columns": [1], + }, + ], + }, + "test_data": { + "type": "list:list", + "elements": [ + { + "identifier": "sample1", + "elements": [ + {"identifier": "floorward", "class": "File", "contents": "TestData123forward"}, + {"identifier": "reverb", "class": "File", "contents": "TestData123reverse"}, + ], + }, + ], + }, + } + with self.dataset_populator.test_history(require_new=False) as history_id: + inputs = stage_rules_example(self.galaxy_interactor, history_id, example_with_mapping_error) + hdca = inputs["input"] + inputs = {"input": {"src": "hdca", "id": hdca["id"]}, "rules": example_with_mapping_error["rules"]} + + self.dataset_populator.wait_for_history(history_id) + response = self._run("__APPLY_RULES__", history_id, inputs, assert_ok=False) + assert_status_code_is(response, 400) + assert "Unknown indicator of paired status encountered (floorward)" in response.json()["err_msg"] + def test_apply_rules_flatten_paired_unpaired(self): self._apply_rules_and_check(rules_test_data.EXAMPLE_FLATTEN_PAIRED_OR_UNPAIRED) From 7efe923b4caefac3183a2b62c104e602fa035831 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Fri, 13 Jun 2025 16:32:29 -0400 Subject: [PATCH 03/19] Sharper exception handling in apply rules test. --- lib/galaxy/managers/collections.py | 2 +- lib/galaxy_test/api/test_tools.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/galaxy/managers/collections.py b/lib/galaxy/managers/collections.py index 1b1fbd942ec..e917ce69b0e 100644 --- a/lib/galaxy/managers/collections.py +++ b/lib/galaxy/managers/collections.py @@ -759,7 +759,7 @@ class DatasetCollectionManager: allow_identifiers = ["F", "R", "1", "2", "R1", "R2", "forward", "reverse"] if collection_type_at_depth == "paired_or_unpaired": allow_identifiers.extend(["unpaired", "u"]) - raise Exception( + raise RequestParameterInvalidException( f"Unknown indicator of paired status encountered ({identifier}) - only values from ({allow_identifiers}) are allowed." ) diff --git a/lib/galaxy_test/api/test_tools.py b/lib/galaxy_test/api/test_tools.py index f5f2199a553..e974cf38e1c 100644 --- a/lib/galaxy_test/api/test_tools.py +++ b/lib/galaxy_test/api/test_tools.py @@ -23,6 +23,7 @@ from galaxy.util import galaxy_root_path from galaxy.util.unittest_utils import skip_if_github_down from galaxy_test.base import rules_test_data from galaxy_test.base.api_asserts import ( + assert_error_code_is, assert_has_keys, assert_status_code_is, ) @@ -922,6 +923,7 @@ class TestToolsApi(ApiTestCase, TestsTools): self.dataset_populator.wait_for_history(history_id) response = self._run("__APPLY_RULES__", history_id, inputs, assert_ok=False) assert_status_code_is(response, 400) + assert_error_code_is(response, 400008) assert "Unknown indicator of paired status encountered (floorward)" in response.json()["err_msg"] def test_apply_rules_flatten_paired_unpaired(self): From 9aee42bc22ff1089237d204789f6aee80db1b1b1 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Sat, 17 May 2025 13:17:25 -0400 Subject: [PATCH 04/19] Implement hash validation in rule builder. --- .../components/RuleBuilder/rule-definitions.js | 16 ++++++++++++++++ .../src/components/RuleCollectionBuilder.vue | 18 ++++++++++++++++++ 2 files changed, 34 insertions(+) diff --git a/client/src/components/RuleBuilder/rule-definitions.js b/client/src/components/RuleBuilder/rule-definitions.js index d0f0d0a0586..9888d1535ee 100644 --- a/client/src/components/RuleBuilder/rule-definitions.js +++ b/client/src/components/RuleBuilder/rule-definitions.js @@ -860,6 +860,22 @@ const MAPPING_TARGETS = { label: _l("Genome"), modes: ["raw", "ftp"], }, + hash_sha1: { + label: _l("Hash (SHA1)"), + modes: ["raw", "ftp"], + }, + hash_md5: { + label: _l("Hash (MD5)"), + modes: ["raw", "ftp"], + }, + hash_sha256: { + label: _l("Hash (SHA256)"), + modes: ["raw", "ftp"], + }, + hash_sha512: { + label: _l("Hash (SHA512)"), + modes: ["raw", "ftp"], + }, file_type: { label: _l("Type"), modes: ["raw", "ftp"], diff --git a/client/src/components/RuleCollectionBuilder.vue b/client/src/components/RuleCollectionBuilder.vue index f64e92a814c..dc5d9f77d46 100644 --- a/client/src/components/RuleCollectionBuilder.vue +++ b/client/src/components/RuleCollectionBuilder.vue @@ -1861,6 +1861,24 @@ export default { const info = data[dataIndex][infoColumn]; res["info"] = info; } + const hashTypes = [ + { key: "hash_md5", function: "MD5" }, + { key: "hash_sha1", function: "SHA1" }, + { key: "hash_sha256", function: "SHA256" }, + { key: "hash_sha515", function: "SHA512" }, + ]; + + hashTypes.forEach(({ key, function: hashFunction }) => { + if (mappingAsDict[key]) { + const hashColumn = mappingAsDict[key].columns[0]; + const hash = data[dataIndex][hashColumn]; + if (res.hashes === undefined) { + res["hashes"] = []; + } + res["hashes"].push({ hash_function: hashFunction, hash_value: hash }); + } + }); + const tags = []; if (mappingAsDict.tags) { const tagColumns = mappingAsDict.tags.columns; From cabb3789a20cf8aa36c3f7d497cb1626ddbc621e Mon Sep 17 00:00:00 2001 From: John Chilton Date: Sun, 18 May 2025 08:31:13 -0400 Subject: [PATCH 05/19] Bug fix - missing help text in rule builder. --- client/src/components/RuleBuilder/ColumnSelector.vue | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/src/components/RuleBuilder/ColumnSelector.vue b/client/src/components/RuleBuilder/ColumnSelector.vue index 5acc6d14739..8a89c966272 100644 --- a/client/src/components/RuleBuilder/ColumnSelector.vue +++ b/client/src/components/RuleBuilder/ColumnSelector.vue @@ -11,7 +11,7 @@
- {{ label }} + {{ label }}
  1. From 3c70d03fde5c4736b99d5388e16f78803d5fb823 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Sun, 18 May 2025 08:41:25 -0400 Subject: [PATCH 06/19] Refactor HelpText style for reuse in other context. Apply help text styling in rule builder. --- client/src/components/Help/HelpText.vue | 11 +---------- client/src/components/Help/help-text.scss | 8 ++++++++ client/src/components/RuleBuilder/ColumnSelector.vue | 6 ++++-- 3 files changed, 13 insertions(+), 12 deletions(-) create mode 100644 client/src/components/Help/help-text.scss diff --git a/client/src/components/Help/HelpText.vue b/client/src/components/Help/HelpText.vue index b35a420a7da..f24ea8c4fa7 100644 --- a/client/src/components/Help/HelpText.vue +++ b/client/src/components/Help/HelpText.vue @@ -23,13 +23,4 @@ const helpTarget = ref(); - + diff --git a/client/src/components/Collections/auto_pairing_spec.yml b/client/src/components/Collections/auto_pairing_spec.yml deleted file mode 100644 index 8b81d649abc..00000000000 --- a/client/src/components/Collections/auto_pairing_spec.yml +++ /dev/null @@ -1,76 +0,0 @@ -- doc: Simple _R1 _R2 fastqs split. - inputs: - - input_R1.fastq - - input_R2.fastq - paired: - input: - forward: input_R1.fastq - reverse: input_R2.fastq - -- doc: Compressed _R1 _R2 fastqs split. - inputs: - - input_R1.fastq.gz - - input_R2.fastq.gz - paired: - input: - forward: input_R1.fastq.gz - reverse: input_R2.fastq.gz - -- doc: Joining common prefixes on _R1 _R2. - inputs: - - input541_R1.fastq - - input541_R2.fastq - - input521_R1.fastq - - input521_R2.fastq - paired: - input541: - forward: input541_R1.fastq - reverse: input541_R2.fastq - input521: - forward: input521_R1.fastq - reverse: input521_R2.fastq - -- doc: Simple .1.fastq/.2.fastq split. - inputs: - - input.1.fastq - - input.2.fastq - paired: - input: - forward: input.1.fastq - reverse: input.2.fastq - -- doc: Simple .1.fastq.gz/.2.fastq.gz split. - inputs: - - input.1.fastq.gz - - input.2.fastq.gz - paired: - input: - forward: input.1.fastq.gz - reverse: input.2.fastq.gz - -- doc: Simple _1/_2 split. - inputs: - - input_1.fastq - - input_2.fastq - paired: - input: - forward: input_1.fastq - reverse: input_2.fastq - -- doc: Simple _1/_2 split on compressed files (.gz) - inputs: - - input_1.fastq.gz - - input_2.fastq.gz - paired: - input: - forward: input_1.fastq.gz - reverse: input_2.fastq.gz - -- doc: Simple _1/_2 split on compressed files (.bz2) - inputs: - - input_1.fastq.bz2 - - input_2.fastq.bz2 - paired: - input: - forward: input_1.fastq.bz2 - reverse: input_2.fastq.bz2 diff --git a/client/src/components/Collections/auto_pairing_spec.yml b/client/src/components/Collections/auto_pairing_spec.yml new file mode 120000 index 00000000000..0b9861a7800 --- /dev/null +++ b/client/src/components/Collections/auto_pairing_spec.yml @@ -0,0 +1 @@ +../../../../lib/galaxy/model/dataset_collections/auto_pairing_spec.yml \ No newline at end of file diff --git a/client/src/components/Collections/common/useWorkbooks.ts b/client/src/components/Collections/common/useWorkbooks.ts new file mode 100644 index 00000000000..86043e9b352 --- /dev/null +++ b/client/src/components/Collections/common/useWorkbooks.ts @@ -0,0 +1,101 @@ +import { faUpload } from "@fortawesome/free-solid-svg-icons"; +import { FontAwesomeIcon } from "@fortawesome/vue-fontawesome"; +import { computed, ref } from "vue"; + +import HiddenWorkbookUploadInput from "@/components/Collections/wizard/HiddenWorkbookUploadInput.vue"; + +export const fileToBase64 = (file: File): Promise => + new Promise((resolve, reject) => { + const reader = new FileReader(); + reader.onload = () => resolve(reader.result as string); + reader.onerror = (error) => reject(error); + reader.readAsDataURL(file); + }); + +export type WorkbookHandler = (workbookContentBase64: string) => Promise; + +export function useWorkbookDropHandling(workbookHandler: WorkbookHandler) { + const uploadErrorMessage = ref(undefined); + const isDragging = ref(false); + const isProcessingUpload = ref(false); + + async function onFileUpload(event: Event) { + const input = event.target as HTMLInputElement; + const file = input.files?.[0] ?? null; + if (file) { + const base64Content = await readAsBase64(file); + workbookHandler(base64Content); + } + } + + async function readAsBase64(file: File) { + const fileContent = await fileToBase64(file); + const base64Content = fileContent.split(",")[1] as string; + return base64Content; + } + + function checkDrop(event: DragEvent): File | undefined { + const file = event.dataTransfer?.files[0]; + if (!file || !file.name.endsWith(".xlsx")) { + uploadErrorMessage.value = "Please drop a valid XLSX file."; + return undefined; + } + return file; + } + + const handleDrop = async (event: DragEvent) => { + const file = checkDrop(event); + if (!file) { + return; + } + + isDragging.value = false; + isProcessingUpload.value = true; + try { + // Read and base64 encode the file + const base64Content = await readAsBase64(file); + await workbookHandler(base64Content); + } catch (error) { + console.error("Error uploading file:", error); + uploadErrorMessage.value = "There was an error processing the file."; + } finally { + isProcessingUpload.value = false; + } + }; + + const uploadRef = ref>(); + + interface HasBrowse { + browse: () => void; + } + + function browseFiles() { + const ref = uploadRef.value; + if (ref) { + (ref as unknown as HasBrowse).browse(); + } + } + + const dropZoneClasses = computed(() => { + const classes = ["dropzone"]; + if (isDragging.value) { + classes.push("highlight"); + } + return classes; + }); + + return { + browseFiles, + dropZoneClasses, + FontAwesomeIcon, + faUpload, + readAsBase64, + isDragging, + isProcessingUpload, + handleDrop, + HiddenWorkbookUploadInput, + onFileUpload, + uploadErrorMessage, + uploadRef, + }; +} diff --git a/client/src/components/Collections/wizard/CardDownloadWorkbook.vue b/client/src/components/Collections/wizard/CardDownloadWorkbook.vue new file mode 100644 index 00000000000..61e632024bd --- /dev/null +++ b/client/src/components/Collections/wizard/CardDownloadWorkbook.vue @@ -0,0 +1,20 @@ + + + diff --git a/client/src/components/Collections/wizard/CardEditWorkbook.vue b/client/src/components/Collections/wizard/CardEditWorkbook.vue new file mode 100644 index 00000000000..10fe17b728e --- /dev/null +++ b/client/src/components/Collections/wizard/CardEditWorkbook.vue @@ -0,0 +1,15 @@ + + + diff --git a/client/src/components/Collections/wizard/CardUploadWorkbook.vue b/client/src/components/Collections/wizard/CardUploadWorkbook.vue new file mode 100644 index 00000000000..a98f3243246 --- /dev/null +++ b/client/src/components/Collections/wizard/CardUploadWorkbook.vue @@ -0,0 +1,58 @@ + + + + diff --git a/client/src/components/Collections/wizard/ConfigureFetchWorkbook.vue b/client/src/components/Collections/wizard/ConfigureFetchWorkbook.vue new file mode 100644 index 00000000000..d265efcde36 --- /dev/null +++ b/client/src/components/Collections/wizard/ConfigureFetchWorkbook.vue @@ -0,0 +1,50 @@ + + + + + diff --git a/client/src/components/Collections/wizard/HiddenWorkbookUploadInput.vue b/client/src/components/Collections/wizard/HiddenWorkbookUploadInput.vue new file mode 100644 index 00000000000..0e54ba459dc --- /dev/null +++ b/client/src/components/Collections/wizard/HiddenWorkbookUploadInput.vue @@ -0,0 +1,24 @@ + + + diff --git a/client/src/components/Collections/wizard/SourceFromWorkbook.vue b/client/src/components/Collections/wizard/SourceFromWorkbook.vue new file mode 100644 index 00000000000..6c2daa1b90e --- /dev/null +++ b/client/src/components/Collections/wizard/SourceFromWorkbook.vue @@ -0,0 +1,41 @@ + + + diff --git a/client/src/components/Collections/wizard/UploadFetchWorkbook.vue b/client/src/components/Collections/wizard/UploadFetchWorkbook.vue new file mode 100644 index 00000000000..428d17fbc46 --- /dev/null +++ b/client/src/components/Collections/wizard/UploadFetchWorkbook.vue @@ -0,0 +1,51 @@ + + + diff --git a/client/src/components/Collections/wizard/WhichWorkbookCollectionType.vue b/client/src/components/Collections/wizard/WhichWorkbookCollectionType.vue new file mode 100644 index 00000000000..e05697967f6 --- /dev/null +++ b/client/src/components/Collections/wizard/WhichWorkbookCollectionType.vue @@ -0,0 +1,92 @@ + + + diff --git a/client/src/components/Collections/wizard/fetchWorkbooks.test.ts b/client/src/components/Collections/wizard/fetchWorkbooks.test.ts new file mode 100644 index 00000000000..1cc0946d842 --- /dev/null +++ b/client/src/components/Collections/wizard/fetchWorkbooks.test.ts @@ -0,0 +1,33 @@ +import { forBuilder } from "./fetchWorkbooks"; +import type { ParsedFetchWorkbook } from "./types"; + +describe("forBuilder", () => { + it("should return the correct ForBuilderResponse for a valid ParsedFetchWorkbook", () => { + const parsedWorkbook: ParsedFetchWorkbook = { + rows: [ + { list_identifiers: "Row1", url: "http://example.com/1", dbkey: "db1" }, + { list_identifiers: "Row2", url: "http://example.com/2", dbkey: "db2" }, + ], + columns: [ + { type: "list_identifiers", title: "Name", type_index: 0 }, + { type: "url", title: "URI", type_index: 0 }, + { type: "dbkey", title: "Genome", type_index: 0 }, + ], + workbook_type: "datasets", + parse_log: [], + }; + + const result = forBuilder(parsedWorkbook); + + expect(result.initialElements).toEqual([ + ["Row1", "http://example.com/1", "db1"], + ["Row2", "http://example.com/2", "db2"], + ]); + expect(result.rulesCreatingWhat).toBe("datasets"); + expect(result.initialMapping).toEqual([ + { type: "list_identifiers", columns: [0] }, + { type: "url", columns: [1] }, + { type: "dbkey", columns: [2] }, + ]); + }); +}); diff --git a/client/src/components/Collections/wizard/fetchWorkbooks.ts b/client/src/components/Collections/wizard/fetchWorkbooks.ts new file mode 100644 index 00000000000..64ac80a2350 --- /dev/null +++ b/client/src/components/Collections/wizard/fetchWorkbooks.ts @@ -0,0 +1,79 @@ +// utilities for populating the rule builder from parsed "fetch workbook"s. +import type { + ColumnMappingType, + ParsedFetchWorkbook, + ParsedFetchWorkbookColumn, + RawRowData, + RuleBuilderMapping, + RulesCreatingWhat, +} from "./types"; + +export function hasData(parsedWorkbook: ParsedFetchWorkbook): boolean { + return parsedWorkbook.rows.length > 0; +} + +export interface ForBuilderResponse { + initialElements: string[][]; + rulesCreatingWhat: RulesCreatingWhat; + initialMapping: RuleBuilderMapping; +} + +export function forBuilder(parsedWorkbook: ParsedFetchWorkbook): ForBuilderResponse { + const initialElements: RawRowData = []; + const rulesCreatingWhat = creatingWhat(parsedWorkbook); + for (const row of parsedWorkbook.rows) { + const rowAsString: string[] = []; + for (const column of parsedWorkbook.columns) { + const rowKey: string = columnToRowKey(column); + const cellValue = row[rowKey]; + if (cellValue === undefined) { + throw Error("Error processing server response."); + } + rowAsString.push(cellValue || ""); + } + initialElements.push(rowAsString); + } + const initialMapping = buildInitialMapping(parsedWorkbook); + return { + initialElements, + initialMapping, + rulesCreatingWhat, + }; +} + +function columnToRowKey(column: ParsedFetchWorkbookColumn): string { + if (column.type_index == 0) { + return column.type; + } else { + return `${column.type}_${column.type_index}`; + } +} + +function creatingWhat(parsedWorkbook: ParsedFetchWorkbook): RulesCreatingWhat { + if (parsedWorkbook.workbook_type == "datasets") { + return "datasets"; + } else { + return "collections"; + } +} + +function buildInitialMapping(parsedWorkbook: ParsedFetchWorkbook): RuleBuilderMapping { + const columnMappings: RuleBuilderMapping = []; + for (let index = 0; index < parsedWorkbook.columns.length; index++) { + const column = parsedWorkbook.columns[index] as ParsedFetchWorkbookColumn; + const type: ColumnMappingType = column.type; + if (column.type_index > 0) { + for (const columnMapping of columnMappings) { + if (columnMapping.type == type) { + columnMapping.columns.push(index); + } + } + } else { + columnMappings.push({ + type: type, + columns: [index], + }); + } + } + return columnMappings; +} diff --git a/client/src/components/Collections/wizard/types.ts b/client/src/components/Collections/wizard/types.ts index e1ff85de05b..7ce84043291 100644 --- a/client/src/components/Collections/wizard/types.ts +++ b/client/src/components/Collections/wizard/types.ts @@ -1,18 +1,32 @@ import type { components } from "@/api/schema"; +import type { MAPPING_TARGETS } from "@/components/RuleBuilder/rule-definitions"; export type RulesCreatingWhat = "datasets" | "collections"; -export type RulesSourceFrom = "remote_files" | "pasted_table" | "dataset_as_table" | "collection"; +export type RulesSourceFrom = "remote_files" | "pasted_table" | "dataset_as_table" | "collection" | "workbook"; export type ListUriResponse = components["schemas"]["ListUriResponse"]; export type RemoteFile = components["schemas"]["RemoteFile"]; export type RemoteDirectory = components["schemas"]["RemoteDirectory"]; +export type ParsedFetchWorkbookForCollections = components["schemas"]["ParsedFetchWorkbookForCollections"]; +export type ParsedFetchWorkbookForDatasets = components["schemas"]["ParsedFetchWorkbookForDatasets"]; +export type ParsedFetchWorkbook = ParsedFetchWorkbookForCollections | ParsedFetchWorkbookForDatasets; +export type ParsedFetchWorkbookColumn = components["schemas"]["ParsedColumn"]; +export type ParsedFetchWorkbookForCollectionCollectionType = + components["schemas"]["ParsedFetchWorkbookForCollections"]["collection_type"]; export type RawRowData = string[][]; -export type InitialElements = RawRowData | HDCADetailed; // types and helpers around initializing the rule builder with data export type RuleSelectionType = "raw" | "remote_files"; export type RuleElementsType = RemoteFile[] | string[][]; +export type ColumnMappingType = keyof typeof MAPPING_TARGETS; +export type ParsedFetchWorkbookColumnType = ParsedFetchWorkbookColumn["type"]; + +export interface RuleBuilderSingleMapping { + type: ColumnMappingType; + columns: number[]; +} +export type RuleBuilderMapping = RuleBuilderSingleMapping[]; // it would be nice to have a real type from the rule builder but // it is older code. This is really outlining what this component can @@ -24,4 +38,5 @@ export interface RuleBuilderOptions { elements?: RuleElementsType | undefined; content?: string; selectionType: RuleSelectionType; + initialMappings?: RuleBuilderMapping; } diff --git a/client/src/components/Collections/wizard/workbook-dropzones.scss b/client/src/components/Collections/wizard/workbook-dropzones.scss new file mode 100644 index 00000000000..3f7c822753b --- /dev/null +++ b/client/src/components/Collections/wizard/workbook-dropzones.scss @@ -0,0 +1,9 @@ +@import "theme/blue.scss"; + +.dropzone.highlight { + border-width: 2px; + border-color: $border-color; + border-style: dashed; + border-radius: $border-radius-large; + -moz-border-radius: $border-radius-large; +} diff --git a/client/src/components/RuleBuilder/rule-definitions.js b/client/src/components/RuleBuilder/rule-definitions.js index 1c1c87c071d..abf36951413 100644 --- a/client/src/components/RuleBuilder/rule-definitions.js +++ b/client/src/components/RuleBuilder/rule-definitions.js @@ -2,7 +2,7 @@ import pyre from "pyre-to-regexp"; import _ from "underscore"; import _l from "utils/localization"; -import MAPPING_TARGETS from "./column-targets.yml"; +import MAPPING_TARGETS from "./rule_targets.yml"; const NEW_COLUMN = "new"; diff --git a/client/src/components/RuleBuilder/rule_targets.yml b/client/src/components/RuleBuilder/rule_targets.yml new file mode 120000 index 00000000000..323050dbe11 --- /dev/null +++ b/client/src/components/RuleBuilder/rule_targets.yml @@ -0,0 +1 @@ +../../../../lib/galaxy/model/dataset_collections/rule_targets.yml \ No newline at end of file diff --git a/client/src/components/RuleCollectionBuilder.vue b/client/src/components/RuleCollectionBuilder.vue index 373043b30a0..78b880116e8 100644 --- a/client/src/components/RuleCollectionBuilder.vue +++ b/client/src/components/RuleCollectionBuilder.vue @@ -690,6 +690,11 @@ export default { required: false, type: Object, }, + initialMapping: { + // only respected if elementsType is raw currently - other element types have their own default behaviors that make sense (e.g. assigning ftp paths to a URI implicitly) + required: false, + type: Array, + }, defaultHideSourceItems: { type: Boolean, required: false, @@ -736,6 +741,8 @@ export default { mapping = [{ type: "url", columns: [0] }]; } else if (this.elementsType == "datasets") { mapping = [{ type: "list_identifiers", columns: [1] }]; + } else if (this.initialMapping) { + mapping = this.initialMapping; } else { mapping = []; } @@ -1294,6 +1301,11 @@ export default { this.$refs.hotTable.$el.click(); }, 200); } + // is this comparable to watch immediate in newer Vue code?, I just need that event to + // to flair if it is initially okay also. + if (this.validInput) { + this.$emit("validInput", true); + } }, methods: { restoreRules(event) { diff --git a/lib/galaxy/app_unittest_utils/fetch_workbook.xlsx b/lib/galaxy/app_unittest_utils/fetch_workbook.xlsx new file mode 100644 index 0000000000000000000000000000000000000000..8d1e3e394f03de6df30de887a4f8d468d20b631e GIT binary patch literal 7341 zcmZ{J1yodR*Y?mLozf}Noq}}8P}0&!4CRnRgOr4X64D_lAl=^uARuK$@^bEr7X=`scEs)5TV`)`?ht;s_4sPx=m&9< zOZs0tO%^+1&!jL8$psm~F6V=D`GY7I=x~6|`bT+n0NE zy+L2lgm#}tiijvTRaejRIi7zOHh&kBpM=LO0jFV6rd67KCUGAP^2t;#twfB?Z)>*B z%X^J#3!C4WvLJAp$zc*<;tQxU(V?c}$1HA_4gjjfX?j#NM&L;_Ei1Mr&QYciv_D;- z1kwVJ$>ZuGx-euAXh!xtk^xWnC#F)I$r2FJ1EVV$RAx`_)+q8coC$RldT6-(L1Fa75aM3A{NJ1Gg8GoJ0_ zEwD`P?Tk6gpF1@fPHGp4gBv~?6}1BN1%p%gz1eV-^?F6~F7t|r$Eyl!XSWmbuae$Z z>3k``Iivu3Wpp~vvvt_6GN^>-lS)fROF@3TT`xKIc<$o^aUDNL13 zRQ)@Izxi`3i?-Sn5dcsn0RV8J{&WGeIhcd2K)+XxUmo4n>bEB0!)ko`QxMDjHS=_& z5m5x3T+9luozawRFzI(a%UEpZ8cGJiTKw)3QJ-{+VL6`#H~JYw8E^fOY`esC1)qBH zz%^NIZ8b0*;`LK9Fw4d2?VxQyt=pbh&kz-MGE8w;M7pyc4-I)P612}~k-+$dvfL!Y-18 zrGS}_D#+Jh{1lBh$}CEx=(D@G93B9F#?$>4B3s|P#G$If<|j{TKJsJQuy{Bp7u%j% zP(D?nPaEbi=u$c5r4o#hCZo&n6UGhPe6YS=3rYL)te9Pvy`A{iB}G$MpES^4V`G}i z=QU`b^qhw_!mw9-+;OT)wsw>r1lSV#W@*H5_G{ZT%zPbT_Sv13nn{*zP=OrLUcW=- z5nJUOG2tk~8<65h^7aM-}qFWuixqSR{6 zt1bF2@Z1gIjL4*G`t#F9jKecon3N1kV$~^MJ8ubP}ZE zc+R#0Sa|OcT|O+0kksMz_|x{Ib}J3(R`_B<(10jw~$3S~)`3Q$ai zbrY*Q>X=epF1;r^FUYhoaX2)_+ODJ{1)B8+v=UK^V64scX;CjawRX)|J3w>79f86N zWJxay_dY{%5=K)Uz>C0-G#lh?@vR))AH}c9D6m(C_sPF~)^oO?uU?bz-&Y>%7<(5f zOvdlSy;v@xdf59To2z=Jq<>#;Y@;vxjQ52@L^xKQtKR|cM%vxTk2NMLMD6{f#;za){>d0do?Vgi;<%?Z*$)0R~?iBx2gTpX(;*=<4!p36T<{h z6d^@@+;Tlrm}|)5co{;49*27SrXM{KMV7$d>kd>_s&7ukZ38<#%bWm)Z939(W)$xf zEY53<`smeGo!ed91gu(y*W`kdacNzM_6;;$s*d<)X|Q%ekOu2&ZwmJjAFgWam}3XE ztu_$|64qP|z_n`*7GGXwhLYu=(C~fR`a(-epo~eN0B4g`Jv|gfcUVEckHJbXwJYYE z7nUQ7fs-T=rpH5Lz>GRlBKabgeKsD%zA>*xJqs5gjKNiLJ`RA2#X2C8&g--}Z(P?PK5$kpIUk5}2u;Z-H_sLml~9>_=0L-t)j$e;7Z z*6YP6Lot0e;#37W8DmK}^RJb8&QI+hR=yA-oJ&lnU<@LbCa2hXaezk=UCqNYUA5#y zKo>nd9#fp?c3@S{O=i8RKjP|6O{S%xs2G!$p)r(SGy#XVQjd>jnD1X?fuSE;d(_4p zP?x5JLUNw{ybNWshA;8HIKNfFzv9NXRgtXAUu{ z880}7i3uvgdtQd-&xP;0ZDqvOot!M%=adOWm1VCqvF*YoUzs;|7*Twh9vO4Ku8IT%@W-CS;C^U|ttER6Kkjjn+FFy}6|7jkWxJo4`?Q#xdn_`}qwk*7bDnxr4#8I7;(mOj%G>ylIh>c;DoAgzTL zHx~M!zpq51_fg^-&!mS#bmqLD+ob-0SYmB||2;WOVdJC{saCLXy)r{A(U2q@ov7gRWyU!4;`TmUn%xtT1AQWgdJ^!h*x6Ddbg*wH-p(?? zE5GmNC6{?BqeG4~#6K-XBA*atr5U_`-a1QP9l7PsC29#1JUL>bs47t-h+s0OaG#gv8*iZZ?0poqxdchZhP z=VvSUZXrBK4?_cH>d(zGXqp>bP+EcGV|gah1PK8Em ztbdwXlQIi~<|eSijd{7H(T9oQk{DTTBo#By-v2EPH_aiI9Xttyp~xrXiDMTdr6*Tv zI#^%A}IJaxg^wBfr;x_}`3CXln+Qv)cs zoL{9EM-#7*FgV);stOIE*{XjCVZY*m6i3*>yv|txMIN_Ds3*OB!{#?nYIx*aV;^Ou z-~}3-MB99cF>S$r={X7-m(^XK!nYe5Kef_tWTMy6uz2wJQhess@TEDCPKgrs?Wd&3 zY9W$$lN^UI3PdnKZ+B0dExUiM993ovj(P9 z-bMzfFcKlNYm5f6D5?nEP$XHMi?OU)|`N{mneg%AnXw7&v$f{x~)I3RP<qxS%l)E~-BfdJWSNt5N`^jzG$;G9iE`^#zLKr3 zi|lxtbzqFvsuX95$Kpx;SV~wax7}ujhi5DA(ENiZpeaUj0&RD*9QGo4a~hsrb6yvf z1;gyMh#%s^poIhw_)QG18b{pViG(eleKxVI<1N%`|6LZ~Q;!r9Lh;HBEuqkV;}zoI z3I;)bC6JlwdNzsdc<)onAG~iB(YZ~4@M;RG7FJpEzpdkt8z*$Yk9w9L{;akksIE;nKPg);*gB-V*;v{XQS74 zEi?j&%QjE00dY;33S& zUvge)2CW!~!VeqF&ag%01R3gd#7_${uLT*M8PY{R*TGP$vz2~Qgt{qkEFgqvH$i6y z&s1)y1{wf7&C4}HcrWW56dQ;4>Y1n~kqY4|rNHpQ)>G>K;;gW~7)%iGTc$<5c*RKO zPY-qz;7PyxGtwRs>LgnK6CZb#r28oU4;=#MF zFu%eB$6Cl>(t<@PW`@g(x1OlXrx9BDY0X3|>WeJq<#Ntc<+Aw?cYL`4iv^sB0vvL( z(oV8C)({~#G}LfarGbOpXP(~`)f}VgbRtks(&VxG$_qxS;i=k6J9qfJALz1?Z@muI zB5F5o5;efz2j)}j%m*wXu5$&+_pA4jWyyGB=L|y-j4Ev+Z9-@4n%vdzwfZ}fSyl%R zv0IMgWo~UEJ7f*+9WP8A3Dcgg6?WRg{fzRh1d6@xah35iqoa~KNaN!%J7$`bQp8kR zs0<}}UQp{#a#F&Bm_njMAuJ#|LWj|mN#YVUUy?TW#+Xfg(h|wkTX`h7d~S|HIO}Ki zy`C5Yf}~nuY4IJ8-0;QDS>5`EK-jHRvHoQlg!rb{ zrQv9Gdfct7J5~%&_byI%uc&%zUJjlGe?S90aK`}C2r0<QKZ7ULGv*DBoZHz}fr@WH+rxFcg|11r9 z9Pk4*Y*=)2rt3Xh9zBvQ!5U4srOU}VpowTWEUfSi{N4?3eKlo7E z^12SYeM2a67NE_A|K^vkHZFPSP=5IgWgpDn{PHUwFtV|E_lsfj)n=_%d9mKJ4S7*s z7JeMl6W{d*0SkOguubMO-)pnukl{v1`lK5pgxp)9>FgsdYahM4W?G@mea>Xp36e_oDXNQy$x!wF;#@A0OmJD z3zqY=mGiMW3&r>1PW1@=(zyW^SmPP<&cS85i}r0pGLr%%V%T04ri+;!UBcggl^9H}E$jOrB~XB?|N> zo-nrzJ}vRqu;YPw#LA4*y`LJPpBKIU@DMc>)ri`1$bq;?sl=?hr9DT0VCe}Z{oPl#TmH|< z>?abP-4@t>p1U}Fha>d;zU*hQJ_g~i7~(Lr-%JD>gmCzZxprZCF>~MX$`i{*Cgc*S^#`9K8g6{h7RUb3ZOclfJB99z8e6ZnD1K z?2kefm6^D<|fNY_!>+krmOeAq_2I0qV}8}y)~E4X+&>Akqp zMa}+zr%@dJL%Ca-BSR4RM8dX^2-&{(mg>r@5kg{7`K&g0oTKK1sw<&2Yabf7&u(!{ z%T-}>RKl3Waoh>I^%jrQr%I>?9rM4%zNn$t(E55U@-T>m&LeqevKdU}k?-L< z;MU8>u#*;rfdMmEWuht~8-$A2;7btm%Gl_e#gdk%MzyC*8EF4$k;$i<#Mi>vUF5GJ zSSTo~Mf>jJ0~2 z6LM^DHUX{&{hdx1JQyo*xMwX)Il(yaZn)qBK?IxrlYTg4qrC!h-&Y8@IJ?_Bd3&fB z2@u>G;m9uLuXy|$m-Mtmff-rUlBYEz0o;}Y?2eq4kRh@jD9iMguX$hlsqPl|mRD6# zCNP+;Kh#Lfo+GCCx>5|E5TO8HqPqfye~wWxUA9P_BOlk_nku*sylct0Mq4M%lOxaN z5%LIdpbYGKMd8H|30Y5%uTwu*b`ZbmVQ88?X6`*bFkS2J9AAVQ;=fHLF9wTk0HvRB z_}{%S9SIw22avUcftsr=2%`57(L^aIwy|URU&zc+9TU)6Yf<_oIJ6AnFe+obT>%=8 zm#_?$msgpsR3fL1_|AM{>HK=pSYPszxJV$8g)0515U803hW+}es{TR(a^#jpZO>Q3FRN${yBN3%BB zBlF2fazj`CM4qaQDmT3vEMg-=t%)~%xAFEs{RzG=PX+b!Xt`{!>NTA}U#3wautGCa z2bO)2MAJ!KOXa?QMv$}wuX4m!^-e;2(0X9liSxl`yJ3V-RHKDF`pjg=XIy%ANhM+( zz&1EyBVDy)ulnK&a`f!lo7C9v0e9DaJMW}f9ILzejS3Q34FcT*=c&R(>ch_l1wp9~ zoCGAj^XUeL;=cP8oJi#4gNIWO%37Au@dIUDH4 z>Hg}E)vbz!8R#2cs84>i=b^nxh`FIX$VAn_-oo1KpWIS6Ozf9Kq@WIIIc|*iX7meR zWepFXDi%EjsJhGOwy!S5N1dN%!t!c0Uhrf*%3LRm>^zJGFoXYy9tZ! z&)`Y#!tQJxn??26E^4@#eB%wp$OSt^x`5t=C zlJC-yS=<3d)-21Hr)E9v?Wt>(7T;1+GwfOzubf;K=H!$KKZsGlG1RC;)ZJg_y3>Z~ z#`gSF!B$6+>W+9J1i%v-@%t7OB%X}M@w$u06cyzRg{4Jdc0ugks_VWL&ppFrA+Z22 zkY{}3VUHh8anyM1MAGLW$Gb#!y3RJ2-c;_kJ;n{v5!N9mUMoHDd_u6sxqHJjgODkG;e+~}}?2~^E%HQWWP^sK{^3K)i-nhv60PyYi%+Q^dt literal 0 HcmV?d00001 diff --git a/lib/galaxy/app_unittest_utils/fetch_workbook_list_pairs_without_ids_example.xlsx b/lib/galaxy/app_unittest_utils/fetch_workbook_list_pairs_without_ids_example.xlsx new file mode 100644 index 0000000000000000000000000000000000000000..9cbe4a2a0ca86fd71439f689f965982846413122 GIT binary patch literal 5476 zcmai21z40@yQRB(2mvVpk)Z|Y66sXBhh{)RQo06|92%4k=>}w|pD zaX{j9=Ofsy{zTTC)R$?angrdOymr@!iD1X6Ch^Ywpb699UU|b<82D!`K%i`>rSSHa zk8xo&@$(nm`ihUmL>77R*2ik`R~QQ7MT)TpA3b|KSh69vCyABj^RDqV>+(h?Y8vX7 z{R@?eka_`}W2sEqANLtUD9nGLA6qS!7Gsuch^e0;_xK0#MWTcYCUG>v*{2kw&qcBC zUr-;;^=Y=f|76lPN7jo>r`CP9NyfV`yo%MhEdeC&u^rf8gGv`T(ARkX0cY&Xp?q`!yM$Ji@siNV!d1%qOJ}Lkz`XFW-B6~O zmZ+g>j`Z|(CG$c1pR#b!kY}7@t$@<2cF`{z;I^Ps*XK0BP0kd#+Z7Ba5+u%aNc`ip zX%@S_2Eb(IGg|pUg?8EaM9H=>OF2~)ozLr;`kNuAiwlB9x0CBWQi8dCp(Z~I$TKWg z`$V4Z70OZ;WEgDmUNEl=8@G_>-u{#h!Cp%}g$STs${L|nmZ5Ad7TeY5{V<^T_{x6% z?m_;A)V)#K4>?(|wLJ6XhQc{qN8IEyz`+TzF>H7Aob;|$2jxrm{G5C;L7<+7OO=hm zK;TxthpJf+j1=%>2-55_=xDn$nL+@3=AaVJMn|9qt$Ow1P!_@lR?Bk>ssQ?O!Q4L3 z7oPCKCbWEmiEY>mn0zyMli6Lx@=3pHPI7cb5G%-M@u6fAD+8tbvE1d0_`0y=nE`Qw z5pMGQoe?Zdp8-{a5o}Z)tumU6)KMJXqQppeavdqHo;eBx=NN8GG^0NRoQi0r~D4DMgY@e+&DO=?)pFitd)754!1o8g;Edt{-Fzp~6zcO{h5 ziu?Iae~8OypiD|NbunchMw_HBub$&R1XLZXoVnEhF`~*Jb<#_2+zL)Pe*1lEzLXXa z8sBMFaq_NpnW^<{;Tn1P%=)@l=Ih3Zs{OYoY!!-)-f_EzLf^+%>S9$ni`19x=Ca8J z%b0Of#{nlBZ6Jown(za#@7$qxyX(ezzLwc{n17f7)1Z*`LVtwMrjJnNF&DuCLbfom z1~{>sln^pzjQ($HL;7oN+-yLOAnxnWYX$rGWClFTKy)5-fx6t@_zoWhrI%F74~kXM zZcP=h7t+!6-HXrm>hnFX3_grA^njdq>=gY_u^}bh=3XFFT+EK z*cZPt?2LwyX@fgmyQe4cJRACfR6Erzhh)oT%}4%Sn2Eep%m-JaG$&0D3x6Imezn|=ZHjtdK5dpA^w%a2u%XOwi$VDVXM z#`HtLMZ$h7%KGw~t^NRC4uufZ(B`k~1uSC5#G+urcPVs-5jD`(F%B*sUer%!)^kPe zaHitiXiY`JJxF|ov=WK1N_Q*o2IQGtydkrlF84$yj26zb*mRcpekFEMn#%%CZmyx# zi$d0<9*(W*0?(tZaq{xyfPb51$GmIMyejl8QCd&WdjBX^@W7h_S1X|6kqLK~ZYgIM z9WjrG30qhFV7|F!!~GtDeP~8i0M6&p0*y4#xYpeBX-(GvJ(QNR;+e#c*Z( z8tGkZ3Yh@g(pzkSrKIN)LQ0*Xvu1JH!1Llr2_=@LfP&O%xE0`zecjaH?2pFfq|dzr zv>TMEvc%XcA^DkaoXGpvCxn6q9{FkK5I#D=+Fp;l?BLM2n7v0zRUj=mg{K8}*cOax zg})JxWa1V*56G4af=8PhefIGn!$;u-|X6d7jpi z{hsd0YRtU*KFpsMzzv;_pfp;%}9<9fv2GSy&@X6OgXnjZDvW4@o^jK}C zzxovOs?b2Wfo69QUZITO-wKWFSE0Gun7M!~wcK2;{MVK3D$^WL?FJH^p7J9kSekp>RJd&q`{`KLLyJSJF?hX^@}kaItO0L)QF`7@PD>l>R#86 zZbj9wz+9|bend9s!j+euR~@_y$kgDaYNA_QvL4iedTs1m1LpNUnYK7EH$s1ku0C9+ zF{b1sxol9rxEI69)9BzkG>narZ<%A>{a`_ld|y6mVGpDsJ2qAPVcBGk)yjme^qk5- z7@#E#8Juf;#SN|S7Pm5wZY`45?kZ$zq#LGvG1)?FY@P-%d70II%BP{!CoxKDF%4JX<7KK{k6U50e>^}DtlN)84!QU$X9cxhb6SUoPyjJ1UAwYf+|3X~nZ!mef!I-0^2 zMVcLEWC2msqNJ)ZttYl>RA4MK+ER;l>E2xd}R%%J7t2uGft+^-LFV=wxMGzL3o(f_Wq+D1K1 zkmSZ~YHU*zGBfO|_H^fVYaB~ZB7r+41FSp<<-(%-c;tLDdjNGeTW{Izxw)upR}Gs; zx^q+|kEsPZ>GFuo+1-ZbD5N5Gjfm5FWu95T*WvRa-IFPHT&`1uzOY)4SUDEG0PJ$Z z@VE({Jlqx3`JuI4!Bq%Kb;wadpAe3Xm4v8H!{pA87;T3#=Tlsee;8X zVei2-@bpeTyIMR=Yxr#L*@(u)N!gi+o9SucL1t>{w{RACMmM$g-mS-HPazexOBIX9 zq2IU#{2Fo_Pv{@=K<$0~#l~lpZl{BR>yLcsY~ge4yY$gy{>SuAfg|Q0L%(o**b<=W zFcfSSC$;kM*Qs8(%GTv$Xqqb`Q+KfbEn5kH+az-*C;RK3EM5uGlM#if4L^{b54Moy zj8K@=PDH$?Z*@$PnlC-HOy{3%ONf=tamXn@%Wx8(x&$v!XYwxwy9ly5q{gF_I%`g8 zskXJxQ_#eR^p&kl>-F`MqfAk%HPf#RJzN{a{yY<#z|kX;%O8P*N2v597?v>V)y;J8 z91=E-nw*UiAwQ;*)~4Q_$RZ^Dbq#2!$AMg-YKS-K8SFVMn)AF!n@&E>mfl$&dnbtV zi{^d4*kMU^8}tL|^aqSgU#S4wL&@HOebijhiY1;~t*puk2~iDG9lFZYMk-^0jcY%~1eP z%1fM$qCn;(8biNl)_65DFEJq_tdJox)ibGbY|ZtEICT!(JvunbHg37xLRhp))OA;< z@;;~RAwn@&|Fx^5{}qb!l*L+<0w;bQG`gE(in0HJG&gVsX3?i-VJ2;=;SPlBti2gbyzb%-w99t$wrh~ z+whj~7Ws=5GlK<3Ji9L&3SU9wcU;)6`Q1WSUtC5^3h zx%y*mH&5}|B5K-n+Kp@Hzl9vVB;oc{`ES12mD)E^)if- zg|;PKTU(khag!5H_x%b=lfBFd56l=iv-(_2ZOFfMQX@|swTL0%YUp#ZRo^j%A-UJF zbKmbz0ZU6Xc956{2Z+mitcVufvIg+F*o+l0NeraOJ2fWQK`~D64IR(6)2KgBDIG!a z&BZm{+W1bN{uZq6dE5QC=x$?>eGEQJO5ZKK`+shf`YIIekby|Pz;Ea3Z0$^c=khG(YFIWmtLfuUj zi%;BMa}x_SkMx<}Fa(Ypx77DO;Dgr{?q}A(Fi2AuIeFN7b0jN*1pV|NF|hr0RWfL& zePyQw%aMY)cyrplBjVykOSMEBm*`{7c)MZbM8+KJJc53yuOR*RSa#@2P@yloW!jzA z2VPJE0-khnbA0>ssBpjb{BpE+HZOxdv@s_@mTh}Yoob8*Bvu?-#CiBcsMtxTxA6p6 zM-PjUa>qGZ_C7M_?wXKfm%o&z8=rov_=tzm#*V0_W>qjxM`%Z2fnk->S((G*v0pHA zGmxDy@znyRK&*QE+PjwpjNB3g8Drt9m7qb!kEpV}{VebypQ!Fok3QiBO+ z7Ml{`B!U;0TOXf?Rf=p3Y+DbtU9;{OQL8S1U>tyU#X2$y8Pe}ltDA=~SEp8gkAFJD z`V-}5fBJWntIi04a-%2xeSGsL%FRyds+av8&k+Co?2ENyy6G z!pq*p*5ls?B7)w|PT3)_+&F1pBkE*HGg7QrGU7u#<1QB9nQfLnqC{dR2`Rx?O(M(o0kf zql(xz{Y;B$C;}dL!qsnGy|O4uy7sn~>;r2d;z|k8klKS=LuKnKU*!oi{JI*SaV@Q* z0U3bh-BYc}@Ola21BL8MN79gR8p|X61DnP25`xMbS9OnZdxAp6qVXamQ=p9-ywlee zPh^RtPZ{>!_35<^jGOhnqw2+F(&^-HQu6JKsOBUw&5& z?U$_M;d81sov(O>WNR?f)gT}wu(S`u$Hl=R0N~*0{(n?@xw<=g*jZb9dI+6;{h~5y z)+L|yQqME<1+*JX)S+mWp>Epct1^HaDr%bJDt^q7 zEL4MxHF910q${aBm97qrd(2eIJ{HSX?8!A`j#^kyj4Q?gLkx`QYD#&ybSN9L2*nhr zEc&LtzbuSx9r^yE<6d;~SdAzo#h+{X0v!SuOuM!P1@#(H-pki^Mk4!q3cZUSXCMbx zL`=xxoTKU0yCSMO4wE`ELxM|?m(wYJb?FCRPN63trqg(;O$(|3`FA;crcUt>UvTd@ z(*?2ZQuB&N_#3Ej8Fd!3RzXJu8Zv_dIW+e#zo7LJM7_D|+=It2x)xBmn~5+LIzoHL zE)>YdhNXTTs2!7Q1~fP>-D2zyO0bV2yuewaJNU}0|8!(@tez&KA*!mae?A8IezG&N z7h!*pZBNTiC{3*&b8&M2?JEe;^Dg|%a=}rsXmMZCXMlu*fZJz(q~Ip%fr@jPN%gR~ z6x_CZ8BXDYUn>iYy(<17#oHnZ*1$23DCh4dATd5x4RDF9>Gd6r3Cz)Rh*xe&k}PLS zCc%SIchRD2wKR-NbX{4HK{IMg3sp-(SkZ7HTo<-6q5JudPBx&G?YkvM7^liS>vcCYe7zz5^RXybBF&EIeYdksYC{{{5i19CVM}K zQDdgy1Iy+3noe9c4`$~faY53AuM)L$VV`qVsd~W z44{6o9{}`!03AN6)uQ948xYcnJyn37j+RQR-iE(BDM%*%lH&Dct!wIZGlAc*Cs4a{ zxW4)blY0b&wtbHCD7K;!QYfl&~Yoo^phR2B8I@*~T zflu5x@$L5wOTJqrG+l#>j6@xI06ltQ+3R*#%n*hER=C+m!H-bMKpP{U*! z!)-u^-$zclq9w*9Uw?++FnJXb<5>@y)A_RMi^UA&O|Df}C--E$!xoqvnwqBJCV>sz zegM>liI9Io*Z~{`qug+^on}HUEZKH?+e$TWmWNH|gcp7e46b)xSa5Ol1Vp(Xtyp`< zDE17M9H(c^T%$it+-<>ITPoe`5B7&%4+lJKUga(1ylP4z>jLabW7>;ECbhhU3W$gT z#w~2$6}N4$mpqTtQwM&HLR2YgP+r#RY!O|LdfE8Zp9H|Te}{l*x`wl5C66#G&gxq%_bE=PaJw6h0}5hG}rV4=zlf*|4GKajcsj znmcM-N}1>#ytGc6u1rDnA-o{F)Rnq_ZBi<9P%ZFg9#HK&;nv!dpY6~ar*nc}x2;LA|fB$85GhXXceqadbn)cJ3spTi{j(ix#Da3k} z>nb0Q6PKtt=eJHY_x{**tT4o@v(OB$3sjhJ3|dW7Nj6x51KD=TaHIg*AzB$vqDKp4 z@p;7$+C6$kz6SHWC8V7&MN_fakLN$kpGbuZPytEi1Mh81`*H_1ypslPFDk>}!0N2| z+Uu0fk%tZb>Ybe0+{pKrNaa+(49L&#(|U+swz~&6Z%)8}+__c8(uV@3o$kT>%Ur)j z2qm!`NnzQEakLzp!`yoarWE~mOF;H7c07HYtj}nvyQ$@xB@8;jKLMs}-$7=UIQR22Ytb6f9~#p+?Gj!dE8E8DCHZxI)K|F1nlX7-{ql|#+2#+{pQ(t zy93xaFP4(w6fFVluX1Ng>7|RWMq%$I;JRf#fnJ@DC)Xikfdxp7v3T*lQ znpLLwTx(*2a-LIO<#Cp)-1JYE1%_<##V~hCZl`p_g>pB&X?^Y1ws{&xM0j7thnYKl z{nU8Vv^vc!ABV4f93q;SO-O|HTzM`YNlXgVxE6CYv_lRs>K=Dl`)eL(UH&vqTP`+h7X|aS6d0jjFZ;F{R z5cX9%`mNzq-;h2AfjIRt@68r2&BVl*hUs=gO$HOKM<>XhwH^1tiCq*6vxi25*S>(? zDN9sAQZ=ktBFI1H5N@o=AqEsX=>t8njGm1s>aP44VexsCUL-A+c)d7;BZbj8@PREU zayFJA>J3-aFqQVPLM4&jTI3U5D9Kl*4ZJOqN|l92aau8Frd?|wui_eJ54iqn+8KWB z!7tNwekaasR|P`Bp;H*7g4vG9>B*WzJ@u$8Egq}cLITj}lxQ_4L~G_~KU;F~3P_RG zH0o#f@y`)E7x<;c0}qAZcut{!%HAD_2V=PqXlQv>d7v# z>H8uiZZSyMugAMb z$9~E!Bd%rZM*GTMUgtKHV~5{8M_T|9YUU$Z7TPIqYHKLapJ4y_kc>ML*=o>cT08$K zd_R^_$Xi>a!4)mbPqa%vDk7tN1V}GuxhYrJ-50}hI|zVPZ3;bFe;va|(_gI>3FeAl zbe&NYJFGw+Dh$FO0eeT-?CZH%JkEU~!sSH9oKvp^kuS0@>)71XizP`-JlqW|EKl{Z z1b(w%<;&@FH@Bnygl52<+(%u-A-Mq;;I82@@5Z@b$15b=pXQTdV8kR3Ldrs~?5=#X^rP;mHQu8mPQAg%ljp8sU^Lj=hhww$@heAiso8qAIG{Iz^zU8IqctBa?#i>KL5 zA2(|c6Ks;!=?`jSRDtbnU#TTrF0+Pb?6)ryUQYrVe%87>F68rWasjCZ=O|?j88b!I z_sWQE)D`V!BfBqx(--+fczg5Yt3o9M??lCS@2+W6C7~T(pzm_-U$>NN&Jf(EC|R`9 zNw#y3JJ3TojNm3i@@(_T`xREL;RA#Y_!>!4%R3b}(YD`wl8nejGUY7E95Z7g0^15I zFLc?(hQ4a$NQ_#t?=^Qtnhe4|IHTl5#IDfsf@ii!CAdcSN^qW!K~Y~{b3ETMqs*<0 zRb(2Qxu>os0VKz~rY31QH3lZ6CX03n4-< z@~pQJ+Apk!Ag=++z5SdU!+tTH554*>CyNDw{ai z!>wnmJ4e>)N?;hLzkqF;xOh}JzaL7RpRutICH@)z?GeSFDCdW+f1+q$eiwssZqWKY zo&Sk)e#XEKXul&EQ%wGe@*l^wKLO8sDXge|$0UXd4B(%G%AWw|?FQCz{SJA;Gl2hN z!2U!y-;%JlNsKNp`bD%fIsE--%sv-rQ1 z-am29XDqe^{SHx#K|NQH{#!Bk}q(~DH!GM7DUP2cT z2u(zKM}^QtAoLe>?v>-*<2!TTeRuB6UCAW>|INSG-v77PUVE*z)s%>c=?Q>9AVIob zu{Oa8VZ>j%S@9V=7`mF{*Jh-XrPu3K;isNr zvtfGt=q+`|m`~pg5pz$%->f^Oc2>J>oXT<6jOe^BE*8_0c$_X>*S$!+YFiS|F6uvI z5YQ&C6WwaFR|*k+-BX)&VcAnJtN4_6M6>n{O$qTSLC~jxQmQ!?RE+p5a*U#jCMI`L z?u#@q*|ViG4K}mbNR&)8yK$&I6j&}yu`QE&ZdVi=WC5~Z*s|Q!eMGwzzjuj$S#m$ZvadwE5v2$4!=}o-uD!EXhuckx>Br`G(AR#0mAR{6mQ2XDgbhUM` zbTl(DadPDQ_V=C2@%y$+Z4lBa{oW&%HOVs{uRKtoaFFgL;IR-=_Ec8l3NO4?%<8S+ z>#eA$c@AOCy!}QmX3`7MnNaOsh;ZE6+^ZRltr&2r&?&$mr-!{<1Yh$GX>|_fZVdPM z?oH=PB8Hb4a&udkJ%m$xe3D#B3iuA~wruLW>uQov2Oa5+7Os*t!>gBjR8?IQE)R?w zZClkPX@;og4~^Ee9UtAyw7$+%Wz*-I8=xrWbMZ1qm>XnN!o6eJ`JY%g_Q zb{()d>NQ4;^KONFUbT6WG9F^%x+-PqT2VWe@4B>fP|~?M{vfLv#H*UMoiK0^xN8e@ za^-+m-?!4QANF!Ks}k#)ng8Nabu{mzr)oX%tY^YXtB2XsYf~t4Ht}P~hLo?T>s~s} zyIo|p2i;{a?gQW3PT%)UZQ2(KRMnnu{WuwF;-L`qZNlK0ZeYg;KT7b-G8E+TGM2CzUG( z=GWV{Md#96D?Y33_AEOc*j4V7_6)Q<8ra3*n#X)AkNoOBJkvoBpC>)A*G5;1skpc} z2Srejq!}H9S+4f21*x%YRvjhYrmp1F8I+!u)X{3>D0$DWbwe??C8TMLRhg}X< zj6KhJnz>!UE%chWPA=IQ)->1H@+}~e52_E;*1{b;&0J&|7TDCskjb2<^mDeoMo?rko&U%9=CO z+&3M0-WkWhVug}2J6M%r=7Nf`ipe+ojf-Ki!by1@#>y~D!F#bC$v5ABhhB8hE2GQ= zF7)+LfF)^z-+)*mKR=d2Ffq)(K_1LV4&ah8fB~_@e(5ZQ^u#Ft22C&{CBRa~fES1* z@oQu$WF&?KG?;=JsQ|e$1|mQ#soyk9;W=VdK!XdIkp{3NV;~8{lKBxr3t5O^fepc6 z#!~>Us|NBwEYJ@MEo3D|1vWeZGtvPpuNtTTvE+U-&_Z@%SWp8B%y=4*d(}V_h^6o| zgcfoWqkLQ4E-zM48xdT17t@6=tJi}88lS^#d!mLE&~2?`@i1Pzvs#54-i^}tPxTH3woFe zErPSU1h-)RS#XDr=CD0LYGaLrM#4`1@I!#q_7sgIE|%u-6Vbn{lmFuw|D?IGOK=w^ zdKe;zkTpfNV?jx&&=UB+9^5-iiEj5VD4y!%_G%QTeEPVcbV`W_r%j3efVZLl&z}F{ z+XSVi7sBaXf}1dEA0QM6Swkc{7WD4~aXC@wyo`dSPn3w=w^90qr5BNN;fUSED0J<` zf8%gWkl9#Jd}?|zT*D>!Ek`tPgqcDYI+IW$t4(r=^cjr*MWwGaF9!IBZmJ2gcQ+ifMmpi0#g6` zz@~j)+2+qHrpa5Ad|Tn+B`NwM16wXt9(DNG+IY{)-C1(<(9Jp29-*7ogMQfNSFa9c zrbO5Ai5^sro7U(o9~BSt4dsqcmvqkze*M_GHRw==aIczRhB+rPkwoxY*Q=zBJvPESJ{L3eV_TBdiE2p{X+^BfntZ)%Y1*74EbmGqiV?CMR zT%kQws}E;h$KBf$l|BP~!I#RLNthki(GIkrA)PPI$v4?GU)yRiW*T2B?Gr|hip|(! zd!03>b=78ne}?gb?p}Ti#6W z4l&D2$E1dn?QIg{v)~cHuSli)PO6iciM0vex8H9GaJQDO?I;WV0m&Xs==LCUQLL!N zZQ;)^B&?E4tw!KQMcrXVB*wZ&2mCHp`-7U6oP&IRomC0VRSj%Imwn+BhX%X~nD@O^ z#n1-su8U1E!uvZ9j_k?(qC@3E7eEEm7tgGjZcLwjZ5##%#UbdpN`qg;wZ1LBmffYG zGf!e_!Hsbb&RqP`5+>@QW(c>aDwn7byvCS!u%ooUAqs<{JDb&mlk)uSZa&s{CwGH_ zkWe;h#!W%s3BU%!Ds@Z8BeWlU+XpDTK`2t~`j)vdb~egkjjT_&bd&6LciA98_PLp@ z-CTQNmTK3u@Q<47!IgSfsH6?EH3goIj`F20>qZJo-*II0MDCRjwKA1Qu$@_EUBNWlyS zS9Hl)k{9O#Ccmz~qi6~25Ae#wh4;2xQ&B9{k{|C=iJX$lN#1liuhaC{hUi(R{xwtx z{b`G{*L$Kv#horlKc|lDTA|#hr#s zl~%X}o2l#YO8XbUs!`HPG=KU1guo&yb^2-aPE!@N|GGv^hR~VLiuN+$S|;|Tt%d+` zN4S0v`K_YhqUv`Va4wB2jkG|p@;h0!xU$|O#}ay-b_8ob1u7(G6M4;un!V%g1=CEw z;wLGyyx46e-onW=2TiFC(I|WNOWf94k#}U|uj`ZZB6buXOJ?>FiT2YSMa8KdHgS+d zghodupPGKNU~>nntF?LSocr?UHW1Hk4K4Z&!Gfvk@m_)V=}B8!CXtG6H@6b9t9uZe zyxk80#fi186Z!2-d8V^@`+YUMcSlsU9u6@O2aGs89kF^kqOn{#70{4qBblCpNb1(o z3^>d$i*S9pvU%8$%9G-IywWHZ41_uGwWccSU#&Fy?<>vG%+SHaSl!9txPEtBwkvzw zZPCmR4orK#%Hni`rUT{aBFB4xJC9ZAWXHuRp6RBC27cweW z*#QkpShULLGF8t8C0eg<5gs<#qD#j_Bd|2TZK!&2sl0R#b{uqH$vU`}XdR|=E?!y2 znZIFPe8K5jr~VDGD;)AORYatN8lFx^G5cS&1Z5b>tX7bBp75s~GshhmJ%+>c(uVg_ ze_pA>?+nL9Pe&(rE0b@OwchErdCCv=@ka;Dd%o`!d6*0g&AFjbClq4!QZG^<{PYpa zXl8U)4`VZL?&}fjxa@Nc8?&n^ke`_}>rt|{NJd9Fu)zHg<#Sle&Q4DrYZu!BG9)Xoo z9eWVH4`W7?yEfDEc_Y+-WV2T61=Vr5#|zp;dJ4HOaS(IvFVciB75(MaTQhnXIOWXc z)S_f=TCkNC$tYeh_DZlNf4|&_&F`|il{d{1rnOfmV=&uB_&nM{e~2n7>`D;P+Snq1 zSmt26AP7z?{gKUBgvU9HLYT7DdVURZwLk27Vm9Z^IKXNQZw#MXfnM9_5Rx=H%Sq$~ zzY71CZ9j`K>{+g_v+wDf9trB4(krwpeYrlt9^rFu#Kv7JS(ew^TeGr*xmlO~BPg%i zGxLz9%4@%8KVh{)73HD>YVA|NmXVFv@fhPgD<81Qn_S9d5Fd%EKF^1&`YVH>`ED@Bk%Ez}t>w4kc#IOhI1Ub0?M2+32vC&c3RU>-`rhpQbDFoq5tlYdkY+Er;v`J>_m-^7 zXSlBwl9(X{W*IU3MF>>e%D%~ET+#?-;SvHb#FjMYQ!zT?H#y7QX_|&sJoO#G$YfsG2Wgx^|PWe z3rRUGB7HD5`muTh2scpm>{Kx!KKQW=hL&dSTZ#;ae%x*l2 zA_Mn-5G-PYDv#`LrHV|6gRe5^Jv?gNKk79rIXLUF?#JQU$dw&NOYA{s_3=K*YDmJ~ zC2&${MK#TDry~*#?@wCm6%Dn(hn)o|sn_rMT@35FZ8~e$TLnBNF)0M4O1_KIl(&wm zmaF3Iyq>fGQPVdLtGg4uRSvLcym)vCA*777W9TxMR@LHUUFUXquEG=JukS9B>)$M` zhrG+v5p9m$p}G`Wg1%F)S32=Ia4V9Q&rL<3!nW}WH~9vnUjT7!mnbQZSwj-l+8)lV zs$T|Wl=vsU)Afy_$Q~JLe||7YFP&S6>DFyK!-(`k0hkrlZ_mqR zq0(9AxlPNPmf`IatHqWJEc7s*| z=Q0jH(T+r$t5TH>v_=8HOUk>>UCI6p2@!NK8$ex@>P(ckt&FvZBi-Tc*&eHBQu9vC z>jQXYP#P>Rt}&-n+o-u+a8B!c!+cu*A*+%tKgC_xA*<9x=o{|zhTIt6D|hSgzMKr0 zX`9YcjDHdKU;6gr`18K3t&Nk3jg!8HyPb)n?zc@*y%QhAgZBZ4&@ZQjZNyjtleZdX zfeLXncd^P32Kn5_hbBuDwOMmm{NCx+mbW3`OJ!LbsU@wXbV*ZO0vv5Gqzn8-eC|Db z)VlFWg+8v)vas<1+m?cnWKA;f)~Q!h#;WmV4pG~;V=Q_J>o*#Mj{JRnR8;Sx@GYfD@$-GGBoy)7jlPvQL zb~ia)HGKkDYxp^6;-5{D2}i5de{6|FLHVSppALkm6q;b5-9*>gI@p$aJj0uVT-(Ls z;l7mTVite*sWrS;5IzGAIMLtqSlr{iyX+0TtnI+bH*`$J0c6NK;gPh9WMdQPs`V;A&879f9cNIy3`X>@<& ez7Id1f3UpOlt}O!H~|3#{*#P%h1bs=-~A8A_r`+& literal 0 HcmV?d00001 diff --git a/lib/galaxy/app_unittest_utils/fetch_workbook_paired_with_hashes.xlsx b/lib/galaxy/app_unittest_utils/fetch_workbook_paired_with_hashes.xlsx new file mode 100644 index 0000000000000000000000000000000000000000..de9faf250abc1eaa4cd7ae3a32901c5330c1fc69 GIT binary patch literal 8303 zcmd^E2UJtbx~2+9fJiSQph#5+gaFbaA_xcwNS7`UdZ+<3fD{2K3Id1T6cMHO-lQ7| zO*)|p9D0)`5MI!CpB~pezO(MT@25EAfouuthRa(o8@kZjPbP-;)u7%^pqf3_asMIH zfOZAL$X3V0QYJ`VckSzos}GG|7oYPEZ#GcYk(QVe2K?S%O7V$3>$$`mGPDvx2c5Gd zzjd87@nK748rRHH6Ja9Z>^4?)D6kxIc273t{1|U^+DG-w&8{U})R%S?@Gosgirp5nRM&Tpw+z;IZac zVNJ;Dhw7*l-=D=4@pX!p@$lOh8BgSWD7jc+qJEo%l*HU7fEXVSkAx5pPyK(R(gW^d z=W1EMgxYNwKYfHkD?*Joe$-S!XvmI zywgv@&Z~DZcK@zp2mZ!-YAP0A%%sb$;Y)LS%~;7^Kcd30K)`Cc6^Rhd6ByD(4J3t) zn4D~Fdw3(t#{_kB!^U=OtK*MlR;_$Jj|B^4ax35-V*RC1Nw>mLqYsX|hh3=cZOmMk zTg7ml9z1JcI(m6$*Rcp%J6?Zn(xJS6W~sgxv+9-Hy?lT%uS%5uyy3#{om)^47uSDh z8Rg;8Z+p_yiKsSOp=zGQToJ3eQ-|?J4q`A{E;SQ*1$%-5LmtnhSrweTWV~yW*GN7# zF!o?xh5znY!4>%x%MEvOMy8Ky z77^7_J_i?)q}(>;Bx108h-x44ajDe>k&W3x87Nj{H*LMaw86exU~-i*0_$#2Uv-TV zfj~fv1Sc?SzJ`<7p)FfqS5Lu)j>XSd-+X9tC~CTo5>{O(oRvJ9iKxyl5POp* z@H%xgygefY`l02R=%jR5(70g~*M!3}2P)PJRQ1r$#Wm~8Am5tinst+IlavOisCVxE zw3qyT@l5*y-6kqD?E>EnjBXJX3Y977Fwr?Ws(pdf-+aA5x1;6kQ@PbY*N*!haxIo{ zt#-~q3x2;N0F{wk^>jI)qC42;MGB*QkrPUu!d+0`S8?3iN3RQFG7Ij=@|5&{j9XYh zpP*V;v-UUR1}XDWu{zx+8jNKi8?(W8FCJ@?C($FWp4W??H^}bfROPY}R*F_iROz#L z&J`_^klAUW%4H|4AKjU#GH6lE6%9$q{RYWL^CqhFT7+`l6-ju}NvoP=E<4uz1UL z7n1O{Q%g0=R#-dwO`^&v)Gv~d)hVKyWh1N_ZIh_dZ;|*7a#PK+6E=!oOjH@NsOIX2 zB;?;lANXvt##A@~j8^<*<2NV@Y|ccgLg?qf4x%I)^lOv?4wDcN%9?VJ zsu1}_u!E?Hbo?7tfWu@2aGH{rNAmE0n2C2$fKUEG8ACb=E#$4bq9YOOAQ+-kuaz6(SkO0x( zr1P^e?)z2+WBHYC%U1Gf_**nZp!U08dgl{y}hy z`@&HaiSPV0%Y_G1;YURzzH3cv$PI3`qh6AKS|k6*XZ(}u9V>u@6pj+G1tRDJI&y?b zSW?ak_Rj}*A@s;7ViU`zIQ{(~cY5(lCN@oR1{-Dc$RgtZ-$}~$!HgCCJFP2+p7#5({`-0x&E8E13mAuT z)BqMLdN}u>OcGhHUz>pSF7hrhTt)lbTc-+;04%hc6%H-$_(%Y}eUQ%JtNChzj<1)G zH#a<4$AZZ{J&R1>7nq^Dhs&kD71!>;7s{#y3i+ZkQMo7K@0q5i)wQpr}B7jy|gUIw2$(Y1qzgwsWC8PsbW^N9naPXM z-x0q~&+$SqMIaMDJEpUP)RyYZT=Cm{%P%$?yDb(hn2pk22yE=y41A@>U1!=zefCi* z8p&~Z@iOeVK0Ol9N0JnlG^NdW`?!(?%ta?fCzlOUB&5u`mbKv@D#QO~lbsBFRRq|< zUnJ)(!WJN~JaH$mGhlWLnLh{ZazBWxavP~XKjXp2@f2>-Wk9>@&EnSk@}U@7?Z_iJ|DrE3HqW2ZePKBNlS8(=Gn7Pr-eEWVJoPb{?` z1s4@{g%%N87@Zu05cWp{I(B>mf_|u~xaO+H%R^$m;Imj$0Y&s+PgOBT0Ci9C7kq?;%6ii>D->}-AX3nz+1p{IvY4}Tn-o&)NFTR=mLD6uI*vj?_+AAn? zX{#kv{DHa|*tV)%xE^NbchFp72K|qps_N zt5bFK>r@ADxJQi_$$G{9fqIfWXs_CnLYdcW?`5c%vS_6E>$J4s3Lez(wODBM-9sL^ z2G@FBycEMSX{N|$#7{M{og{B|Mn$_2l)kgJ<)odxq$7w_7*0YD1XX;Hw<9agCz$-a z`SEN^U|#?-b3d%7<))fasjdR1S1n>n{%zup8>`{FC`ZCnl*!GkU|Kp`=3Cv7ArfvE zuSb!`iitDbyY2eEeqO`UJ$ai+i&C*1;VytiT+YnR-Ijl7nzT_JM;iU8AfP@uP)I+n z#Xhjbd{>zIsW&fu6KUA1ZCY|vN2r7dUMyY99-qSXz>t~j4|nH$(QB<>>C09gBWoR7 z1ncGrYmuPxdvSq96nAN->kr;lQTlIc)w~p;M^$u`L26lemUkNiBwWEJkI6KOf{Lm? zYJvH+q?)KnfvO+nF2|JhoVb?I8g@u>_MOcNe!By^X->)0`Tn9+reE=INwWegd$$D& zC(~SXpq=7Roq4#gIOs-b%PQP5Arp{1P>Pbt>?IWMqd9pRqmF&YOB@~&8IgEyx^3Q3 zd&Nk1N8`NL>S8a{IYP5S8F@i z#+0i`zNa(o?7S(27q^jBMft0lM*D51xmugKSX$h5b2;4{Jlzv4iynL04*~|IJ@N=~ zDKjkH`N$Q;)9{Q-OBF$I`iYi)w-^+Ru@y*BH9ow>k2%!cJa*JEjK9?I^k~ zy>`C04awnWZVo)I92=xEf$Ns%YX+U021S(SneYsVVUkpptN7mBA00eQem0~U{Vw&4 z;_{;6eGcbDl8(#C9;y;(qWImLUg9GDLKr~LGk3bI{`lMXv~^z>o%p=iYY?keU8+%C zyRassU0Yq9GS3~qxti69$q zlUaSZC99pYgcXU2)m@-t_!9lZ)$L>xqwHLidCX#Dadjx}{c+bu>BNqg@7C*!oGSMP z-VT{DC$gTOI+zo`Gs*MGL?htyu!XYMI1dpd>oP3;(#a%*1l|lf9IZ8>7#%JA zFB2xdiLM3Zk-&N%^ys|p2oG;dOfQ}4@ec@Lu5))NZF^|{*tM;&qcn2Vk7!Yud^w?~ zcvwA*jT=*SVc`uhSXP)VFfPcQ`9>os7r$h7tzlOA;3l<3SoU_g%9O& zMT;#c({0!1m})`&*|q@%5|xwot6W8uc_&hDHww9lGk-@K2!DD zJa=i~P5v8^#0AeO%1Qcl;RTY;bpy0C!I@Q;@duL_;EW!M&Yi#CnIg4KPhL76aoZ4f zEzf8;lT)9&ljKEqkGSyo=f%sG+)~OQyJVZAP$qeme$uu(xSb$KhP{>MYmsM}FSF|= z-5IWrX$bOhJZ5Xlb&7&0JPeK>u}PcFA>$F=W06)T^FAAKQNAr)gx)Li(O@Qiuhu5x zc7n2o{<%h>CuBM|SXi!8^UnJ2Bw7?Gl#anHE3}`Yg#AqPvGPvmqB#XMS=lJ{9>*N6 z6*v=x0*s>;|B%6S=CWrIycnvA}i{J#s#z1Y98 zw`5uE?P^6JuK?pdD@Vf6yvGi4MC8$Z@wu-&eTzkaT2RlkEOv8(C)`JW)WH*4EqB%1 zRi}Cp+HWMX0Lb6D7lREqm^gtQqbfdf_d6Q`T6+~&dIRh$>0OSlj<-<1wB^oZkofXc zP10%bcsl*o`gG0RBmC2WvwZ7=3KqBC*Pi)R6r%brqRioNyRU~5&u`-nC4fP9dL;EG z0+i(WLllS8b?n~?j1mISfqMJrMz|;{)`8CzV^lxJO8X$OB3QApLF>VEs`n$|+}p%3 z>sS0S(pE=@%MRr;S8f&(Tf>BA834#6sDn>3lr>pWs2VQUmczKL6~fLh0-VoRxGni= z`^?2yEYFSAbD`=S3NKr?%{Tb~JSceJ$2(?~#ah%JYN-^Xo&la8ft@m1Zz&18N~y$O zW_mQmIt!#MDp{}-ms3k-45UOx-39yu^G{T^WY6m2^NC$J6Kvu)Z+!{*bbnl?u0xC! z`gslk$D7@obTBR&Y5|Y93z1W9>ib;^?bfuKb?T`i zJtsXW!a0RxNv{^Zz;0`IN*KRN_omdRqji46vR1>OzH%VO~+h*z$ zlDzqaFd>&sOD3zeBaBVgm++La-2F5&p7`lVha$KCObuCEkDbjwougMh?6G@ij$ zyP8gB;TeTOT=ocs*%x3;!Yb~YXneYtuV%&@H;0DN_pMIT6^Luh|M>k7HHxn%{CrekOuX^Dw3YXsi zj|i|FTXYb|YwUrEyN$D?iZN7rE2{Sg1idgrlO;+9oNw9vJ{s4Sw@ZST%U*A%l(e3q zNtogn;%$F%y})18NB_yQ*6rWbXk(h}3Y+d<-c>Z0sYw*rJ@;nH;!do!%hNs0=e9lg zu^ca8nUtNf^OgqhNo|R5$B4~t=4my-K6}O7qZCS#F{iLg3JdmWc$H7oVjbcCp`J7P z$&5|ADZIo81^whuD>&H=NWhEgF zYg=GR$((4Ww%9skl6{WBNrlft$0vZZ2EtAXiBxY`Xo<+;5QI`D_Xn#LTB13+ z2yeD`US95g7}osQqvKjE*q8kL^QA{VwbnV;exnROS?%%;DwJ44t--qMzq0NSQmPKY zG0sGE$~ry)E#CJ7+#k;)PY-baXn%H``?HrH^VmOnxs7WV=jDgo_4`TY&t85^+)i`e z@1~E7-hcG+cQfFh9sa0KPY2a^%LjgS_;dF6vx6Ts$*GL}Zj89%sSf#B)c)+@$HnPX zyL>lG`fncoN6GwL^~Xqhx_W)LR2;|uO~>Ej=FgRX3@WF=_Pbrd6>*jSB=r95=f@d) t8iKx?G0PtU($5ut9CUx=Ug`Ybo!;uViE#@!9^P5pClMD48(2?E{{?Z`j_d#c literal 0 HcmV?d00001 diff --git a/lib/galaxy/dependencies/pinned-requirements.txt b/lib/galaxy/dependencies/pinned-requirements.txt index 61c3b77ca10..0ff279e8b11 100644 --- a/lib/galaxy/dependencies/pinned-requirements.txt +++ b/lib/galaxy/dependencies/pinned-requirements.txt @@ -118,6 +118,7 @@ numpy==2.0.2 ; python_full_version < '3.10' numpy==2.2.6 ; python_full_version == '3.10.*' numpy==2.3.0 ; python_full_version >= '3.11' oauthlib==3.2.2 +openpyxl==3.1.5 orjson==3.10.18 oyaml==1.0 packaging==24.2 ; python_full_version < '3.10' diff --git a/lib/galaxy/model/dataset_collections/auto_identifiers.py b/lib/galaxy/model/dataset_collections/auto_identifiers.py new file mode 100644 index 00000000000..519f50915f9 --- /dev/null +++ b/lib/galaxy/model/dataset_collections/auto_identifiers.py @@ -0,0 +1,71 @@ +"""Code around assigning implicit list identifiers for collections.""" + +import os.path +from typing import ( + List, + Optional, + Set, + Tuple, +) +from urllib.parse import urlparse + +from pydantic import BaseModel + + +class FillIdentifiers(BaseModel): + fill_inner_list_identifiers: bool = False + deduplication_pattern: str = "_{#}" + deduplication_index_from: int = 1 + + +def filename_to_element_identifier(filename_or_uri: str): + basename = guess_filename_from_url(filename_or_uri) + for zip_extension in [".zip", ".tar.gz", ".tar", ".tgz", ".gz", "bz2", ".bz"]: + if basename.endswith(zip_extension): + basename = basename[: -len(zip_extension)] + if "." in basename: + basename = basename.split(".")[0] + return basename + + +def fill_in_identifiers( + uris_to_identifiers: List[Tuple[str, Optional[str]]], config: Optional[FillIdentifiers] +) -> List[Optional[str]]: + if config is None: + config = FillIdentifiers() + + new_identifiers: List[Optional[str]] = [] + seen_identifiers: Set[Optional[str]] = set() + for uri, identifier in uris_to_identifiers: + if identifier is None and config.fill_inner_list_identifiers: + basename = filename_to_element_identifier(uri) + identifier = basename + assert identifier + if identifier in seen_identifiers: + # Handle deduplication + index = config.deduplication_index_from + suffix = config.deduplication_pattern.replace("{#}", str(index)) + new_identifier = f"{basename}{suffix}" + while new_identifier in seen_identifiers: + index += 1 + new_identifier = f"{basename}{config.deduplication_pattern.format(index=index)}" + identifier = new_identifier + seen_identifiers.add(identifier) + new_identifiers.append(identifier) + else: + seen_identifiers.add(identifier) + new_identifiers.append(identifier) + + return new_identifiers + + +def guess_filename_from_url(url: str) -> str: + """ + Guess the filename for a given URL. + + :param url: The URL to extract the filename from. + :return: The guessed filename. + """ + parsed_url = urlparse(url) + filename = os.path.basename(parsed_url.path) + return filename or "" diff --git a/lib/galaxy/model/dataset_collections/auto_pairing.py b/lib/galaxy/model/dataset_collections/auto_pairing.py new file mode 100644 index 00000000000..f7490898529 --- /dev/null +++ b/lib/galaxy/model/dataset_collections/auto_pairing.py @@ -0,0 +1,153 @@ +import re +from dataclasses import dataclass +from typing import ( + Dict, + Generic, + List, + Optional, + Protocol, + Tuple, + TypeVar, +) + +from .auto_identifiers import filename_to_element_identifier + + +class HasName(Protocol): + name: str + + +T = TypeVar("T", bound=HasName) + +# matches pairing.ts in the client +COMMON_FILTERS: Dict[str, Tuple[str, str]] = { + "illumina": ("_1", "_2"), + "Rs": ("_R1", "_R2"), + "dot12s": (".1.fastq", ".2.fastq"), +} + + +def paired_element_list_identifier(forward: str, reverse: str) -> Optional[str]: + for forward_filter, reverse_filter in COMMON_FILTERS.values(): + if forward_filter in forward and reverse_filter in reverse: + forward_base = filename_to_element_identifier(re.sub(f"{forward_filter}", "", forward)) + reverse_base = filename_to_element_identifier(re.sub(f"{reverse_filter}", "", reverse)) + common_base = longest_prefix(forward_base, reverse_base) + return common_base + + forward_base = filename_to_element_identifier(forward) + reverse_base = filename_to_element_identifier(reverse) + common_base = longest_prefix(forward_base, reverse_base) + return common_base + + +def longest_prefix(str1: str, str2: str) -> str: + prefix = "" + min_len = min(len(str1), len(str2)) + for i in range(min_len): + if str1[i] == str2[i]: + prefix += str1[i] + else: + break + return prefix + + +@dataclass +class Pair(Generic[T]): + name: str + forward: T + reverse: T + + +@dataclass +class PartialPair(Generic[T]): + name: str + forward: Optional[T] + reverse: Optional[T] + + def to_pair(self) -> Pair[T]: + assert self.forward + assert self.reverse + return Pair(name=self.name, forward=self.forward, reverse=self.reverse) + + +@dataclass +class AutoPairResponse(Generic[T]): + paired: List[Pair[T]] + unpaired: List[T] + + +def auto_pair(elements: List[T]) -> AutoPairResponse[T]: + filter_type = guess_initial_filter_type(elements) + if filter_type: + forward_filter, reverse_filter = COMMON_FILTERS[filter_type] + forward_elements, reverse_elements = split_elements_by_filter(elements, forward_filter, reverse_filter) + partial_pairs: Dict[str, PartialPair[T]] = {} + for forward_element in forward_elements: + forward_base = filename_to_element_identifier(re.sub(f"{forward_filter}", "", forward_element.name)) + + if forward_base not in partial_pairs: + partial_pairs[forward_base] = PartialPair(name=forward_base, forward=forward_element, reverse=None) + for reverse_element in reverse_elements: + reverse_base = filename_to_element_identifier(re.sub(f"{reverse_filter}", "", reverse_element.name)) + if reverse_base not in partial_pairs: + partial_pairs[reverse_base] = PartialPair(name=reverse_base, forward=None, reverse=reverse_element) + else: + partial_pairs[reverse_base].reverse = reverse_element + + unpaired: List[T] = elements.copy() + for forward_element in forward_elements: + unpaired.remove(forward_element) + for reverse_element in reverse_elements: + unpaired.remove(reverse_element) + + full_pairs: List[Pair[T]] = [] + for partial_pair in partial_pairs.values(): + if partial_pair.forward is None: + assert partial_pair.reverse + unpaired.append(partial_pair.reverse) + elif partial_pair.reverse is None: + assert partial_pair.forward + unpaired.append(partial_pair.forward) + else: + full_pairs.append(partial_pair.to_pair()) + return AutoPairResponse(paired=full_pairs, unpaired=unpaired) + else: + return AutoPairResponse(paired=[], unpaired=elements) + + +def guess_initial_filter_type(elements: List[T]) -> Optional[str]: + illumina = 0 + dot12s = 0 + Rs = 0 + + # Iterate through elements and count occurrences of filter patterns + for element in elements: + if ".1.fastq" in element.name or ".2.fastq" in element.name: + dot12s += 1 + elif "_R1" in element.name or "_R2" in element.name: + Rs += 1 + elif "_1" in element.name or "_2" in element.name: + illumina += 1 + + # Determine the most likely filter type + if illumina == 0 and dot12s == 0 and Rs == 0: + return None + elif illumina > dot12s and illumina > Rs: + return "illumina" + elif dot12s > illumina and dot12s > Rs: + return "dot12s" + elif Rs > illumina and Rs > dot12s: + return "Rs" + else: + return "illumina" + + +def split_elements_by_filter(elements: List[T], forward_filter: str, reverse_filter: str) -> Tuple[List[T], List[T]]: + filters = [re.compile(forward_filter), re.compile(reverse_filter)] + split: Tuple[List[T], List[T]] = ([], []) + for element in elements: + for i, filter in enumerate(filters): + if element.name and filter.search(element.name): + split[i].append(element) + return split diff --git a/lib/galaxy/model/dataset_collections/auto_pairing_spec.yml b/lib/galaxy/model/dataset_collections/auto_pairing_spec.yml new file mode 100644 index 00000000000..8b81d649abc --- /dev/null +++ b/lib/galaxy/model/dataset_collections/auto_pairing_spec.yml @@ -0,0 +1,76 @@ +- doc: Simple _R1 _R2 fastqs split. + inputs: + - input_R1.fastq + - input_R2.fastq + paired: + input: + forward: input_R1.fastq + reverse: input_R2.fastq + +- doc: Compressed _R1 _R2 fastqs split. + inputs: + - input_R1.fastq.gz + - input_R2.fastq.gz + paired: + input: + forward: input_R1.fastq.gz + reverse: input_R2.fastq.gz + +- doc: Joining common prefixes on _R1 _R2. + inputs: + - input541_R1.fastq + - input541_R2.fastq + - input521_R1.fastq + - input521_R2.fastq + paired: + input541: + forward: input541_R1.fastq + reverse: input541_R2.fastq + input521: + forward: input521_R1.fastq + reverse: input521_R2.fastq + +- doc: Simple .1.fastq/.2.fastq split. + inputs: + - input.1.fastq + - input.2.fastq + paired: + input: + forward: input.1.fastq + reverse: input.2.fastq + +- doc: Simple .1.fastq.gz/.2.fastq.gz split. + inputs: + - input.1.fastq.gz + - input.2.fastq.gz + paired: + input: + forward: input.1.fastq.gz + reverse: input.2.fastq.gz + +- doc: Simple _1/_2 split. + inputs: + - input_1.fastq + - input_2.fastq + paired: + input: + forward: input_1.fastq + reverse: input_2.fastq + +- doc: Simple _1/_2 split on compressed files (.gz) + inputs: + - input_1.fastq.gz + - input_2.fastq.gz + paired: + input: + forward: input_1.fastq.gz + reverse: input_2.fastq.gz + +- doc: Simple _1/_2 split on compressed files (.bz2) + inputs: + - input_1.fastq.bz2 + - input_2.fastq.bz2 + paired: + input: + forward: input_1.fastq.bz2 + reverse: input_2.fastq.bz2 diff --git a/lib/galaxy/model/dataset_collections/rule_target_column_specification.yml b/lib/galaxy/model/dataset_collections/rule_target_column_specification.yml new file mode 100644 index 00000000000..ab4c4ff981d --- /dev/null +++ b/lib/galaxy/model/dataset_collections/rule_target_column_specification.yml @@ -0,0 +1,136 @@ +# The idea that we reserve some column names for reserved purposes in Galaxy +# is going to span fetch workbooks and sample sheet workbooks - and we need +# to validate it both on the backend and the frontend. So having a set of tests +# that validate what are reserved column names that we can run across both languages +# utilities for doing this is valuable. + +- doc: "Name (with uppercase) maps directly to the name target type" + column_header: "Name" + maps_to: "name" +- doc: "name maps directly to the name target type" + column_header: "name" + maps_to: "name" +- doc: "NAME (all uppercase) maps directly to the name target type" + column_header: "NAME" + maps_to: "name" +- doc: "'List Name' maps to the collection_name target type" + column_header: "List Name" + maps_to: "collection_name" +- doc: "'list name' maps to the collection_name target type" + column_header: "list name" + maps_to: "collection_name" +- doc: "'listname' maps to the collection_name target type" + column_header: "listname" + maps_to: "collection_name" + +- doc: "'collection name' maps to the collection_name target type" + column_header: "collection name" + maps_to: "collection_name" + +- doc: "uri maps to the url target type" + column_header: "uri" + maps_to: "url" +- doc: "url maps to the url target type" + column_header: "url" + maps_to: "url" + +- doc: "'url deferred' maps to the url_deferred target type" + column_header: "url deferred" + maps_to: "url_deferred" + +- doc: "'deferred url' maps to the url_deferred target type" + column_header: "deferred url" + maps_to: "url_deferred" + +- doc: "genome maps to the dbkey target type" + column_header: "genome" + maps_to: "dbkey" +- doc: "dbkey maps to the dbkey target type" + column_header: "dbkey" + maps_to: "dbkey" + +- doc: "filetype maps to the file_type target type" + column_header: "file type" + maps_to: "file_type" +- doc: "extension maps to the file_type target type" + column_header: "extension" + maps_to: "file_type" + +- doc: "info maps to the info target type" + column_header: "info" + maps_to: "info" + +- doc: "tag maps to the tags target type" + column_header: "tag" + maps_to: "tags" + +- doc: "grouptag maps to the group_tags target type" + column_header: "grouptag" + maps_to: "group_tags" + +- doc: "nametag maps to the name_tag target type" + column_header: "nametag" + maps_to: "name_tag" + +- doc: "'List Identifier' maps to the list_identifiers target type" + column_header: "List Identifier" + maps_to: "list_identifiers" + +- doc: "'paired identifier' maps to the paired_identifier target type" + column_header: "Paired Identifier" + maps_to: "paired_identifier" +- doc: "'paired indicator' maps to the paired_identifier target type" + column_header: "Paired Identifier" + maps_to: "paired_identifier" + +- doc: "'MD5Sum' maps to hash_md5" + column_header: "MD5Sum" + maps_to: "hash_md5" +- doc: "'MD5' maps to hash_md5" + column_header: "MD5" + maps_to: "hash_md5" +- doc: "'Hash MD5' maps to hash_md5" + column_header: "Hash MD5" + maps_to: "hash_md5" +- doc: "'MD5 Hash' maps to hash_md5" + column_header: "MD5 Hash" + maps_to: "hash_md5" + +- doc: "'SHA-1 Sum' maps to hash_sha1" + column_header: "SHA-1 Sum" + maps_to: "hash_sha1" +- doc: "'SHA1' maps to hash_sha1" + column_header: "SHA1" + maps_to: "hash_sha1" +- doc: "'Hash SHA-1' maps to hash_sha1" + column_header: "Hash SHA-1" + maps_to: "hash_sha1" +- doc: "'SHA-1 Hash' maps to hash_sha1" + column_header: "SHA-1 Hash" + maps_to: "hash_sha1" + +- doc: "'SHA-256 Sum' maps to hash_sha256" + column_header: "SHA-256 Sum" + maps_to: "hash_sha256" +- doc: "'SHA256' maps to hash_sha256" + column_header: "SHA256" + maps_to: "hash_sha256" +- doc: "'Hash SHA-256' maps to hash_sha256" + column_header: "Hash SHA-256" + maps_to: "hash_sha256" +- doc: "'SHA-256 Hash' maps to hash_sha256" + column_header: "SHA-256 Hash" + maps_to: "hash_sha256" + +- doc: "'SHA-512 Sum' maps to hash_sha512" + column_header: "SHA-512 Sum" + maps_to: "hash_sha512" +- doc: "'SHA512' maps to hash_sha512" + column_header: "SHA512" + maps_to: "hash_sha512" +- doc: "'Hash SHA-512' maps to hash_sha512" + column_header: "Hash SHA-512" + maps_to: "hash_sha512" +- doc: "'SHA-512 Hash' maps to hash_sha512" + column_header: "SHA-512 Hash" + maps_to: "hash_sha512" diff --git a/lib/galaxy/model/dataset_collections/rule_target_columns.py b/lib/galaxy/model/dataset_collections/rule_target_columns.py new file mode 100644 index 00000000000..6232a1cf8ed --- /dev/null +++ b/lib/galaxy/model/dataset_collections/rule_target_columns.py @@ -0,0 +1,162 @@ +import re +from dataclasses import dataclass +from typing import ( + Dict, + List, + Optional, +) + +from pydantic import BaseModel + +from galaxy.model.dataset_collections.rule_target_models import ( + ColumnTarget, + RuleBuilderMappingTargetKey, + target_model_by_type, +) + +COLUMN_TITLE_PREFIXES: Dict[str, RuleBuilderMappingTargetKey] = { + "name": "name", + "listname": "collection_name", + "collectionname": "collection_name", + "uri": "url", + "url": "url", + "urldeferred": "url_deferred", + "deferredurl": "url_deferred", + "genome": "dbkey", + "dbkey": "dbkey", + "genomebuild": "dbkey", + "filetype": "file_type", + "extension": "file_type", + "fileextension": "file_type", + "info": "info", + "tag": "tags", + "grouptag": "group_tags", + "nametag": "name_tag", + "listidentifier": "list_identifiers", + "pairedidentifier": "paired_identifier", + "hashmd5sum": "hash_md5", + "hashmd5": "hash_md5", + "md5sum": "hash_md5", + "md5": "hash_md5", + "sha1hash": "hash_sha1", + "hashsha1sum": "hash_sha1", + "hashsha1": "hash_sha1", + "sha1sum": "hash_sha1", + "sha1": "hash_sha1", + "sha256hash": "hash_sha256", + "hashsha256sum": "hash_sha256", + "hashsha256": "hash_sha256", + "sha256sum": "hash_sha256", + "sha256": "hash_sha256", + "sha512hash": "hash_sha512", + "hashsha512sum": "hash_sha512", + "hashsha512": "hash_sha512", + "sha512sum": "hash_sha512", + "sha512": "hash_sha512", +} + + +def column_title_to_target_type(column_title: str) -> Optional[RuleBuilderMappingTargetKey]: + normalized_title = re.sub(r"[\s\(\)\-\_]|optional", "", column_title.lower()) + if normalized_title not in COLUMN_TITLE_PREFIXES: + for key in COLUMN_TITLE_PREFIXES.keys(): + if normalized_title.startswith(key): + normalized_title = key + break + elif normalized_title.endswith(key): + normalized_title = key + break + + if normalized_title not in COLUMN_TITLE_PREFIXES: + return None + + column_type: RuleBuilderMappingTargetKey = COLUMN_TITLE_PREFIXES[normalized_title] + return column_type + + +@dataclass +class HeaderColumn: + type: RuleBuilderMappingTargetKey + title: str # user facing + # e.g. for paired data will have two columns of URIs, record types maybe have any number + # and after dataset hash may have multiples of those also + type_index: int + + @property + def width(self): + if self.type in ["url", "url_deferred", "ftp_path"]: + return 50 + else: + return 20 + + @property + def name(self): + if self.type_index == 0: + return self.type + else: + return f"{self.type}_{self.type_index}" + + @property + def help(self) -> str: + column_target = _column_header_to_column_target(self) + return column_target.help if column_target.help else "" + + @property + def parsed_column(self) -> "ParsedColumn": + """Return a ParsedColumn representation of this header.""" + return ParsedColumn( + type=self.type, + type_index=self.type_index, + title=self.title, + ) + + +def _column_header_to_column_target(column_header: HeaderColumn) -> ColumnTarget: + return target_model_by_type(column_header.type) + + +class ParsedColumn(BaseModel): + type: RuleBuilderMappingTargetKey + type_index: int + title: str + + @property + def name(self): + if self.type_index == 0: + return self.type + else: + return f"{self.type}_{self.type_index}" + + +def column_titles_to_headers(column_titles: List[str]) -> List[HeaderColumn]: + headers: List[HeaderColumn] = [] + headers_of_type_seen: Dict[RuleBuilderMappingTargetKey, int] = {} + + for column_title in column_titles: + column_type_: Optional[RuleBuilderMappingTargetKey] = column_title_to_target_type(column_title) + if not column_type_: + # make a note in parse log that this column was skipped + continue + column_type: RuleBuilderMappingTargetKey = column_type_ + if column_type in headers_of_type_seen: + type_index = headers_of_type_seen[column_type] + headers_of_type_seen[column_type] += 1 + else: + type_index = 0 + headers_of_type_seen[column_type] = 1 + header_column = HeaderColumn( + type=column_type, + type_index=type_index, + title=column_title, + ) + if header_column.type == "paired_identifier" and implied_paired_or_unpaired_column_header(header_column): + header_column.type = "paired_or_unpaired_identifier" + headers.append(header_column) + + return headers + + +def implied_paired_or_unpaired_column_header(column_header: HeaderColumn) -> bool: + # URI 2 - Reverse (Optional), Optional URI 2, Optional Paired Identifier all + # imply paired_or_unpaired lists using this logic. + return "optional" in column_header.title.lower() diff --git a/lib/galaxy/model/dataset_collections/rule_target_models.py b/lib/galaxy/model/dataset_collections/rule_target_models.py new file mode 100644 index 00000000000..f4986f189b2 --- /dev/null +++ b/lib/galaxy/model/dataset_collections/rule_target_models.py @@ -0,0 +1,93 @@ +from typing import ( + Dict, + List, + Literal, + Optional, +) + +import yaml +from pydantic import ( + BaseModel, + RootModel, +) + +from galaxy.util.resources import resource_string + +RuleBuilderImportType = Literal["datasets", "collections"] +RuleBuilderModes = Literal[ + "raw", + "ftp", + "datasets", + "library_datasets", + "collection_contents", +] + + +class ColumnTarget(BaseModel): + label: str + help: Optional[str] + modes: Optional[List[RuleBuilderModes]] = None + importType: Optional[RuleBuilderImportType] = None + multiple: Optional[bool] = False + columnHeader: Optional[str] = None + advanced: Optional[bool] = False + requiresFtp: Optional[bool] = False + example_column_names: Optional[List[str]] = None + + @property + def example_column_names_as_str(self) -> Optional[str]: + if self.example_column_names: + return '"' + '", "'.join(self.example_column_names) + '"' + return "" + + +RuleBuilderMappingTargetKey = Literal[ + "list_identifiers", + "paired_identifier", + "paired_or_unpaired_identifier", + "collection_name", + "name_tag", + "tags", + "group_tags", + "name", + "dbkey", + "hash_sha1", + "hash_md5", + "hash_sha256", + "hash_sha512", + "file_type", + "url", + "url_deferred", + "info", + "ftp_path", +] + + +ColumnTargetsConfig = Dict[RuleBuilderMappingTargetKey, ColumnTarget] +ColumnTargetsConfigRootModel = RootModel[ColumnTargetsConfig] + + +def target_models() -> ColumnTargetsConfig: + column_targets_str = resource_string(__name__, "rule_targets.yml") + column_targets_raw = yaml.safe_load(column_targets_str) + return ColumnTargetsConfigRootModel.model_validate(column_targets_raw).root + + +def target_model_by_type(type: RuleBuilderMappingTargetKey) -> ColumnTarget: + return target_models()[type] + + +# Any URI-based workbook should allow specifying these metadata column types. +COMMON_COLUMN_TARGETS = [ + target_model_by_type("name"), + target_model_by_type("dbkey"), + target_model_by_type("info"), + target_model_by_type("file_type"), + target_model_by_type("tags"), + target_model_by_type("group_tags"), + target_model_by_type("name_tag"), + target_model_by_type("hash_md5"), + target_model_by_type("hash_sha1"), + target_model_by_type("hash_sha256"), + target_model_by_type("hash_sha512"), +] diff --git a/client/src/components/RuleBuilder/column-targets.yml b/lib/galaxy/model/dataset_collections/rule_targets.yml similarity index 72% rename from client/src/components/RuleBuilder/column-targets.yml rename to lib/galaxy/model/dataset_collections/rule_targets.yml index f6c4ece83a4..63182753f0e 100644 --- a/client/src/components/RuleBuilder/column-targets.yml +++ b/lib/galaxy/model/dataset_collections/rule_targets.yml @@ -47,6 +47,7 @@ name_tag: modes: - "raw" - "ftp" + example_column_names: ["Name Tag", "nametag"] tags: multiple: true label: "General Purpose Tag(s)" @@ -60,6 +61,7 @@ tags: - "datasets" - "library_datasets" - "collection_contents" + example_column_names: ["Tag"] group_tags: multiple: true label: "Group Tag(s)" @@ -73,38 +75,64 @@ group_tags: - "datasets" - "library_datasets" - "collection_contents" + example_column_names: ["Group Tag", "grouptag"] name: label: "Name" importType: "datasets" + help: | + This is just the name of the dataset as it appears in the Galaxy history + panel. + example_column_names: ["Name"] dbkey: label: "Genome" + help: | + This should be the genome build (e.g. hg19, mm9, etc...) that the data + is associated with. If this is not set, there will be no genome associated + with the dataset. modes: - "raw" - "ftp" + example_column_names: ["DBKey", "DB Key", "Genome", "Genome Build"] hash_sha1: label: "Hash (SHA1)" + help: | + This is the SHA1 hash of the URI, this is used to verify that the file + has not been corrupted during transfer. modes: - "raw" - "ftp" advanced: true + example_column_names: ["SHA1", "sha-1 hash", "SHA1-Sum"] hash_md5: label: "Hash (MD5)" + help: | + This is the MD5 hash of the URI, this is used to verify that the file + has not been corrupted during transfer. modes: - "raw" - "ftp" advanced: true + example_column_names: ["MD5", "MD5 hash", "MD5-Sum"] hash_sha256: label: "Hash (SHA256)" + help: | + This is the SHA256 hash of the URI, this is used to verify that the file + has not been corrupted during transfer. modes: - "raw" - "ftp" advanced: true + example_column_names: ["SHA256", "sha-256 hash", "SHA256-Sum"] hash_sha512: label: "Hash (SHA512)" + help: | + This is the SHA512 hash of the URI, this is used to verify that the file + has not been corrupted during transfer. modes: - "raw" - "ftp" advanced: true + example_column_names: ["SHA512", "sha-512 hash", "SHA512-Sum"] file_type: label: "Type" modes: @@ -112,6 +140,7 @@ file_type: - "ftp" help: | This should be the Galaxy file type corresponding to this file. + example_column_names: ["File Type", "file extension", "Extension"] url: label: "URL" modes: @@ -134,6 +163,7 @@ info: modes: - "raw" - "ftp" + example_column_names: [Info] ftp_path: label: "FTP Path" modes: diff --git a/lib/galaxy/model/dataset_collections/workbook_util.py b/lib/galaxy/model/dataset_collections/workbook_util.py new file mode 100644 index 00000000000..47546464a8c --- /dev/null +++ b/lib/galaxy/model/dataset_collections/workbook_util.py @@ -0,0 +1,206 @@ +"""Shared code for working with workbook data. + +Module was developed for sample sheet work but extracted for reuse with fetch +workbooks. +""" + +import base64 +from dataclasses import dataclass +from io import BytesIO +from textwrap import wrap +from typing import List + +from openpyxl import ( + load_workbook, + Workbook, +) +from openpyxl.styles import Font +from openpyxl.worksheet.datavalidation import DataValidation +from openpyxl.worksheet.worksheet import Worksheet + +from galaxy.exceptions import RequestParameterInvalidException +from galaxy.model.dataset_collections.rule_target_models import ( + ColumnTarget, +) + +Base64StringT = str + + +def freeze_header_row(worksheet: Worksheet) -> None: + worksheet.freeze_panes = "A2" + + +def make_headers_bold(worksheet: Worksheet, headers: list) -> None: + for index in range(len(headers)): + make_bold(worksheet, 1, index) + + +def make_bold(worksheet: Worksheet, row: int, column: int): + """Make the target cell bold in the specified worksheet.""" + bold_font = Font(bold=True) + worksheet[f"{index_to_excel_column(column)}{row}"].font = bold_font + + +def index_to_excel_column(index: int) -> str: + """Converts a numeric index (0-based) into an Excel column label.""" + if index < 0: + raise ValueError("Index must be 0 or greater") + + column_label = "" + while index >= 0: + column_label = chr(index % 26 + 65) + column_label + index = index // 26 - 1 # Move to the next "digit" in base-26, adjusting for zero-based indexing + + return column_label + + +def uri_data_validation(column: str) -> DataValidation: + # Add data validation for "URI" column + # We cannot assume http/https since drs, gxfiles, etc... are all fine + uri_validation = DataValidation(type="custom", formula1=f'=ISNUMBER(FIND("://", {column}2))', allow_blank=True) + uri_validation.error = "Invalid URI" + uri_validation.errorTitle = "Error" + uri_validation.showErrorMessage = True + return uri_validation + + +def add_column_validation(column: str, data_validation: DataValidation, worksheet: Worksheet): + data_validation.add(f"{column}2:{column}1048576") + worksheet.add_data_validation(data_validation) + + +def set_column_width(worksheet: Worksheet, column_index: int, width: int): + worksheet.column_dimensions[index_to_excel_column(column_index)].width = width + + +def load_workbook_from_base64(content: str) -> Workbook: + decoded_content = base64.b64decode(content) + file_like = BytesIO(decoded_content) + try: + workbook = load_workbook(file_like, data_only=True) + except Exception: + raise RequestParameterInvalidException( + "The provided content is not a valid Excel file. Please check the content and try again." + ) + return workbook + + +def workbook_to_bytes(workbook: "Workbook") -> BytesIO: + output = BytesIO() + workbook.save(output) + output.seek(0) + return output + + +@dataclass +class HasHelp: + title: str + help: str + + +@dataclass +class HelpConfiguration: + instructions: List[str] + columns: List[HasHelp] + text_width: int + column_width: int + help_row_start: int = 3 + + +@dataclass +class ExtraColumnsHelpConfiguration: + instructions: List[str] + text_width: int + column_targets: List[ColumnTarget] + + +def wrap_instructions(instruction: str, help_width: int) -> List[str]: + return wrap(instruction, width=help_width) + + +def add_extra_column_help_as_new_sheet(workbook: Workbook, extra_columns_help: ExtraColumnsHelpConfiguration) -> None: + worksheet = workbook.create_sheet(title="Supplying Extra Metadata") + worksheet.cell(row=1, column=1, value="Metadata") + worksheet.cell(row=1, column=2, value="Description") + worksheet.cell(row=1, column=3, value="Example Column Names") + make_bold(worksheet, 1, 0) + make_bold(worksheet, 1, 1) + make_bold(worksheet, 1, 2) + + set_column_width(worksheet, 0, 40) + set_column_width(worksheet, 1, 140) + set_column_width(worksheet, 2, 40) + + current_row = 2 + for column_target in extra_columns_help.column_targets: + worksheet.cell(row=current_row, column=1, value=column_target.label) + worksheet.cell(row=current_row, column=2, value=column_target.help) + worksheet.cell(row=current_row, column=3, value=column_target.columnHeader or "") + current_row += 1 + + help_label_index = 6 + current_row = 2 + worksheet.cell(row=current_row, column=help_label_index, value="Instructions") + make_bold(worksheet, current_row, help_label_index - 1) + current_row += 1 + write_instructions_to_sheet( + worksheet, extra_columns_help.instructions, current_row, help_label_index, extra_columns_help.text_width + ) + + +def add_instructions_to_sheet(worksheet: Worksheet, help_configuration: HelpConfiguration): + columns = help_configuration.columns + num_columns = len(columns) + help_label_index = num_columns + 2 + # why is the width not +1 + set_column_width(worksheet, help_label_index, help_configuration.column_width) + + help_start_row = help_configuration.help_row_start + current_row = help_start_row + + worksheet.cell(row=current_row, column=help_label_index, value="Instructions") + make_bold(worksheet, current_row, help_label_index - 1) + + current_row += 1 + + current_row = write_instructions_to_sheet( + worksheet, help_configuration.instructions, current_row, help_label_index, help_configuration.text_width + ) + + current_row += 2 + worksheet.cell(row=current_row, column=help_label_index, value="Columns") + make_bold(worksheet, current_row, help_label_index - 1) + current_row += 1 + + for column in columns: + worksheet.cell(row=current_row, column=help_label_index, value=column.title) + worksheet.cell(row=current_row, column=help_label_index + 1, value=column.help) + current_row += 1 + + +def write_instructions_to_sheet( + worksheet: Worksheet, instructions: List[str], start_row: int, help_label_index: int, help_width: int +) -> int: + current_row = start_row + + for instruction_index, instruction in enumerate(instructions): + worksheet.cell(row=current_row, column=help_label_index, value=f"> {instruction_index + 1}.") + instruction_lines = wrap_instructions(instruction, help_width) + for line in instruction_lines: + worksheet.cell(row=current_row, column=help_label_index + 1, value=line) + current_row += 1 + + return current_row + + +def read_column_header_titles(worksheet: Worksheet) -> List[str]: + """Read the first row of the worksheet and return a list of these column titles.""" + index = 1 + titles: List[str] = [] + while True: + value = worksheet.cell(1, index).value + if not value: + break + titles.append(value) + index += 1 + return titles diff --git a/lib/galaxy/tools/fetch/__init__.py b/lib/galaxy/tools/fetch/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/lib/galaxy/tools/fetch/workbooks.py b/lib/galaxy/tools/fetch/workbooks.py new file mode 100644 index 00000000000..254706789c5 --- /dev/null +++ b/lib/galaxy/tools/fetch/workbooks.py @@ -0,0 +1,466 @@ +from dataclasses import dataclass +from typing import ( + Dict, + List, + Optional, + Tuple, + Union, +) + +from openpyxl import Workbook +from openpyxl.worksheet.worksheet import Worksheet +from pydantic import ( + BaseModel, + Field, +) +from typing_extensions import Literal + +from galaxy.exceptions import RequestParameterInvalidException +from galaxy.model.dataset_collections.auto_identifiers import ( + fill_in_identifiers, + FillIdentifiers, +) +from galaxy.model.dataset_collections.auto_pairing import paired_element_list_identifier +from galaxy.model.dataset_collections.rule_target_columns import ( + column_titles_to_headers, + HeaderColumn, + implied_paired_or_unpaired_column_header, + ParsedColumn, +) +from galaxy.model.dataset_collections.rule_target_models import ( + COMMON_COLUMN_TARGETS, +) +from galaxy.model.dataset_collections.workbook_util import ( + add_column_validation, + add_extra_column_help_as_new_sheet, + add_instructions_to_sheet, + Base64StringT, + ExtraColumnsHelpConfiguration, + freeze_header_row, + HasHelp, + HelpConfiguration, + load_workbook_from_base64, + make_headers_bold, + read_column_header_titles, + set_column_width, + uri_data_validation, +) + +FetchWorkbookType = Literal["datasets", "collection", "collections"] +FetchWorkbookCollectionType = Literal["list", "list:paired", "list:list", "list:list:paired", "list:paired_or_unpaired"] + +DEFAULT_FETCH_WORKBOOK_TYPE: FetchWorkbookType = "datasets" +DEFAULT_FETCH_WORKBOOK_COLLECTION_TYPE: FetchWorkbookCollectionType = "list" +DEFAULT_WORKBOOK_TITLE: str = "Galaxy Data Import" + +INSTRUCTION_USE_THIS = "Use this spreadsheet to describe your datasets. For each dataset (i.e. each file), ensure all the labeled columns are specified and correct." +INSTRUCTION_EXTRA_COLUMNS = "Additional columns can be added if we you wish to specify additional metadata." # mention second tab when implemented +INSTRUCTION_ONCE_COMPLETE_DATASETS = ( + "Once data entry is complete, drop this file back into Galaxy to finish creating datasets for your inputs." +) +INSTRUCTION_ONCE_COMPLETE_COLLECTION = ( + "Once data entry is complete, drop this file back into Galaxy to finish creating a collection for your inputs." +) +INSTRUCTION_ONCE_COMPLETE_COLLECTIONS = ( + "Once data entry is complete, drop this file back into Galaxy to finish creating collections for your inputs." +) +INSTRUCTIONS_BY_TYPE: Dict[FetchWorkbookType, List[str]] = { + "datasets": [ + INSTRUCTION_USE_THIS, + INSTRUCTION_EXTRA_COLUMNS, + INSTRUCTION_ONCE_COMPLETE_DATASETS, + ], + "collection": [ + INSTRUCTION_USE_THIS, + INSTRUCTION_EXTRA_COLUMNS, + INSTRUCTION_ONCE_COMPLETE_COLLECTION, + ], + "collections": [ + INSTRUCTION_USE_THIS, + INSTRUCTION_EXTRA_COLUMNS, + INSTRUCTION_ONCE_COMPLETE_COLLECTIONS, + ], +} + +EXCEPTION_TOO_MANY_URI_COLUMNS = "Too many URI/URL columns in the supplied workbook." +EXCEPTION_NO_URIS_FOUND = "Failed to find any URI or URL like column in the supplied workbook." +EXTRA_COLUMN_INSTRUCTIONS = [ + "Extra metadata for the uploaded datasets can be specified by just adding columns with special headers to the first sheet of this workbook.", + "The columns can appear in order on the first sheet of the workbook and the names are fairly flexible - extra white space and casing do not matter.", + "The list of supported column metadata types appear in this help sheet to the left along with example column names Galaxy would support.", +] + + +@dataclass +class GenerateFetchWorkbookRequest: + type: FetchWorkbookType = DEFAULT_FETCH_WORKBOOK_TYPE + collection_type: FetchWorkbookCollectionType = DEFAULT_FETCH_WORKBOOK_COLLECTION_TYPE + title: str = DEFAULT_WORKBOOK_TITLE + + +WorkbookContentField: Base64StringT = Field( + ..., + title="Workbook Content (Base 64 encoded)", + description="The workbook content (the contents of the xlsx file) that have been base64 encoded.", +) + + +class ParseFetchWorkbook(BaseModel): + content: Base64StringT = WorkbookContentField + fill_identifiers: Optional[FillIdentifiers] = None + + +def header_column_to_parsed_column(header_column: HeaderColumn) -> "ParsedColumn": + return header_column.parsed_column + + +def generate(request: GenerateFetchWorkbookRequest) -> Workbook: + workbook = Workbook() + worksheet = workbook.active + worksheet.title = request.title + header_columns = _request_to_columns(request) + + header_titles = [h.title for h in header_columns] + worksheet.append(header_titles) + make_headers_bold(worksheet, header_titles) + freeze_header_row(worksheet) + for i, column in enumerate(header_columns): + set_column_width(worksheet, i, column.width) + + uri_data_validator = uri_data_validation("A") + add_column_validation("A", uri_data_validator, worksheet) + + columns_for_help = [HasHelp(c.title, c.help) for c in header_columns] + help_configuration = HelpConfiguration( + instructions=INSTRUCTIONS_BY_TYPE[request.type], + columns=columns_for_help, + text_width=50, + column_width=50, + ) + add_instructions_to_sheet( + worksheet, + help_configuration, + ) + extra_column_configuration = ExtraColumnsHelpConfiguration( + EXTRA_COLUMN_INSTRUCTIONS, text_width=50, column_targets=COMMON_COLUMN_TARGETS + ) + add_extra_column_help_as_new_sheet(workbook, extra_column_configuration) + return workbook + + +ParsedRow = Dict[str, Optional[str]] +ParsedRows = List[ParsedRow] + + +class ParseLogEntry(BaseModel): + message: str + + +class SplitUpPairedDataLogEntry(ParseLogEntry): + message: str + old_forward_column: ParsedColumn + old_reverse_column: ParsedColumn + new_paired_status_column: int + + +class InferredCollectionTypeLogEntry(ParseLogEntry): + message: str + from_columns: List[ParsedColumn] + + +ParseLog = List[ParseLogEntry] + + +class BaseParsedFetchWorkbook(BaseModel): + rows: ParsedRows + columns: List[ParsedColumn] + workbook_type: FetchWorkbookType + parse_log: ParseLog + + +class ParsedFetchWorkbookForDatasets(BaseParsedFetchWorkbook): + workbook_type: FetchWorkbookType = "datasets" + + +class ParsedFetchWorkbookForCollections(BaseParsedFetchWorkbook): + workbook_type: FetchWorkbookType = "collection" + collection_type: FetchWorkbookCollectionType + + +ParsedFetchWorkbook = Union[ParsedFetchWorkbookForDatasets, ParsedFetchWorkbookForCollections] + + +def parse(payload: ParseFetchWorkbook) -> ParsedFetchWorkbook: + parse_log: ParseLog = [] + workbook = load_workbook_from_base64(payload.content) + column_headers = _read_column_headers(workbook.active) + _validate_parsed_column_headers(column_headers) + raw_rows = _load_row_data(workbook, payload) + # the rule builder does require splitting the paired data in this way but it might + # be worth it to do it with an "initial rule" instead to demo how you'd do it + # with actual rule builder rules? Not sure. + rows, column_headers, split_data_log_entry = _split_paired_data_if_needed(raw_rows, column_headers) + if split_data_log_entry: + parse_log.append(split_data_log_entry) + columns = [ParsedColumn(title=c.title, type=c.type, type_index=c.type_index) for c in column_headers] + if _is_fetch_workbook_for_collections(column_headers): + collection_type, log_entry = _infer_fetch_workbook_collection_type(column_headers) + parse_log.append(log_entry) + assert collection_type in ["list", "list:paired", "list:list", "list:list:paired", "list:paired_or_unpaired"] + rows = _fill_in_identifier_column_if_needed(rows, column_headers, payload.fill_identifiers) + return ParsedFetchWorkbookForCollections( + collection_type=collection_type, rows=rows, columns=columns, parse_log=parse_log + ) + else: + return ParsedFetchWorkbookForDatasets(rows=rows, columns=columns, parse_log=parse_log) + + +def _validate_parsed_column_headers(column_headers: List[HeaderColumn]) -> None: + uri_like_columns = _uri_like_columns(column_headers) + if len(uri_like_columns) > 2: + raise RequestParameterInvalidException( + f"{EXCEPTION_TOO_MANY_URI_COLUMNS}. Relevant headers are {[c.title for c in uri_like_columns]}" + ) + if len(uri_like_columns) == 0: + raise RequestParameterInvalidException(EXCEPTION_NO_URIS_FOUND) + + +def _request_to_columns(request: GenerateFetchWorkbookRequest) -> List[HeaderColumn]: + if request.type == "datasets": + return [ + HeaderColumn("url", "URI", 0), + HeaderColumn("name", "Name", 0), + ] + else: + if request.collection_type == "list": + header_columns = [ + HeaderColumn("url", "URI", 0), + HeaderColumn("list_identifiers", "List Identifier", 0), + ] + elif request.collection_type == "list:paired": + header_columns = [ + HeaderColumn("url", "URI 1 (Forward)", 0), + HeaderColumn("url", "URI 2 (Reverse)", 1), + HeaderColumn("list_identifiers", "List Identifier", 0), + ] + elif request.collection_type == "list:list": + header_columns = [ + HeaderColumn("url", "URI", 0), + HeaderColumn("list_identifiers", "Outer List Identifier", 0), + HeaderColumn("list_identifiers", "Inner List Identifier", 1), + ] + elif request.collection_type == "list:list:paired": + header_columns = [ + HeaderColumn("url", "URI 1 (Forward)", 0), + HeaderColumn("url", "URI 2 (Reverse)", 1), + HeaderColumn("list_identifiers", "Outer List Identifier", 0), + HeaderColumn("list_identifiers", "Inner List Identifier", 1), + ] + elif request.collection_type == "list:paired_or_unpaired": + header_columns = [ + HeaderColumn("url", "URI 1 (Forward)", 0), + HeaderColumn("url", "URI 2 (Optional/Reverse)", 1), + HeaderColumn("list_identifiers", "List Identifier", 0), + ] + else: + raise NotImplementedError() + if request.type == "collections": + header_columns.append( + HeaderColumn( + type="collection_name", + title="Collection Name", + type_index=0, + ) + ) + return header_columns + + +def _load_row_data(workbook: Workbook, payload: ParseFetchWorkbook) -> ParsedRows: + sheet = workbook.active # Get the first sheet + + rows: ParsedRows = [] + + column_headers = _read_column_headers(sheet) + columns_to_read = len(column_headers) + index_of_first_uri = _index_of_fist_uri_column(column_headers) + + for row_index, row in enumerate(sheet.iter_rows(max_col=columns_to_read, values_only=True)): + if row_index == 0: # skip column headers + continue + if not row[index_of_first_uri]: + break + parsed_row: ParsedRow = {} + for value, column in zip(row, column_headers): + parsed_row[column.name] = value + rows.append(parsed_row) + return rows + + +def _split_paired_data_if_needed( + rows: ParsedRows, column_headers: List[HeaderColumn] +) -> Tuple[ParsedRows, List[HeaderColumn], Optional[SplitUpPairedDataLogEntry]]: + split_rows: ParsedRows = [] + uri_like_columns = _uri_like_columns(column_headers) + if len(_uri_like_columns(column_headers)) != 2: + return rows, column_headers, None + + hash_columns_to_split: List[Tuple[HeaderColumn, HeaderColumn]] = [] + for column_type in ["hash_sha1", "hash_md5", "hash_sha256", "hash_sha512"]: + hash_columns = [c for c in column_headers if c.type == column_type] + if len(hash_columns) == 2: + hash_columns_to_split.append((hash_columns[0], hash_columns[1])) + + url_column_0 = uri_like_columns[0] + url_column_1 = uri_like_columns[1] + + # Split the data up + split_rows = [] + for row in rows: + row_to_copy = row.copy() + + url_0 = row_to_copy.pop(url_column_0.name) + url_1 = row_to_copy.pop(url_column_1.name) + + # before we break up the pairs, do any implicit identifier filling in + list_identifier_columns = [c for c in column_headers if c.type == "list_identifiers"] + if len(list_identifier_columns) > 0: + inner_list_identifier_column = list_identifier_columns[-1] + list_identifier = row_to_copy[inner_list_identifier_column.name] + if not list_identifier and url_0 and url_1: + list_identifier = paired_element_list_identifier(url_0, url_1) + row_to_copy[inner_list_identifier_column.name] = list_identifier + + row_0 = row_to_copy.copy() + row_1 = row_to_copy.copy() + row_0[url_column_0.name] = url_0 + row_1[url_column_0.name] = url_1 + row_0["paired_identifier"] = "1" + row_1["paired_identifier"] = "2" + + for hash_column_0, hash_column_1 in hash_columns_to_split: + # remove all the hashes from copied row before we... + row_0.pop(hash_column_0.name, None) + row_1.pop(hash_column_0.name, None) + row_0.pop(hash_column_1.name, None) + row_1.pop(hash_column_1.name, None) + + # ... merge both hash columns in the original into one column in the duplicates + row_0[hash_column_0.name] = row.pop(hash_column_0.name) + row_1[hash_column_0.name] = row.pop(hash_column_1.name) + + split_rows.append(row_0) + split_rows.append(row_1) + + # Adjust the columns accordingly + column_headers.remove(url_column_1) + for _, hash_column_1 in hash_columns_to_split: + column_headers.remove(hash_column_1) + + column_headers.append( + HeaderColumn( + type="paired_identifier", + title="Paired Identifier", + type_index=0, + ) + ) + split_log_entry = SplitUpPairedDataLogEntry( + message="Merged paired data URIs columns into single column and added paired identifier column.", + old_forward_column=header_column_to_parsed_column(url_column_0), + old_reverse_column=header_column_to_parsed_column(url_column_1), + new_paired_status_column=len(column_headers) - 1, + ) + return split_rows, column_headers, split_log_entry + + +def _fill_in_identifier_column_if_needed( + rows: ParsedRows, columns: List[HeaderColumn], config: Optional[FillIdentifiers] +) -> ParsedRows: + list_identifiers_columns = [c for c in columns if c.type == "list_identifiers"] + uri_columns = _uri_like_columns(columns) + if len(uri_columns) == 0 or len(list_identifiers_columns) == 0: + # this doesn't look like a list creation, don't need identifiers to be filled in + return rows + if config is None or not config.fill_inner_list_identifiers: + # no filling in identifiers requested, just return the rows as is + return rows + + uri_column = uri_columns[0] + inner_list_identifier_column = list_identifiers_columns[-1] + uris_to_identifiers: List[Tuple[str, Optional[str]]] = [] + for row in rows: + uri = row.get(uri_column.name) + if not uri: + continue + list_identifier = row.get(inner_list_identifier_column.name) + uris_to_identifiers.append((uri, list_identifier)) + new_identifiers = fill_in_identifiers(uris_to_identifiers, config) + for row, new_identifier in zip(rows, new_identifiers): + row[inner_list_identifier_column.name] = new_identifier + return rows + + +def _read_column_headers(worksheet: Worksheet) -> List[HeaderColumn]: + column_titles = read_column_header_titles(worksheet) + return column_titles_to_headers(column_titles) + + +def _uri_like_columns(column_headers: List[HeaderColumn]) -> List[HeaderColumn]: + return [c for c in column_headers if c.type == "url" or c.type == "url_deferred"] + + +def _index_of_fist_uri_column(column_headers: List[HeaderColumn]) -> int: + for index, column_header in enumerate(column_headers): + if column_header.type == "url" or column_header.type == "url_deferred": + return index + raise RequestParameterInvalidException( + f"{EXCEPTION_NO_URIS_FOUND}. Relevant headers are {[c.title for c in column_headers]}" + ) + + +def _infer_fetch_workbook_collection_type( + column_headers: List[HeaderColumn], +) -> Tuple[str, InferredCollectionTypeLogEntry]: + paired_identifier_columns = [c for c in column_headers if c.type == "paired_identifier"] + paired_or_unpaired_identifier_columns = [c for c in column_headers if c.type == "paired_or_unpaired_identifier"] + any_paired = len(paired_identifier_columns) > 0 + uri_columns = _uri_like_columns(column_headers) + num_uris = len(uri_columns) + + inference_on_columns: List[ParsedColumn] = [] + + list_type: str = "" + for column_header in column_headers: + if column_header.type == "list_identifiers": + inference_on_columns.append(header_column_to_parsed_column(column_header)) + if list_type: + list_type = f"list:{list_type}" + else: + list_type = "list" + if any_paired or num_uris == 2: + if num_uris == 2: + inference_on_columns.append(header_column_to_parsed_column(uri_columns[0])) + inference_on_columns.append(header_column_to_parsed_column(uri_columns[1])) + if implied_paired_or_unpaired_column_header(uri_columns[1]): + collection_type = f"{list_type}:paired_or_unpaired" + else: + collection_type = f"{list_type}:paired" + else: + paired_identifier_column = paired_identifier_columns[0] + inference_on_columns.append(header_column_to_parsed_column(paired_identifier_column)) + if implied_paired_or_unpaired_column_header(paired_identifier_column): + collection_type = f"{list_type}:paired_or_unpaired" + else: + collection_type = f"{list_type}:paired" + elif len(paired_or_unpaired_identifier_columns) > 0: + inference_on_columns.append(header_column_to_parsed_column(paired_or_unpaired_identifier_columns[0])) + collection_type = f"{list_type}:paired_or_unpaired" + else: + collection_type = list_type + return collection_type, InferredCollectionTypeLogEntry( + message="Inferred collection type from column headers.", + from_columns=inference_on_columns, + ) + + +def _is_fetch_workbook_for_collections(column_headers: List[HeaderColumn]) -> bool: + return _infer_fetch_workbook_collection_type(column_headers)[0] != "" diff --git a/lib/galaxy/webapps/galaxy/api/common.py b/lib/galaxy/webapps/galaxy/api/common.py index 44d68626fc7..228d68871ec 100644 --- a/lib/galaxy/webapps/galaxy/api/common.py +++ b/lib/galaxy/webapps/galaxy/api/common.py @@ -1,5 +1,6 @@ """This module contains utility functions shared across the api package.""" +from io import BytesIO from typing import ( Any, List, @@ -13,6 +14,7 @@ from fastapi import ( Query, Request, ) +from starlette.responses import StreamingResponse from typing_extensions import Annotated from galaxy.schema import ( @@ -294,3 +296,12 @@ def query_parameter_as_list(query): return elements return parse_elements + + +def serve_workbook(content: BytesIO, filename: Optional[str]) -> StreamingResponse: + filename = filename or "galaxy_sample_sheet_workbook.xlsx" + return StreamingResponse( + content, + media_type="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + headers={"Content-Disposition": f"attachment; filename={filename}"}, + ) diff --git a/lib/galaxy/webapps/galaxy/api/tools.py b/lib/galaxy/webapps/galaxy/api/tools.py index e6dc8dd8ebd..c5c0b9c923d 100644 --- a/lib/galaxy/webapps/galaxy/api/tools.py +++ b/lib/galaxy/webapps/galaxy/api/tools.py @@ -16,12 +16,14 @@ from typing import ( from fastapi import ( Body, Depends, + Query, Request, Response, UploadFile, ) from fastapi.responses import FileResponse from starlette.datastructures import UploadFile as StarletteUploadFile +from starlette.responses import StreamingResponse from galaxy import ( exceptions, @@ -33,6 +35,7 @@ from galaxy.managers.collections import DatasetCollectionManager from galaxy.managers.context import ProvidesHistoryContext from galaxy.managers.hdas import HDAManager from galaxy.managers.histories import HistoryManager +from galaxy.model.dataset_collections.workbook_util import workbook_to_bytes from galaxy.schema.fetch_data import ( FetchDataFormPayload, FetchDataPayload, @@ -40,6 +43,15 @@ from galaxy.schema.fetch_data import ( from galaxy.tool_util.verify import ToolTestDescriptionDict from galaxy.tool_util_models import UserToolSource from galaxy.tools.evaluation import global_tool_errors +from galaxy.tools.fetch.workbooks import ( + FetchWorkbookCollectionType, + FetchWorkbookType, + generate, + GenerateFetchWorkbookRequest, + parse, + ParsedFetchWorkbook, + ParseFetchWorkbook, +) from galaxy.util.hash_util import ( HashFunctionNameEnum, memory_bound_hexdigest, @@ -53,6 +65,7 @@ from galaxy.web import ( ) from galaxy.webapps.base.controller import UsesVisualizationMixin from galaxy.webapps.base.webapp import GalaxyWebTransaction +from galaxy.webapps.galaxy.api.common import serve_workbook from galaxy.webapps.galaxy.services.tools import ToolsService from . import ( APIContentTypeRoute, @@ -86,6 +99,22 @@ class PNGIconResponse(FileResponse): media_type = "image/png" +FetchWorkbookTypeQueryParam: FetchWorkbookType = Query( + default="datasets", + title="Workbook Type", + description="Generate a workbook for simple datasets or a collection.", +) +FetchWorkbookCollectionTypeQueryParam: FetchWorkbookCollectionType = Query( + default="list", + title="Collection Type", + description="Generate workbook for specified collection type (not all collection types are supported)", +) +FetchWorkbookFilenameQueryParam: Optional[str] = Query( + None, + description="Filename of the workbook download to generate", +) + + router = Router(tags=["tools"]) FetchDataForm = as_form(FetchDataFormPayload) @@ -123,6 +152,39 @@ class FetchTools: ): return self.service.create_fetch(trans, payload, files) + @router.get( + "/api/tools/fetch/workbook", + summary="Generate a template workbook to use with the activity builder UI", + response_class=StreamingResponse, + operation_id="tools__fetch_workbook_download", + ) + def fetch_workbook( + self, + trans: ProvidesHistoryContext = DependsOnTrans, + type: FetchWorkbookType = FetchWorkbookTypeQueryParam, + collection_type: FetchWorkbookCollectionType = FetchWorkbookCollectionTypeQueryParam, + filename: Optional[str] = FetchWorkbookFilenameQueryParam, + ): + generate_request = GenerateFetchWorkbookRequest( + type=type, + collection_type=collection_type, + ) + workbook = generate(generate_request) + contents = workbook_to_bytes(workbook) + return serve_workbook(contents, filename) + + @router.post( + "/api/tools/fetch/workbook/parse", + summary="Generate a template workbook to use with the activity builder UI", + operation_id="tools__fetch_workbook_parse", + ) + def parse_workbook( + self, + trans: ProvidesHistoryContext = DependsOnTrans, + payload: ParseFetchWorkbook = Body(...), + ) -> ParsedFetchWorkbook: + return parse(payload) + @router.get( "/api/tools/{tool_id:path}/icon", summary="Get the icon image associated with a tool", diff --git a/lib/galaxy_test/api/test_tools.py b/lib/galaxy_test/api/test_tools.py index e974cf38e1c..64d0d4326c9 100644 --- a/lib/galaxy_test/api/test_tools.py +++ b/lib/galaxy_test/api/test_tools.py @@ -24,6 +24,7 @@ from galaxy.util.unittest_utils import skip_if_github_down from galaxy_test.base import rules_test_data from galaxy_test.base.api_asserts import ( assert_error_code_is, + assert_file_looks_like_xlsx, assert_has_keys, assert_status_code_is, ) @@ -413,6 +414,10 @@ class TestToolsApi(ApiTestCase, TestsTools): ) assert sanitized_address.strip() == cool_name_without_quote + def test_fetch_workbook_generate(self): + workbook_path = self.dataset_populator.download_fetch_workbook() + assert_file_looks_like_xlsx(workbook_path) + @skip_without_tool("composite_output") def test_test_data_filepath_security(self): test_data_response = self._get("tools/composite_output/test_data_path?filename=../CONTRIBUTORS.md", admin=True) diff --git a/lib/galaxy_test/base/api_asserts.py b/lib/galaxy_test/base/api_asserts.py index ec51a38844f..fcc3c5c8344 100644 --- a/lib/galaxy_test/base/api_asserts.py +++ b/lib/galaxy_test/base/api_asserts.py @@ -113,3 +113,11 @@ def _as_dict(response: Union[Response, dict]) -> Dict[str, Any]: assert_has_key = assert_has_keys + + +def assert_file_looks_like_xlsx(xlsx_file: str): + # Check the file header + with open(xlsx_file, "rb") as file: + header = file.read(4) + # The ZIP file signature is 0x50 0x4B 0x03 0x04 + return header == b"\x50\x4b\x03\x04" diff --git a/lib/galaxy_test/base/populators.py b/lib/galaxy_test/base/populators.py index 9f4e4d61991..c6d6f178e00 100644 --- a/lib/galaxy_test/base/populators.py +++ b/lib/galaxy_test/base/populators.py @@ -420,8 +420,11 @@ class BasePopulator(metaclass=ABCMeta): """Perform a _get and store the result in a tempfile.""" get_response = self._get(route, **kwd) get_response.raise_for_status() + return self._get_response_to_tempfile(get_response) + + def _get_response_to_tempfile(self, response, suffix=None) -> str: temp_file = tempfile.NamedTemporaryFile("wb", delete=False, suffix=suffix) - temp_file.write(get_response.content) + temp_file.write(response.content) temp_file.flush() return temp_file.name @@ -1108,6 +1111,12 @@ class BaseDatasetPopulator(BasePopulator): response.raise_for_status() return response.json() + def download_fetch_workbook(self) -> str: + url = "tools/fetch/workbook" + download_response = self._get(url) + api_asserts.assert_status_code_is_ok(download_response) + return self._get_response_to_tempfile(download_response) + def run_tool_payload(self, tool_id: Optional[str], inputs: dict, history_id: str, **kwds) -> dict: # Remove files_%d|file_data parameters from inputs dict and attach # as __files dictionary. diff --git a/packages/data/MANIFEST.in b/packages/data/MANIFEST.in index e08e1a3a820..7bf27d3257f 100644 --- a/packages/data/MANIFEST.in +++ b/packages/data/MANIFEST.in @@ -3,6 +3,7 @@ include galaxy/datatypes/set_metadata_tool.xml include galaxy/datatypes/converters/*.xml graft galaxy/datatypes/display_applications/configs/ graft galaxy/datatypes/display_applications/xsd/ +include galaxy/model/dataset_collections/rule_targets.yml include galaxy/model/migrations/alembic.ini recursive-include galaxy/model/migrations/alembic/ *.py include galaxy/model/migrations/README.md diff --git a/packages/data/setup.cfg b/packages/data/setup.cfg index f2a436f2f90..bc3b09764f4 100644 --- a/packages/data/setup.cfg +++ b/packages/data/setup.cfg @@ -49,6 +49,7 @@ install_requires = msal mrcfile numpy + openpyxl>=3.1.5 parsley pycryptodome pydantic[email]>=2.7.4 diff --git a/pyproject.toml b/pyproject.toml index 1e180d4a28c..52c9f499c58 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -62,6 +62,7 @@ dependencies = [ "nodeenv", "numpy>=1.26.0", # Python 3.12 support "numpy>=2.1.0 ; python_version>='3.10'", # Python 3.13 support + "openpyxl>=3.1.5", # Minimum version ever tested with "packaging", "paramiko!=2.9.0,!=2.9.1", # https://github.com/paramiko/paramiko/issues/1961 "Parsley", diff --git a/test/unit/app/tools/test_fetch_workbooks.py b/test/unit/app/tools/test_fetch_workbooks.py new file mode 100644 index 00000000000..794ed742bcc --- /dev/null +++ b/test/unit/app/tools/test_fetch_workbooks.py @@ -0,0 +1,314 @@ +import base64 +import os + +import pytest + +from galaxy.exceptions import RequestParameterInvalidException +from galaxy.model.dataset_collections.auto_identifiers import FillIdentifiers +from galaxy.model.dataset_collections.rule_target_columns import column_titles_to_headers +from galaxy.model.dataset_collections.rule_target_models import target_models +from galaxy.model.dataset_collections.workbook_util import read_column_header_titles +from galaxy.tools.fetch.workbooks import ( + _infer_fetch_workbook_collection_type, + _validate_parsed_column_headers, + DEFAULT_WORKBOOK_TITLE, + EXCEPTION_NO_URIS_FOUND, + EXCEPTION_TOO_MANY_URI_COLUMNS, + generate, + GenerateFetchWorkbookRequest, + parse, + ParseFetchWorkbook, +) +from galaxy.util.resources import resource_path + +WRITE_TEST_WORKBOOKS = False + + +def test_fetch_datasets_workbook(): + request = GenerateFetchWorkbookRequest() + workbook = generate(request) + assert workbook + worksheet = workbook.active + assert worksheet.title == DEFAULT_WORKBOOK_TITLE + assert worksheet.cell(1, 1).value == "URI" + + header_titles = read_column_header_titles(worksheet) + assert header_titles == ["URI", "Name"] + + if WRITE_TEST_WORKBOOKS: + path = "~/fetch_workbook.xlsx" + expanded_path = os.path.expanduser(path) + workbook.save(expanded_path) + + +def test_fetch_list_workbook(): + request = GenerateFetchWorkbookRequest( + type="collection", + collection_type="list", + ) + workbook = generate(request) + assert workbook + worksheet = workbook.active + + header_titles = read_column_header_titles(worksheet) + assert header_titles == ["URI", "List Identifier"] + + +def test_fetch_multiple_lists_workbook(): + request = GenerateFetchWorkbookRequest( + type="collections", + collection_type="list", + ) + workbook = generate(request) + assert workbook + worksheet = workbook.active + + header_titles = read_column_header_titles(worksheet) + assert header_titles == ["URI", "List Identifier", "Collection Name"] + + +def test_fetch_list_paired_workbook(): + request = GenerateFetchWorkbookRequest( + type="collection", + collection_type="list:paired", + ) + workbook = generate(request) + assert workbook + worksheet = workbook.active + + header_titles = read_column_header_titles(worksheet) + assert header_titles == ["URI 1 (Forward)", "URI 2 (Reverse)", "List Identifier"] + + if WRITE_TEST_WORKBOOKS: + path = "~/fetch_workbook_paired.xlsx" + expanded_path = os.path.expanduser(path) + workbook.save(expanded_path) + + +def test_fetch_list_paired_or_unpaired_workbook(): + request = GenerateFetchWorkbookRequest( + type="collection", + collection_type="list:paired_or_unpaired", + ) + workbook = generate(request) + assert workbook + worksheet = workbook.active + + header_titles = read_column_header_titles(worksheet) + assert header_titles == ["URI 1 (Forward)", "URI 2 (Optional/Reverse)", "List Identifier"] + + +def test_parse_datasets(): + content = unittest_file_to_base64("fetch_workbook.xlsx") + parse_request = ParseFetchWorkbook( + content=content, + ) + parsed = parse(parse_request) + assert len(parsed.rows) == 1 + row0 = parsed.rows[0] + assert row0["url"] == "https://raw.githubusercontent.com/galaxyproject/galaxy/dev/test-data/4.bed" + assert row0["name"] == "4.bed" + + assert len(parsed.columns) == 2 + + +def test_parse_paired_list(): + # workbook has URI 1 and URI 2 columns - make sure they are broken out and have a paired_indicator column + # for the rule builder. + content = unittest_file_to_base64("fetch_workbook_paired.xlsx") + parse_request = ParseFetchWorkbook( + content=content, + ) + parsed = parse(parse_request) + assert len(parsed.rows) == 2 + row0 = parsed.rows[0] + assert row0["url"] == "https://raw.githubusercontent.com/galaxyproject/galaxy/dev/test-data/1.bed" + assert row0["list_identifiers"] == "sample1" + assert row0["paired_identifier"] == "1" + + row1 = parsed.rows[1] + assert row1["url"] == "https://raw.githubusercontent.com/galaxyproject/galaxy/dev/test-data/4.bed" + assert row1["list_identifiers"] == "sample1" + assert row1["paired_identifier"] == "2" + + assert len(parsed.columns) == 3 + + +def test_parsed_paired_with_hashes(): + content = unittest_file_to_base64("fetch_workbook_paired_with_hashes.xlsx") + parse_request = ParseFetchWorkbook( + content=content, + ) + parsed = parse(parse_request) + assert len(parsed.rows) == 2 + + row0 = parsed.rows[0] + assert row0["url"] == "https://raw.githubusercontent.com/galaxyproject/galaxy/dev/test-data/4.bed" + assert row0["list_identifiers"] == "sample1" + assert row0["paired_identifier"] == "1" + assert row0["list_identifiers"] == "sample1" + assert row0["hash_md5"] == "37b59762b59fff860460522d271bc111" + assert ( + row0["hash_sha512"] + == "b83327251deae7e8c865948573d325a6657eaef10b274ba98d8ba6835f073f787c5621a4a9afd6db894f87bbd4299770f062369a3a39dea8b0263860559ff939" + ) + + row1 = parsed.rows[1] + assert row1["url"] == "https://raw.githubusercontent.com/galaxyproject/galaxy/dev/test-data/1.bed" + assert row1["list_identifiers"] == "sample1" + assert row1["paired_identifier"] == "2" + assert row1["hash_md5"] == "29e9dd693af0a946e67cd1861b987d13" + assert ( + row1["hash_sha512"] + == "9be59f9dc30bc2b35016dc45f5726a249baef2cdee2eb840665d3cdf4d0a0539c50eea1aa96bfd84f3a619782fc9321e5817e005a8a30c318d45c6759c9601bd" + ) + + +def test_parsed_list_with_auto_identifiers(): + content = unittest_file_to_base64("fetch_workbook_list_without_ids_example.xlsx") + parse_request = ParseFetchWorkbook( + content=content, fill_identifiers=FillIdentifiers(fill_inner_list_identifiers=True) + ) + parsed = parse(parse_request) + assert len(parsed.rows) == 2 + row0 = parsed.rows[0] + assert row0["url"] == "https://ftp.sra.ebi.ac.uk/vol1/fastq/DRR000/DRR000770/DRR000770.fastq.gz" + assert row0["list_identifiers"] == "DRR000770" + row1 = parsed.rows[1] + assert row1["url"] == "https://ftp.sra.ebi.ac.uk/vol1/fastq/DRR000/DRR000771/DRR000771.fastq.gz" + assert row1["list_identifiers"] == "DRR000771" + + +def test_parsed_list_of_pairs_with_auto_identifiers(): + content = unittest_file_to_base64("fetch_workbook_list_pairs_without_ids_example.xlsx") + parse_request = ParseFetchWorkbook( + content=content, fill_identifiers=FillIdentifiers(fill_inner_list_identifiers=True) + ) + parsed = parse(parse_request) + assert len(parsed.rows) == 4 + row0 = parsed.rows[0] + assert row0["url"] == "https://ftp.sra.ebi.ac.uk/vol1/fastq/DRR039/DRR039919/DRR039919_1.fastq.gz" + assert row0["list_identifiers"] == "DRR039919" + + +def test_read_column_headers_from_titles(): + # datasets... + column_headers = column_titles_to_headers(["URI", "Name", "Genome"]) + assert len(column_headers) == 3 + assert column_headers[0].type == "url" + assert column_headers[0].title == "URI" + assert column_headers[1].type == "name" + assert column_headers[1].title == "Name" + assert column_headers[2].type == "dbkey" + assert column_headers[2].title == "Genome" + + # simple list... + column_headers = column_titles_to_headers(["URI", "List Identifier"]) + assert len(column_headers) == 2 + assert column_headers[0].type == "url" + assert column_headers[0].title == "URI" + assert column_headers[1].type == "list_identifiers" + assert column_headers[1].title == "List Identifier" + + # paired list with list two URIs per row.... + column_headers = column_titles_to_headers(["URI 1 (Forward)", "URI 2 (Reverse)", "List Identifier"]) + assert len(column_headers) == 3 + assert column_headers[0].type == "url" + assert column_headers[0].title == "URI 1 (Forward)" + assert column_headers[0].type_index == 0 + + assert column_headers[1].type == "url" + assert column_headers[1].title == "URI 2 (Reverse)" + assert column_headers[1].type_index == 1 + + assert column_headers[2].type == "list_identifiers" + assert column_headers[2].title == "List Identifier" + + # paired list with paired identifier as a row... + column_headers = column_titles_to_headers(["URI", "List Identifier", "Paired Identifier"]) + assert len(column_headers) == 3 + assert column_headers[0].type == "url" + assert column_headers[0].title == "URI" + assert column_headers[0].type_index == 0 + + assert column_headers[2].type == "paired_identifier" + assert column_headers[2].title == "Paired Identifier" + assert column_headers[2].type_index == 0 + + # nested list + column_headers = column_titles_to_headers(["URI", "Outer List Identifier", "Inner List Identifier"]) + + assert len(column_headers) == 3 + assert column_headers[0].type == "url" + assert column_headers[0].title == "URI" + assert column_headers[0].type_index == 0 + + assert column_headers[1].type == "list_identifiers" + assert column_headers[1].title == "Outer List Identifier" + assert column_headers[1].type_index == 0 + + assert column_headers[2].type == "list_identifiers" + assert column_headers[2].title == "Inner List Identifier" + assert column_headers[2].type_index == 1 + + +def test_infer_fetch_workbook_collection_type(): + column_headers = column_titles_to_headers(["URI", "List Identifier", "Genome"]) + collection_type = _infer_fetch_workbook_collection_type(column_headers)[0] + assert collection_type == "list" + + column_headers = column_titles_to_headers(["URI", "List Identifier 1", "List Identifier 2", "Genome"]) + collection_type = _infer_fetch_workbook_collection_type(column_headers)[0] + assert collection_type == "list:list" + + column_headers = column_titles_to_headers(["URI", "List Identifier", "Paired Identifier", "Genome"]) + collection_type = _infer_fetch_workbook_collection_type(column_headers)[0] + assert collection_type == "list:paired" + + # probably more usable - two URI style list:paired + column_headers = column_titles_to_headers(["URI 1", "URI 2", "List Identifier", "Genome"]) + collection_type = _infer_fetch_workbook_collection_type(column_headers)[0] + assert collection_type == "list:paired" + + column_headers = column_titles_to_headers( + ["URI", "List Identifier 1", "List Identifier 2", "Paired Identifier", "Genome"] + ) + collection_type = _infer_fetch_workbook_collection_type(column_headers)[0] + assert collection_type == "list:list:paired" + + column_headers = column_titles_to_headers(["URI 1", "URI 2", "List Identifier 1", "List Identifier 2", "Genome"]) + collection_type = _infer_fetch_workbook_collection_type(column_headers)[0] + assert collection_type == "list:list:paired" + + # paired/unpaired sheets + column_headers = column_titles_to_headers(["URI 1", "URI 2 (Optional)", "List Identifier", "Genome"]) + collection_type = _infer_fetch_workbook_collection_type(column_headers)[0] + assert collection_type == "list:paired_or_unpaired" + + # paired/unpaired sheets + column_headers = column_titles_to_headers(["URI", "List Identifier", "Paired Identifier (Optional)", "Genome"]) + collection_type = _infer_fetch_workbook_collection_type(column_headers)[0] + assert collection_type == "list:paired_or_unpaired" + + +def test_column_target_model_parsing(): + target_models() + + +def test_validate_parsed_column_headers(): + headers = column_titles_to_headers(["URI 1", "URI 2", "URI 3"]) + with pytest.raises(RequestParameterInvalidException) as exception_info: + _validate_parsed_column_headers(headers) + assert EXCEPTION_TOO_MANY_URI_COLUMNS in str(exception_info.value) + + headers = column_titles_to_headers(["Name", "Paired Indicator"]) + with pytest.raises(RequestParameterInvalidException) as exception_info: + _validate_parsed_column_headers(headers) + assert EXCEPTION_NO_URIS_FOUND in str(exception_info.value) + + +def unittest_file_to_base64(filename: str) -> str: + path = resource_path("galaxy.app_unittest_utils", filename) + example_as_bytes = path.read_bytes() + content_base64 = base64.b64encode(example_as_bytes).decode("utf-8") + return content_base64 diff --git a/test/unit/data/dataset_collections/test_auto_identifiers.py b/test/unit/data/dataset_collections/test_auto_identifiers.py new file mode 100644 index 00000000000..90f5c7d177f --- /dev/null +++ b/test/unit/data/dataset_collections/test_auto_identifiers.py @@ -0,0 +1,44 @@ +from galaxy.model.dataset_collections.auto_identifiers import ( + fill_in_identifiers, + FillIdentifiers, +) + + +def test_fill_in_identifiers_simple(): + uris = { + "https://ftp.sra.ebi.ac.uk/vol1/fastq/DRR000/DRR000770/DRR000770.fastq.gz": None, + "https://ftp.sra.ebi.ac.uk/vol1/fastq/DRR000/DRR000771/DRR000771.fastq.gz": None, + } + new_map = run_fill_on_dict(uris) + assert new_map["https://ftp.sra.ebi.ac.uk/vol1/fastq/DRR000/DRR000770/DRR000770.fastq.gz"] == "DRR000770" + assert new_map["https://ftp.sra.ebi.ac.uk/vol1/fastq/DRR000/DRR000771/DRR000771.fastq.gz"] == "DRR000771" + + +def test_fill_in_identifiers_retains_existing_maps(): + uris = { + "https://ftp.sra.ebi.ac.uk/vol1/fastq/DRR000/DRR000770/DRR000770.fastq.gz": "abc", + "https://ftp.sra.ebi.ac.uk/vol1/fastq/DRR000/DRR000771/DRR000771.fastq.gz": None, + } + new_map = run_fill_on_dict(uris) + assert new_map["https://ftp.sra.ebi.ac.uk/vol1/fastq/DRR000/DRR000770/DRR000770.fastq.gz"] == "abc" + assert new_map["https://ftp.sra.ebi.ac.uk/vol1/fastq/DRR000/DRR000771/DRR000771.fastq.gz"] == "DRR000771" + + +def test_fill_in_identifiers_avoids_duplicates(): + uris = { + "https://ftp.sra.ebi.ac.uk/vol1/fastq/DRR000/DRR000770/DRR000770.fastq.gz": None, + "https://ftp.sra.ebi.ac.uk/vol1/fastq/DRR000/DRR000771/DRR000770.fastq.gz": None, + } + new_map = run_fill_on_dict(uris) + assert new_map["https://ftp.sra.ebi.ac.uk/vol1/fastq/DRR000/DRR000770/DRR000770.fastq.gz"] == "DRR000770" + assert new_map["https://ftp.sra.ebi.ac.uk/vol1/fastq/DRR000/DRR000771/DRR000770.fastq.gz"] == "DRR000770_1" + + +def run_fill_on_dict(uris): + config = FillIdentifiers(fill_inner_list_identifiers=True) + as_tuples = list(uris.items()) + new_ids = fill_in_identifiers(as_tuples, config) + new_uris = {} + for uri, identifier in zip(uris.keys(), new_ids): + new_uris[uri] = identifier + return new_uris diff --git a/test/unit/data/dataset_collections/test_auto_pairing.py b/test/unit/data/dataset_collections/test_auto_pairing.py new file mode 100644 index 00000000000..21298ecd7d5 --- /dev/null +++ b/test/unit/data/dataset_collections/test_auto_pairing.py @@ -0,0 +1,48 @@ +from dataclasses import dataclass +from typing import ( + Dict, + List, +) + +import yaml +from pydantic import ( + BaseModel, + RootModel, +) + +from galaxy.model.dataset_collections.auto_pairing import auto_pair +from galaxy.util.resources import resource_string + + +@dataclass +class MockDataset: + name: str + + +class PairedItem(BaseModel): + forward: str + reverse: str + + +class AutoPairingSpecItem(BaseModel): + doc: str + inputs: List[str] + paired: Dict[str, PairedItem] + + +AutoPairingSpecification = RootModel[List[AutoPairingSpecItem]] + + +def test_auto_pairing_specification(): + auto_pairing_specification = resource_string("galaxy.model.dataset_collections", "auto_pairing_spec.yml") + auto_pairing_specification_dicts = yaml.safe_load(auto_pairing_specification) + tests = AutoPairingSpecification.model_validate(auto_pairing_specification_dicts) + for test in tests.root: + inputs = [MockDataset(name) for name in test.inputs] + summary = auto_pair(inputs) + pairs = summary.paired + for name, expected_pair in test.paired.items(): + pair = next((p for p in pairs if p.name == name), None) + assert pair is not None, f"Pair {name} not found" + assert pair.forward.name == expected_pair.forward, f"Forward mismatch for pair {name}" + assert pair.reverse.name == expected_pair.reverse, f"Reverse mismatch for pair {name}" diff --git a/test/unit/data/dataset_collections/test_column_to_target_mapping.py b/test/unit/data/dataset_collections/test_column_to_target_mapping.py new file mode 100644 index 00000000000..cee9674258a --- /dev/null +++ b/test/unit/data/dataset_collections/test_column_to_target_mapping.py @@ -0,0 +1,21 @@ +"""Run rule_target_column_specification.yml defined tests using pytest.""" + +import yaml + +from galaxy.model.dataset_collections.rule_target_columns import column_title_to_target_type +from galaxy.util.resources import resource_string + + +def test_column_to_target_mapping(): + rule_target_tests_str = resource_string("galaxy.model.dataset_collections", "rule_target_column_specification.yml") + rule_target_tests = yaml.safe_load(rule_target_tests_str) + for rule_target_test in rule_target_tests: + # - doc: "NAME (all uppercase) maps directly to the name target type" + # column_header: "NAME" + # maps_to: "name" + doc = rule_target_test.get("doc", "") + column_header = rule_target_test["column_header"] + maps_to = rule_target_test["maps_to"] + assert ( + column_title_to_target_type(column_header) == maps_to + ), f"Failed for column header '{column_header}' with doc '{doc}'" diff --git a/test/unit/data/dataset_collections/test_workbook_util.py b/test/unit/data/dataset_collections/test_workbook_util.py new file mode 100644 index 00000000000..042bcb497d3 --- /dev/null +++ b/test/unit/data/dataset_collections/test_workbook_util.py @@ -0,0 +1,24 @@ +import base64 + +import pytest + +from galaxy.exceptions import RequestParameterInvalidException +from galaxy.model.dataset_collections.workbook_util import ( + index_to_excel_column, + load_workbook_from_base64, +) + + +def test_index_to_excel_column(): + assert index_to_excel_column(0) == "A" + assert index_to_excel_column(25) == "Z" + assert index_to_excel_column(26) == "AA" + assert index_to_excel_column(700) == "ZY" + assert index_to_excel_column(701) == "ZZ" + assert index_to_excel_column(702) == "AAA" + + +def test_loading_invalid_workbook(): + base64Tabular = base64.b64encode(b"1\t2\t3\n").decode("utf-8") + with pytest.raises(RequestParameterInvalidException): + load_workbook_from_base64(base64Tabular) From 3de506375d2d0938fd9cad3fea9522c65a9ca046 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Wed, 14 May 2025 14:11:59 -0400 Subject: [PATCH 14/19] Rebuild schema for fetch workbook APIs. --- client/src/api/schema/schema.ts | 228 ++++++++++++++++++++++++++++++++ 1 file changed, 228 insertions(+) diff --git a/client/src/api/schema/schema.ts b/client/src/api/schema/schema.ts index 5e8b5a76e86..15acbe4b458 100644 --- a/client/src/api/schema/schema.ts +++ b/client/src/api/schema/schema.ts @@ -4626,6 +4626,40 @@ export interface paths { patch?: never; trace?: never; }; + "/api/tools/fetch/workbook": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** Generate a template workbook to use with the activity builder UI */ + get: operations["tools__fetch_workbook_download"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/tools/fetch/workbook/parse": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + /** Generate a template workbook to use with the activity builder UI */ + post: operations["tools__fetch_workbook_parse"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/api/tools/{tool_id}/icon": { parameters: { query?: never; @@ -10780,6 +10814,24 @@ export interface components { */ sorting: boolean; }; + /** FillIdentifiers */ + FillIdentifiers: { + /** + * Deduplication Index From + * @default 1 + */ + deduplication_index_from: number; + /** + * Deduplication Pattern + * @default _{#} + */ + deduplication_pattern: string; + /** + * Fill Inner List Identifiers + * @default false + */ + fill_inner_list_identifiers: boolean; + }; /** FillStepDefaultsAction */ FillStepDefaultsAction: { /** @@ -16885,6 +16937,89 @@ export interface components { * @default [] */ PageSummaryList: components["schemas"]["PageSummary"][]; + /** ParseFetchWorkbook */ + ParseFetchWorkbook: { + /** + * Workbook Content (Base 64 encoded) + * @description The workbook content (the contents of the xlsx file) that have been base64 encoded. + */ + content: string; + fill_identifiers?: components["schemas"]["FillIdentifiers"] | null; + }; + /** ParseLogEntry */ + ParseLogEntry: { + /** Message */ + message: string; + }; + /** ParsedColumn */ + ParsedColumn: { + /** Title */ + title: string; + /** + * Type + * @enum {string} + */ + type: + | "list_identifiers" + | "paired_identifier" + | "paired_or_unpaired_identifier" + | "collection_name" + | "name_tag" + | "tags" + | "group_tags" + | "name" + | "dbkey" + | "hash_sha1" + | "hash_md5" + | "hash_sha256" + | "hash_sha512" + | "file_type" + | "url" + | "url_deferred" + | "info" + | "ftp_path"; + /** Type Index */ + type_index: number; + }; + /** ParsedFetchWorkbookForCollections */ + ParsedFetchWorkbookForCollections: { + /** + * Collection Type + * @enum {string} + */ + collection_type: "list" | "list:paired" | "list:list" | "list:list:paired" | "list:paired_or_unpaired"; + /** Columns */ + columns: components["schemas"]["ParsedColumn"][]; + /** Parse Log */ + parse_log: components["schemas"]["ParseLogEntry"][]; + /** Rows */ + rows: { + [key: string]: string | null; + }[]; + /** + * Workbook Type + * @default collection + * @enum {string} + */ + workbook_type: "datasets" | "collection" | "collections"; + }; + /** ParsedFetchWorkbookForDatasets */ + ParsedFetchWorkbookForDatasets: { + /** Columns */ + columns: components["schemas"]["ParsedColumn"][]; + /** Parse Log */ + parse_log: components["schemas"]["ParseLogEntry"][]; + /** Rows */ + rows: { + [key: string]: string | null; + }[]; + /** + * Workbook Type + * @default datasets + * @enum {string} + */ + workbook_type: "datasets" | "collection" | "collections"; + }; /** PastedDataElement */ PastedDataElement: { /** Md5 */ @@ -36422,6 +36557,99 @@ export interface operations { }; }; }; + tools__fetch_workbook_download: { + parameters: { + query?: { + /** @description Generate a workbook for simple datasets or a collection. */ + type?: "datasets" | "collection" | "collections"; + /** @description Generate workbook for specified collection type (not all collection types are supported) */ + collection_type?: "list" | "list:paired" | "list:list" | "list:list:paired" | "list:paired_or_unpaired"; + /** @description Filename of the workbook download to generate */ + filename?: string | null; + }; + header?: { + /** @description The user ID that will be used to effectively make this API call. Only admins and designated users can make API calls on behalf of other users. */ + "run-as"?: string | null; + }; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + /** @description Request Error */ + "4XX": { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["MessageExceptionModel"]; + }; + }; + /** @description Server Error */ + "5XX": { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["MessageExceptionModel"]; + }; + }; + }; + }; + tools__fetch_workbook_parse: { + parameters: { + query?: never; + header?: { + /** @description The user ID that will be used to effectively make this API call. Only admins and designated users can make API calls on behalf of other users. */ + "run-as"?: string | null; + }; + path?: never; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["ParseFetchWorkbook"]; + }; + }; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": + | components["schemas"]["ParsedFetchWorkbookForDatasets"] + | components["schemas"]["ParsedFetchWorkbookForCollections"]; + }; + }; + /** @description Request Error */ + "4XX": { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["MessageExceptionModel"]; + }; + }; + /** @description Server Error */ + "5XX": { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["MessageExceptionModel"]; + }; + }; + }; + }; get_icon_api_tools__tool_id__icon_get: { parameters: { query?: never; From 3db2522609a2deebab6d4fa8ed4223c9c34303f1 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Thu, 19 Jun 2025 21:54:04 -0400 Subject: [PATCH 15/19] Move validInput trigger from mounted to created in RuleCollectionBuilder. --- client/src/components/RuleCollectionBuilder.vue | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/client/src/components/RuleCollectionBuilder.vue b/client/src/components/RuleCollectionBuilder.vue index 78b880116e8..bd25da70950 100644 --- a/client/src/components/RuleCollectionBuilder.vue +++ b/client/src/components/RuleCollectionBuilder.vue @@ -1292,6 +1292,11 @@ export default { console.log("Error in RuleCollectionBuilder, unable to load genomes", err); }); } + // is this comparable to watch immediate in newer Vue code?, I just need that event to + // to flair if it is initially okay also. + if (this.validInput) { + this.$emit("validInput", true); + } }, mounted() { // something bizarre is up with the rendering of hands-on-table, needs a click to render. @@ -1301,11 +1306,6 @@ export default { this.$refs.hotTable.$el.click(); }, 200); } - // is this comparable to watch immediate in newer Vue code?, I just need that event to - // to flair if it is initially okay also. - if (this.validInput) { - this.$emit("validInput", true); - } }, methods: { restoreRules(event) { From 2bee911ce473e381c5ffc856a811f4ed9eb68670 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Thu, 19 Jun 2025 22:29:30 -0400 Subject: [PATCH 16/19] Add max length to pydantic models - not sure this makes sense. --- lib/galaxy/tools/fetch/workbooks.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/galaxy/tools/fetch/workbooks.py b/lib/galaxy/tools/fetch/workbooks.py index 254706789c5..66f2672bfa5 100644 --- a/lib/galaxy/tools/fetch/workbooks.py +++ b/lib/galaxy/tools/fetch/workbooks.py @@ -100,6 +100,7 @@ class GenerateFetchWorkbookRequest: WorkbookContentField: Base64StringT = Field( ..., + max_length=1024 * 1024 * 10, # 10MB title="Workbook Content (Base 64 encoded)", description="The workbook content (the contents of the xlsx file) that have been base64 encoded.", ) From b644145d7e57c7f6f7e17978e593cb326d2742b3 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Thu, 19 Jun 2025 22:45:53 -0400 Subject: [PATCH 17/19] Improved error handling based on PR review. --- client/src/components/Collections/common/useWorkbooks.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/client/src/components/Collections/common/useWorkbooks.ts b/client/src/components/Collections/common/useWorkbooks.ts index 86043e9b352..2ad791d96a7 100644 --- a/client/src/components/Collections/common/useWorkbooks.ts +++ b/client/src/components/Collections/common/useWorkbooks.ts @@ -2,6 +2,8 @@ import { faUpload } from "@fortawesome/free-solid-svg-icons"; import { FontAwesomeIcon } from "@fortawesome/vue-fontawesome"; import { computed, ref } from "vue"; +import { errorMessageAsString } from "@/utils/simple-error"; + import HiddenWorkbookUploadInput from "@/components/Collections/wizard/HiddenWorkbookUploadInput.vue"; export const fileToBase64 = (file: File): Promise => @@ -57,7 +59,7 @@ export function useWorkbookDropHandling(workbookHandler: WorkbookHandler) { await workbookHandler(base64Content); } catch (error) { console.error("Error uploading file:", error); - uploadErrorMessage.value = "There was an error processing the file."; + uploadErrorMessage.value = "Error processing the file: " + errorMessageAsString(error); } finally { isProcessingUpload.value = false; } From 472008ee1a35541d2098f1c268b65b348c96a2c1 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Thu, 19 Jun 2025 22:47:05 -0400 Subject: [PATCH 18/19] Typo fix from PR review. --- client/src/components/Collections/wizard/SourceFromWorkbook.vue | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/src/components/Collections/wizard/SourceFromWorkbook.vue b/client/src/components/Collections/wizard/SourceFromWorkbook.vue index 6c2daa1b90e..c9e2d918e4e 100644 --- a/client/src/components/Collections/wizard/SourceFromWorkbook.vue +++ b/client/src/components/Collections/wizard/SourceFromWorkbook.vue @@ -34,7 +34,7 @@ const whatText = computed(() => { External Workbook
    - This option le ts you fill in URLs/URIs in an external workbook using tools like Excel or Google Sheets and + This option lets you fill in URLs/URIs in an external workbook using tools like Excel or Google Sheets and specify {{ whatText }} in the workbook also.
    From 7bc3afe4ee994c66f44f311d866aea5c5766dfdd Mon Sep 17 00:00:00 2001 From: John Chilton Date: Thu, 19 Jun 2025 22:46:22 -0400 Subject: [PATCH 19/19] Prevent client from sending extra large XLSX files to Galaxy. --- .../src/components/Collections/common/useWorkbooks.ts | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/client/src/components/Collections/common/useWorkbooks.ts b/client/src/components/Collections/common/useWorkbooks.ts index 2ad791d96a7..a4b57535574 100644 --- a/client/src/components/Collections/common/useWorkbooks.ts +++ b/client/src/components/Collections/common/useWorkbooks.ts @@ -25,6 +25,11 @@ export function useWorkbookDropHandling(workbookHandler: WorkbookHandler) { const input = event.target as HTMLInputElement; const file = input.files?.[0] ?? null; if (file) { + if (file.size > 10 * 1024 * 1024) { + // Limit to 10MB + uploadErrorMessage.value = "File size exceeds 10MB limit."; + return; + } const base64Content = await readAsBase64(file); workbookHandler(base64Content); } @@ -50,6 +55,11 @@ export function useWorkbookDropHandling(workbookHandler: WorkbookHandler) { if (!file) { return; } + if (file.size > 10 * 1024 * 1024) { + // Limit to 10MB + uploadErrorMessage.value = "File size exceeds 10MB limit."; + return; + } isDragging.value = false; isProcessingUpload.value = true;