From 9cae33ffa8bd7cdb700ede687d0ec178a6937926 Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Tue, 26 Nov 2024 17:14:35 +0100 Subject: [PATCH 01/21] only match filters that are descendants or data/collection otherwise action filters are also checked which results in an error since there is not text content. --- lib/galaxy/tool_util/linters/output.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/tool_util/linters/output.py b/lib/galaxy/tool_util/linters/output.py index e9ab7c4aaf4..e42c797dca0 100644 --- a/lib/galaxy/tool_util/linters/output.py +++ b/lib/galaxy/tool_util/linters/output.py @@ -83,7 +83,7 @@ class OutputsFilterExpression(Linter): tool_xml = getattr(tool_source, "xml_tree", None) if not tool_xml: return - for filter in tool_xml.findall("./outputs//filter"): + for filter in tool_xml.findall("./outputs/*/filter"): try: ast.parse(filter.text, mode="eval") except Exception as e: From a587336eddf9c4fc19a349adf393a4747537c4b2 Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Tue, 26 Nov 2024 17:18:44 +0100 Subject: [PATCH 02/21] strip text otherwise the parser gives an indentation error also: we strip the text alo in the application --- lib/galaxy/tool_util/linters/output.py | 2 +- test/unit/tool_util/test_tool_linters.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/tool_util/linters/output.py b/lib/galaxy/tool_util/linters/output.py index e42c797dca0..b1d48fd2a60 100644 --- a/lib/galaxy/tool_util/linters/output.py +++ b/lib/galaxy/tool_util/linters/output.py @@ -85,7 +85,7 @@ class OutputsFilterExpression(Linter): return for filter in tool_xml.findall("./outputs/*/filter"): try: - ast.parse(filter.text, mode="eval") + ast.parse(filter.text.strip(), mode="eval") except Exception as e: lint_ctx.warn( f"Filter '{filter.text}' is no valid expression: {str(e)}", diff --git a/test/unit/tool_util/test_tool_linters.py b/test/unit/tool_util/test_tool_linters.py index ea7f4cd8d5d..867e85dc2ba 100644 --- a/test/unit/tool_util/test_tool_linters.py +++ b/test/unit/tool_util/test_tool_linters.py @@ -656,7 +656,7 @@ OUTPUTS_FILTER_EXPRESSION = """ an invalid condition - an and condition + an and condition another invalid condition From a3c1d272005703a44a3d3c79605dfbba570d0fc3 Mon Sep 17 00:00:00 2001 From: guerler Date: Mon, 2 Dec 2024 20:06:32 +0300 Subject: [PATCH 03/21] Update visualization package versions --- config/plugins/visualizations/cytoscape/config/cytoscape.xml | 2 +- config/plugins/visualizations/ngl/config/ngl.xml | 2 +- config/plugins/visualizations/openlayers/config/openlayers.xml | 2 +- config/plugins/visualizations/venn/config/venn.xml | 2 +- config/plugins/visualizations/vizarr/config/vizarr.xml | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/config/plugins/visualizations/cytoscape/config/cytoscape.xml b/config/plugins/visualizations/cytoscape/config/cytoscape.xml index 9cd91fdeb45..79de685142f 100644 --- a/config/plugins/visualizations/cytoscape/config/cytoscape.xml +++ b/config/plugins/visualizations/cytoscape/config/cytoscape.xml @@ -14,7 +14,7 @@ dataset_id - + diff --git a/config/plugins/visualizations/ngl/config/ngl.xml b/config/plugins/visualizations/ngl/config/ngl.xml index 817f94e217c..18607ed2c42 100644 --- a/config/plugins/visualizations/ngl/config/ngl.xml +++ b/config/plugins/visualizations/ngl/config/ngl.xml @@ -13,7 +13,7 @@ dataset_id - + diff --git a/config/plugins/visualizations/openlayers/config/openlayers.xml b/config/plugins/visualizations/openlayers/config/openlayers.xml index 71c36fb1fe1..2fb0ae6ca42 100644 --- a/config/plugins/visualizations/openlayers/config/openlayers.xml +++ b/config/plugins/visualizations/openlayers/config/openlayers.xml @@ -14,7 +14,7 @@ dataset_id - + diff --git a/config/plugins/visualizations/venn/config/venn.xml b/config/plugins/visualizations/venn/config/venn.xml index 25e140440fa..1e9140135ea 100644 --- a/config/plugins/visualizations/venn/config/venn.xml +++ b/config/plugins/visualizations/venn/config/venn.xml @@ -14,7 +14,7 @@ dataset_id - + diff --git a/config/plugins/visualizations/vizarr/config/vizarr.xml b/config/plugins/visualizations/vizarr/config/vizarr.xml index 5b4b20ffe1a..a6bc313c20d 100644 --- a/config/plugins/visualizations/vizarr/config/vizarr.xml +++ b/config/plugins/visualizations/vizarr/config/vizarr.xml @@ -3,7 +3,7 @@ Basic visualization for Zarr-based images like OME-Zarr - + From 91660b9d5eff9320d2faaa36d49aff859f827bba Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Tue, 10 Dec 2024 18:54:07 +0100 Subject: [PATCH 04/21] Fix target_type in auto-conversion context which means we collect and store the converted dataset as the job input. We don't need to wait for the galaxy.json collection, we know the exact target type already. That fixes the retrieval in `get_converted_files_by_type`. I believe this was always the intention and is how it works if the dataset already exists (i.e. on a re-run), and for all converters that don't use galaxy.json The converter records the dataset that the user chose, so there's no gap in the provenance either. This somewhat addresses https://github.com/galaxyproject/total-perspective-vortex/issues/141 so you can (reliably) differetiate your rules on the input datatype and filesize combination. --- lib/galaxy/datatypes/data.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/galaxy/datatypes/data.py b/lib/galaxy/datatypes/data.py index 05f5adbad6b..f5d15fa406a 100644 --- a/lib/galaxy/datatypes/data.py +++ b/lib/galaxy/datatypes/data.py @@ -847,7 +847,11 @@ class Data(metaclass=DataMeta): job, converted_datasets, *_ = converter.execute( trans, incoming=params, set_output_hid=visible, history=history, flush_job=False ) + # We should only have a single converted output, but let's be defensive here + n_converted_datasets = len(converted_datasets) for converted_dataset in converted_datasets.values(): + if converted_dataset.extension == "auto" and n_converted_datasets == 1: + converted_dataset.extension = target_type original_dataset.attach_implicitly_converted_dataset(trans.sa_session, converted_dataset, target_type) trans.app.job_manager.enqueue(job, tool=converter) if len(params) > 0: From a75bca59b45777b6df4769791bab6badc2267bb6 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Wed, 11 Dec 2024 12:18:14 +0100 Subject: [PATCH 05/21] Add test --- lib/galaxy_test/api/test_tools.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/lib/galaxy_test/api/test_tools.py b/lib/galaxy_test/api/test_tools.py index 56fea13f828..4547e7d1331 100644 --- a/lib/galaxy_test/api/test_tools.py +++ b/lib/galaxy_test/api/test_tools.py @@ -2511,6 +2511,24 @@ class TestToolsApi(ApiTestCase, TestsTools): exception_raised = e assert exception_raised, "Expected invalid column selection to fail job" + @skip_without_tool("implicit_conversion_format_input") + def test_implicit_conversion_input_dataset_tracking(self): + with self.dataset_populator.test_history() as history_id: + compressed_path = self.test_data_resolver.get_filename("1.fastqsanger.gz") + with open(compressed_path, "rb") as fh: + dataset = self.dataset_populator.new_dataset( + history_id, content=fh, file_type="fastqsanger.gz", wait=True + ) + outputs = self._run( + "Grep1", history_id=history_id, inputs={"data": {"src": "hda", "id": dataset["id"]}}, assert_ok=True + ) + job_details = self.dataset_populator.get_job_details(outputs["jobs"][0]["id"], full=True).json() + assert job_details["inputs"]["input"]["id"] != dataset["id"] + converted_input = self.dataset_populator.get_history_dataset_details( + history_id=history_id, content_id=job_details["inputs"]["input"]["id"] + ) + assert converted_input["extension"] == "fastqsanger" + @skip_without_tool("column_multi_param") def test_implicit_conversion_and_reduce(self): with self.dataset_populator.test_history() as history_id: From 5229449c95039d7e5e92a16a2017dec5f3317af6 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Sun, 22 Dec 2024 12:43:00 -0500 Subject: [PATCH 06/21] Refactor collection name input out of CollectionCreator. --- .../Collections/common/CollectionCreator.vue | 20 ++------ .../common/CollectionNameInput.vue | 47 +++++++++++++++++++ 2 files changed, 52 insertions(+), 15 deletions(-) create mode 100644 client/src/components/Collections/common/CollectionNameInput.vue diff --git a/client/src/components/Collections/common/CollectionCreator.vue b/client/src/components/Collections/common/CollectionCreator.vue index c60e80099fd..2fd13b20517 100644 --- a/client/src/components/Collections/common/CollectionCreator.vue +++ b/client/src/components/Collections/common/CollectionCreator.vue @@ -1,7 +1,7 @@ + + From e6742ed4e168286355efceb0de701b72e62d2566 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Sun, 22 Dec 2024 12:56:31 -0500 Subject: [PATCH 07/21] Refactor the source options out of collection creator. --- .../Collections/common/CollectionCreator.vue | 30 ++++------ .../common/CollectionCreatorSourceOptions.vue | 56 +++++++++++++++++++ 2 files changed, 67 insertions(+), 19 deletions(-) create mode 100644 client/src/components/Collections/common/CollectionCreatorSourceOptions.vue diff --git a/client/src/components/Collections/common/CollectionCreator.vue b/client/src/components/Collections/common/CollectionCreator.vue index 2fd13b20517..b0331452916 100644 --- a/client/src/components/Collections/common/CollectionCreator.vue +++ b/client/src/components/Collections/common/CollectionCreator.vue @@ -1,7 +1,7 @@ + + From 40eb1d9aab5126d3e74360d432551166357a0d03 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Sun, 22 Dec 2024 13:32:05 -0500 Subject: [PATCH 08/21] Refactor collection creator help out to simplify. --- .../Collections/common/CollectionCreator.vue | 45 ++--------------- .../common/CollectionCreatorHelpHeader.vue | 50 +++++++++++++++++++ 2 files changed, 55 insertions(+), 40 deletions(-) create mode 100644 client/src/components/Collections/common/CollectionCreatorHelpHeader.vue diff --git a/client/src/components/Collections/common/CollectionCreator.vue b/client/src/components/Collections/common/CollectionCreator.vue index b0331452916..b076d4f3b81 100644 --- a/client/src/components/Collections/common/CollectionCreator.vue +++ b/client/src/components/Collections/common/CollectionCreator.vue @@ -1,5 +1,5 @@ + + From df0336185277d4c0b3c7d604c7bf6d0c91c4ee5d Mon Sep 17 00:00:00 2001 From: John Chilton Date: Sun, 22 Dec 2024 14:05:25 -0500 Subject: [PATCH 09/21] Refactor showing of extensions into own component. --- .../Collections/common/CollectionCreator.vue | 10 ++------ .../CollectionCreatorShowExtensions.vue | 25 +++++++++++++++++++ 2 files changed, 27 insertions(+), 8 deletions(-) create mode 100644 client/src/components/Collections/common/CollectionCreatorShowExtensions.vue diff --git a/client/src/components/Collections/common/CollectionCreator.vue b/client/src/components/Collections/common/CollectionCreator.vue index b076d4f3b81..81c1eea4191 100644 --- a/client/src/components/Collections/common/CollectionCreator.vue +++ b/client/src/components/Collections/common/CollectionCreator.vue @@ -14,6 +14,7 @@ import localize from "@/utils/localization"; import { orList } from "@/utils/strings"; import CollectionCreatorHelpHeader from "./CollectionCreatorHelpHeader.vue"; +import CollectionCreatorShowExtensions from "./CollectionCreatorShowExtensions.vue"; import CollectionCreatorSourceOptions from "./CollectionCreatorSourceOptions.vue"; import CollectionNameInput from "./CollectionNameInput.vue"; import HelpText from "@/components/Help/HelpText.vue"; @@ -170,14 +171,7 @@ watch(