From b83647c050d66c5b962650ec76b57d4062fc41f5 Mon Sep 17 00:00:00 2001 From: nekrut Date: Thu, 26 Feb 2026 13:23:59 +0000 Subject: [PATCH 01/15] first pass in unifying and clarifying collection operation interfaces and help sections Co-Authored-By: Claude Opus 4.6 --- lib/galaxy/tools/apply_rules.xml | 8 +++- lib/galaxy/tools/build_list_1.2.0.xml | 14 +++--- lib/galaxy/tools/cross_product_flat.xml | 2 +- lib/galaxy/tools/cross_product_nested.xml | 2 +- .../tools/duplicate_file_to_collection.xml | 10 +++- lib/galaxy/tools/extract_dataset.xml | 15 ++++-- .../tools/filter_empty_collection_1.1.0.xml | 19 ++++++-- .../tools/filter_failed_collection_1.1.0.xml | 13 ++++-- lib/galaxy/tools/filter_from_file_1.1.0.xml | 11 +++-- lib/galaxy/tools/filter_null_1.1.0.xml | 19 ++++++-- lib/galaxy/tools/flatten_collection.xml | 6 +-- .../tools/harmonize_two_collections_list.xml | 40 ++++++++++------ .../tools/keep_success_collection_1.1.0.xml | 17 ++++--- lib/galaxy/tools/merge_collection.xml | 46 ++++++++++--------- lib/galaxy/tools/nest_collection.xml | 10 ++-- lib/galaxy/tools/relabel_from_file.xml | 12 +++-- lib/galaxy/tools/sample_sheet_to_tabular.xml | 16 ++++--- lib/galaxy/tools/sort_collection_list.xml | 17 +++++-- .../tools/split_paired_and_unpaired.xml | 29 +++++++++++- lib/galaxy/tools/tag_collection_from_file.xml | 17 +++++-- lib/galaxy/tools/unzip_collection.xml | 10 ++-- lib/galaxy/tools/zip_collection.xml | 11 +++-- 22 files changed, 232 insertions(+), 112 deletions(-) diff --git a/lib/galaxy/tools/apply_rules.xml b/lib/galaxy/tools/apply_rules.xml index 1df0063d770..0f158e36041 100644 --- a/lib/galaxy/tools/apply_rules.xml +++ b/lib/galaxy/tools/apply_rules.xml @@ -2,7 +2,11 @@ name="Apply rules" version="1.1.0" tool_type="apply_rules_to_collection"> + to reorganize a collection using the rule builder + + model_operation_macros.xml + operation_2409 operation_3695 @@ -40,11 +44,11 @@ More information about the rule processor in general can be found at `our traini .. _our training site: https://training.galaxyproject.org/training-material/search?query=rule+builder - +----- .. class:: infomark -This tool will create new history datasets from your collection but your quota usage will not increase. +@QUOTA_USAGE_NOTE@ ]]> diff --git a/lib/galaxy/tools/build_list_1.2.0.xml b/lib/galaxy/tools/build_list_1.2.0.xml index 457ae2b471d..7564ec121b5 100644 --- a/lib/galaxy/tools/build_list_1.2.0.xml +++ b/lib/galaxy/tools/build_list_1.2.0.xml @@ -2,8 +2,11 @@ name="Build list" version="1.2.0" tool_type="build_list"> - + from individual datasets or existing collections + + model_operation_macros.xml + @@ -21,7 +24,7 @@ - + @@ -42,7 +45,7 @@ Builds a new list collection from individual datasets or collections. Description =========== -This tool combines individual datasets or collections into a new collection. The simplest scenario is building a new colection from individual datasets (case **A** in the image below). You can merge a collection with individual dataset(s). In this case (see **B** in the image below) the individual dataset(s) will be merged with each element of the input collection to create a nested collection. Finally, two or more collection can be merged together creating a nested collection (case **C** in the image below). +This tool combines individual datasets or collections into a new collection. The simplest scenario is building a new collection from individual datasets (case **A** in the image below). You can merge a collection with individual dataset(s). In this case (see **B** in the image below) the individual dataset(s) will be merged with each element of the input collection to create a nested collection. Finally, two or more collections can be merged together creating a nested collection (case **C** in the image below). .. class:: warningmark @@ -51,15 +54,14 @@ This tool combines individual datasets or collections into a new collection. The ------ .. image:: ${static_path}/images/tools/collection_ops/build_list.svg + :alt: Build list operation :width: 800 - :alt: Unzipping operation ------- .. class:: infomark - -This tool will create a new collection from your history datasets but your quota usage will not increase. +@QUOTA_USAGE_NOTE@ ]]> diff --git a/lib/galaxy/tools/cross_product_flat.xml b/lib/galaxy/tools/cross_product_flat.xml index 8653b3ba802..a4cf09edb01 100644 --- a/lib/galaxy/tools/cross_product_flat.xml +++ b/lib/galaxy/tools/cross_product_flat.xml @@ -1,7 +1,7 @@ - + of two collections into flat lists for all-vs-all comparison model_operation_macros.xml diff --git a/lib/galaxy/tools/cross_product_nested.xml b/lib/galaxy/tools/cross_product_nested.xml index 12a52f403b4..46f0bef7bd6 100644 --- a/lib/galaxy/tools/cross_product_nested.xml +++ b/lib/galaxy/tools/cross_product_nested.xml @@ -1,7 +1,7 @@ - + of two collections into nested lists for all-vs-all comparison model_operation_macros.xml diff --git a/lib/galaxy/tools/duplicate_file_to_collection.xml b/lib/galaxy/tools/duplicate_file_to_collection.xml index e52aeb4bbd5..5c0acef63aa 100644 --- a/lib/galaxy/tools/duplicate_file_to_collection.xml +++ b/lib/galaxy/tools/duplicate_file_to_collection.xml @@ -2,8 +2,11 @@ name="Duplicate file to collection" version="1.0.0" tool_type="duplicate_file_to_collection"> - + by duplicating a dataset N times + + model_operation_macros.xml + + + Sample Sheet: "RNA-seq" + + + + + + + Name + rep + treatment + + + + + + sample1 + 1 + control + + + + + + sample2 + 2 + treated + + + + + + sample3 + 3 + control + + + metadata columns + + + + + + + Convert + + + + List: "RNA-seq" + + + + sample1.fastq + + + sample2.fastq + + + sample3.fastq + + + same datasets, no metadata + diff --git a/static/images/tools/collection_ops/sample_sheet_to_tabular.svg b/static/images/tools/collection_ops/sample_sheet_to_tabular.svg new file mode 100644 index 00000000000..2e10fa6d3be --- /dev/null +++ b/static/images/tools/collection_ops/sample_sheet_to_tabular.svg @@ -0,0 +1,105 @@ + + + + + + + + + + + + + + + + + + + + + Sample Sheet: "RNA-seq" + + + + + + + Name + rep + treatment + + + + + + sample1 + 1 + control + + + + + + sample2 + 2 + treated + + + + + + sample3 + 3 + control + + + metadata columns + + + + + + + Extract + metadata + + + + Tabular dataset + + + + + + element_id + rep + treatment + + + + + + sample1 + 1 + control + + + + + + sample2 + 2 + treated + + + + + + sample3 + 3 + control + + + tab-separated metadata export + From 10f29805f87f33b4d7fb9bc7cbf24e9fcc3832f8 Mon Sep 17 00:00:00 2001 From: nekrut Date: Sun, 1 Mar 2026 23:12:58 +0000 Subject: [PATCH 04/15] Replace build_list SVG with clean hand-coded diagram Replace the 207KB Inkscape-generated build_list.svg with a 10KB hand-coded SVG matching the style of the sample sheet tool diagrams. Uses abstract labels (Dataset 1, Dataset A, etc.) for all three cases. Co-Authored-By: Claude Opus 4.6 --- .../tools/collection_ops/build_list.svg | 164 +++++++++++++++++- 1 file changed, 163 insertions(+), 1 deletion(-) diff --git a/static/images/tools/collection_ops/build_list.svg b/static/images/tools/collection_ops/build_list.svg index c69d8fad8d4..75f2bac9891 100644 --- a/static/images/tools/collection_ops/build_list.svg +++ b/static/images/tools/collection_ops/build_list.svg @@ -1 +1,163 @@ - \ No newline at end of file + + + + + + + + + + + + + + + + + + + + + + + + A + + + + Dataset 1 + + + Dataset 2 + + + Dataset 3 + + + + Build list + + + + List + + + Dataset 1 + + + Dataset 2 + + + Dataset 3 + + + + + + + + + + B + + + + List + + + Dataset 1 + + + Dataset 2 + + + + + + + + Dataset A + + + + Build list + + + + Nested list + + + + + Dataset 1 + + Dataset A + + + + + Dataset 2 + + Dataset A + + + each element paired with individual dataset + + + + + + + + + + C + + + + List 1 + + + Dataset 1 + + + Dataset 2 + + + + + + + + List 2 + + + Dataset A + + + Dataset B + + + + Build list + + + + Nested list + + + + + Dataset 1 + + Dataset A + + + + + Dataset 2 + + Dataset B + + + collections matched element-wise + + From 574c8d06e9c0bd2ad1b67de7905b0ccbccb2204a Mon Sep 17 00:00:00 2001 From: nekrut Date: Sun, 1 Mar 2026 23:16:55 +0000 Subject: [PATCH 05/15] Expand help text and add SVG for duplicate file to collection tool Add examples, workflow use case, and a clean SVG schematic showing a single dataset being duplicated into a list collection. Co-Authored-By: Claude Opus 4.6 --- .../tools/duplicate_file_to_collection.xml | 44 +++++++++++++++-- .../duplicate_file_to_collection.svg | 47 +++++++++++++++++++ 2 files changed, 88 insertions(+), 3 deletions(-) create mode 100644 static/images/tools/collection_ops/duplicate_file_to_collection.svg diff --git a/lib/galaxy/tools/duplicate_file_to_collection.xml b/lib/galaxy/tools/duplicate_file_to_collection.xml index 5c0acef63aa..bfdaada2e16 100644 --- a/lib/galaxy/tools/duplicate_file_to_collection.xml +++ b/lib/galaxy/tools/duplicate_file_to_collection.xml @@ -44,15 +44,53 @@ Synopsis ======== -Creates a collection of arbitrary size by duplicating an input dataset N times, where N is a user-specified integer. +Creates a list collection by duplicating a single dataset N times. =========== Description =========== -This tool allows creation of a dataset collection of arbitrary size. It takes an input dataset and an integer parameter, which specifies the number of times to duplicate the dataset in the output collection. In addition, the user can specify the base name for the element identifier to use in the output. For example, if `Number` is specified as 3 and `Element identifier` as 'Element', the output collection will contain three identical datasets, with the identifiers `Element 1`, `Element 2` and `Element 3`. +This tool takes one input dataset and produces a list collection containing N identical copies of it. You specify the number of copies and a base name for the element identifiers. The copies are numbered automatically — for example, with a count of 3 and base name ``sample``, the output collection will contain ``sample 1``, ``sample 2``, and ``sample 3``. ------ +This is useful when a workflow or tool expects a list collection as input but you only have a single dataset. Instead of uploading the same file multiple times, you can duplicate it into a collection in one step. + +======== +Examples +======== + +**Basic duplication** + +Given a single reference genome file and the settings: + +- *Size of output collection*: ``3`` +- *Element identifier*: ``ref`` + +The output is a list collection containing three identical copies:: + + List: "reference.fa (duplicated)" + + ref 1 → reference.fa + ref 2 → reference.fa + ref 3 → reference.fa + +------- + +**Workflow use case** + +If you have a list of samples and a tool that compares each sample against a reference, you can duplicate the reference to match the number of samples. Galaxy will then pair them element-wise: + +- Sample list: ``[sample1.fq, sample2.fq, sample3.fq]`` +- Duplicated reference: ``[ref 1, ref 2, ref 3]`` (all pointing to the same file) + +This avoids having to manually create a matched collection. + +------- + +.. image:: ${static_path}/images/tools/collection_ops/duplicate_file_to_collection.svg + :alt: Duplicate file to collection + :width: 500 + +------- .. class:: infomark diff --git a/static/images/tools/collection_ops/duplicate_file_to_collection.svg b/static/images/tools/collection_ops/duplicate_file_to_collection.svg new file mode 100644 index 00000000000..243d2bb9f70 --- /dev/null +++ b/static/images/tools/collection_ops/duplicate_file_to_collection.svg @@ -0,0 +1,47 @@ + + + + + + + + + + + + + + + + + + + + + reference.fa + + + N = 3, identifier = "ref" + + + + Duplicate + + + + List + + + + ref 1 + + + ref 2 + + + ref 3 + + + all copies are identical + + From d7be4be858296e40de038b57e65319b39b1ca5fd Mon Sep 17 00:00:00 2001 From: nekrut Date: Mon, 2 Mar 2026 00:11:14 +0000 Subject: [PATCH 06/15] Expand help text and add SVG for extract dataset tool Add nested collection explanation with examples showing both flat list extraction (case A) and list:list mapped extraction (case B). Co-Authored-By: Claude Opus 4.6 --- lib/galaxy/tools/extract_dataset.xml | 56 +++++++- .../tools/collection_ops/extract_dataset.svg | 124 ++++++++++++++++++ 2 files changed, 177 insertions(+), 3 deletions(-) create mode 100644 static/images/tools/collection_ops/extract_dataset.svg diff --git a/lib/galaxy/tools/extract_dataset.xml b/lib/galaxy/tools/extract_dataset.xml index 48c25db1771..a1e3fa52d17 100644 --- a/lib/galaxy/tools/extract_dataset.xml +++ b/lib/galaxy/tools/extract_dataset.xml @@ -47,15 +47,65 @@ Synopsis ======== -Extracts a single dataset from a collection based on either position or identifier. +Extracts a single dataset from a collection by position or identifier. =========== Description =========== -This tool allows extracting datasets based on position (**The first dataset** and **Select by index** options) or name (**Select by element identifier** option). It effectively collapses the inner-most collection into a dataset. +This tool pulls one dataset out of a collection and places it in your history as a standalone dataset. You can select the dataset in three ways: -For nested collections (e.g., a list of lists of lists: ``outer:middle:inner``, extracting the inner dataset element) a new list is created where the selected element takes the position of the inner-most collection (so ``outer:middle``, where ``middle`` is not a collection but the inner dataset element). +- **The first dataset** — takes the first element of the collection. +- **Select by index** — takes the element at a specific position (starting from 0). +- **Select by element identifier** — takes the element whose name matches the given identifier. + +The extracted dataset is a copy of the original, so the input collection is not modified. + +======== +Examples +======== + +**Extract from a flat list** (case **A** in the image below) + +Given a list collection with three elements:: + + List: "RNA-seq samples" + + control → control.fastq + treated → treated.fastq + reference → reference.fastq + +Selecting **by element identifier** = ``treated`` produces a single standalone dataset: ``treated.fastq``. Selecting **by index** = ``0`` would extract ``control.fastq`` instead. + +------- + +**Extract from a nested collection** (case **B** in the image below) + +When the input is a nested collection such as ``list:list``, Galaxy automatically maps the extraction over the outer list. The same selection is applied to every inner list, and the result is a flat list — one nesting level is removed. + +For example, given a ``list:list``:: + + list:list: "Paired analysis" + + sampleA → [Dataset 1, Dataset 2, Dataset 3] + sampleB → [Dataset A, Dataset B, Dataset C] + +Extracting **by index** = ``1`` pulls element 1 from each inner list and produces a flat list:: + + List: "Paired analysis (extracted element)" + + sampleA → Dataset 2 + sampleB → Dataset B + +The outer identifiers (``sampleA``, ``sampleB``) are preserved. The inner lists are replaced by the single extracted element from each. + +------- + +.. image:: ${static_path}/images/tools/collection_ops/extract_dataset.svg + :alt: Extract dataset from collection + :width: 800 + +------- .. class:: warningmark diff --git a/static/images/tools/collection_ops/extract_dataset.svg b/static/images/tools/collection_ops/extract_dataset.svg new file mode 100644 index 00000000000..f0d1e776c8a --- /dev/null +++ b/static/images/tools/collection_ops/extract_dataset.svg @@ -0,0 +1,124 @@ + + + + + + + + + + + + + + + + + + + + + + + + A + + + + List + + + 0 + control + + + + 1 + treated + + + 2 + reference + + identifier = "treated" + + + + Extract + + + + treated + + standalone dataset in history + + + + + + + + + + B + + + + Nested list + + + + sampleA + + + 0 + Dataset 1 + + + + 1 + Dataset 2 + + + 2 + Dataset 3 + + + + sampleB + + + 0 + Dataset A + + + + 1 + Dataset B + + + 2 + Dataset C + + + index = 1 + + + + Extract + + + + List + + + Dataset 2 + sampleA + + + Dataset B + sampleB + + one nesting level removed + + From 8b4c2b92d0d27d203841408e4c726897be7a01ba Mon Sep 17 00:00:00 2001 From: nekrut Date: Mon, 2 Mar 2026 01:01:04 +0000 Subject: [PATCH 07/15] Expand help text and add SVG for filter from file tool Rewrite help with clearer descriptions of ABSENT/PRESENT modes, add two-case SVG showing both filtering directions with dual output collections (filtered + discarded). Co-Authored-By: Claude Opus 4.6 --- lib/galaxy/tools/filter_from_file_1.1.0.xml | 63 ++++----- .../tools/collection_ops/filter_from_file.svg | 130 ++++++++++++++++++ 2 files changed, 157 insertions(+), 36 deletions(-) create mode 100644 static/images/tools/collection_ops/filter_from_file.svg diff --git a/lib/galaxy/tools/filter_from_file_1.1.0.xml b/lib/galaxy/tools/filter_from_file_1.1.0.xml index f30146064ea..9bf856e45ec 100644 --- a/lib/galaxy/tools/filter_from_file_1.1.0.xml +++ b/lib/galaxy/tools/filter_from_file_1.1.0.xml @@ -89,63 +89,54 @@ Synopsis ======== -Filters elements from a collection using a list supplied in a file. +Filters elements from a collection using a list of identifiers supplied in a text file. =========== Description =========== -This tool allows filtering elements from a data collection. It takes an input collection and a text file with element identifiers (one per line). The tool behavior is controlled by the **How should the elements to remove be determined?** drop-down, which has the following options: +This tool splits a collection into two: one with elements that pass the filter and one with elements that don't. You provide a text file containing element identifiers (one per line), then choose the filtering mode: -**Remove if identifiers are ABSENT from file** +- **Remove if identifiers are ABSENT from file** — keeps only elements whose identifiers appear in the file. Elements not listed are discarded. +- **Remove if identifiers are PRESENT in file** — removes elements whose identifiers appear in the file. Elements not listed are kept. -Given a collection:: +The tool always produces **two** output collections: ``(filtered)`` containing the elements that passed, and ``(discarded)`` containing the rest. Identifiers in the file that do not match any element in the collection are silently ignored. - Collection: [Dataset A] - [Dataset B] - [Dataset X] +======== +Examples +======== -and a text file:: +**Keep only listed elements** (Remove if ABSENT) - A - B - Z +Given a collection with three elements and a text file listing ``A``, ``B``, and ``Z``:: -the tool will return two collections:: + Collection: Text file: - (filtered): [Dataset A] - [Dataset B] + Dataset A A + Dataset B B + Dataset X Z - (discarded): [Dataset X] +Elements ``A`` and ``B`` are in the file, so they are kept. ``X`` is not in the file, so it is discarded. ``Z`` is in the file but has no matching element, so it is ignored:: ------- + (filtered): Dataset A, Dataset B + (discarded): Dataset X -**Remove if identifiers are PRESENT in file** +------- -Given a collection:: +**Remove listed elements** (Remove if PRESENT) - Collection: [Dataset A] - [Dataset B] - [Dataset X] +With the same inputs but switching to PRESENT mode, the logic is inverted — elements ``A`` and ``B`` are in the file, so they are removed. ``X`` is not listed, so it is kept:: -and a text file:: + (filtered): Dataset X + (discarded): Dataset A, Dataset B - A - B - Z +------- -the tool will return two collections:: +.. image:: ${static_path}/images/tools/collection_ops/filter_from_file.svg + :alt: Filter collection from file + :width: 800 - (filtered): [Dataset X] - - (discarded): [Dataset A] - [Dataset B] - -.. class:: warningmark - -**Note** how the tool deals with the ``Z`` entry. - ------ +------- .. class:: infomark diff --git a/static/images/tools/collection_ops/filter_from_file.svg b/static/images/tools/collection_ops/filter_from_file.svg new file mode 100644 index 00000000000..128244ec61e --- /dev/null +++ b/static/images/tools/collection_ops/filter_from_file.svg @@ -0,0 +1,130 @@ + + + + + + + + + + + + + + + + + + + + + + + + A + + + + List + + + Dataset A + + + Dataset B + + + Dataset X + + + + identifiers.txt + A + B + Z + + + remove if ABSENT + + + + Filter + + + + (filtered) + + + Dataset A + + + Dataset B + + + + (discarded) + + + Dataset X + + + Z has no match — ignored + + + + + + + + + + B + + + + List + + + Dataset A + + + Dataset B + + + Dataset X + + + + identifiers.txt + A + B + Z + + + remove if PRESENT + + + + Filter + + + + (filtered) + + + Dataset X + + + + (discarded) + + + Dataset A + + + Dataset B + + + Z has no match — ignored + + From 171f783b6d256dccabe705be9abcd7c91ba054f1 Mon Sep 17 00:00:00 2001 From: nekrut Date: Mon, 2 Mar 2026 01:11:45 +0000 Subject: [PATCH 08/15] Expand help text and replace SVGs for filter empty, failed, and null tools Co-Authored-By: Claude Opus 4.6 --- .../tools/filter_empty_collection_1.1.0.xml | 47 +- .../tools/filter_failed_collection_1.1.0.xml | 45 +- lib/galaxy/tools/filter_null_1.1.0.xml | 47 +- .../tools/collection_ops/filter_empty.svg | 594 +---------- .../tools/collection_ops/filter_error.svg | 979 +----------------- .../tools/collection_ops/filter_null.svg | 54 + 6 files changed, 287 insertions(+), 1479 deletions(-) create mode 100644 static/images/tools/collection_ops/filter_null.svg diff --git a/lib/galaxy/tools/filter_empty_collection_1.1.0.xml b/lib/galaxy/tools/filter_empty_collection_1.1.0.xml index 02cb73e2631..fde51ce3eaa 100644 --- a/lib/galaxy/tools/filter_empty_collection_1.1.0.xml +++ b/lib/galaxy/tools/filter_empty_collection_1.1.0.xml @@ -42,21 +42,58 @@ Synopsis ======== -Removes empty datasets from a collection. +Removes empty (zero-byte) datasets from a collection. =========== Description =========== -This tool takes a dataset collection and filters out (removes) datasets that have no content (zero bytes). This is useful for continuing a multi-sample analysis when downstream tools require datasets to have content. +This tool takes a dataset collection and filters out elements that have no content. A dataset is considered empty if it contains zero bytes of actual data (including compressed archives that decompress to nothing). -Optionally, you can provide a replacement dataset. When a replacement is specified, empty elements are replaced with that dataset instead of being removed. +This is useful when a multi-sample analysis produces some empty outputs — for example, when a filtering step removes all reads for certain samples. Downstream tools often fail on empty inputs, so removing them lets the rest of the analysis proceed. + +Optionally, you can provide a **replacement dataset**. When specified, empty elements are replaced with that dataset instead of being removed. This preserves the collection size, which is important when element-wise pairing with another collection is needed. + +======== +Examples +======== + +**Remove empty elements** + +Given a list collection where one sample produced no output:: + + List: "Filtered reads" + + sample1 → reads.fastq (1.2 MB) + sample2 → reads.fastq (0 bytes) + sample3 → reads.fastq (800 KB) + +The output keeps only non-empty elements:: + + List: "Filtered reads (filtered empty datasets)" + + sample1 → reads.fastq (1.2 MB) + sample3 → reads.fastq (800 KB) + +------- + +**Replace instead of remove** + +If a replacement dataset is provided, empty elements are swapped rather than dropped, preserving the collection structure:: + + List: "Filtered reads (filtered empty datasets)" + + sample1 → reads.fastq (1.2 MB) + sample2 → replacement.fastq + sample3 → reads.fastq (800 KB) + +------- .. image:: ${static_path}/images/tools/collection_ops/filter_empty.svg - :alt: Filtering empty datasets + :alt: Filter empty datasets from collection :width: 500 ------ +------- .. class:: infomark diff --git a/lib/galaxy/tools/filter_failed_collection_1.1.0.xml b/lib/galaxy/tools/filter_failed_collection_1.1.0.xml index e6c081bf01b..3d58f5a9357 100644 --- a/lib/galaxy/tools/filter_failed_collection_1.1.0.xml +++ b/lib/galaxy/tools/filter_failed_collection_1.1.0.xml @@ -53,15 +53,52 @@ Removes datasets in the error (red) state from a collection. Description =========== -This tool takes a dataset collection and filters out (removes) datasets in the failed (red) state. This is useful for continuing a multi-sample analysis when one or more of the samples fails at some point. +This tool takes a dataset collection and filters out elements whose datasets are in the failed (red) state. A dataset enters this state when the tool that created it encountered an error during execution. -Optionally, you can provide a replacement dataset. When a replacement is specified, failed elements are replaced with that dataset instead of being removed. +This is useful when running a multi-sample analysis and one or more samples fail at some step. Rather than restarting the entire analysis, you can remove the failed elements and continue with the samples that succeeded. + +Optionally, you can provide a **replacement dataset**. When specified, failed elements are replaced with that dataset instead of being removed. This preserves the collection size, which is important when element-wise pairing with another collection is needed. + +======== +Examples +======== + +**Remove failed elements** + +Given a list collection where one sample's alignment failed:: + + List: "Aligned reads" + + sample1 → aligned.bam (ok) + sample2 → aligned.bam (error) + sample3 → aligned.bam (ok) + +The output keeps only successful elements:: + + List: "Aligned reads (filtered failed datasets)" + + sample1 → aligned.bam + sample3 → aligned.bam + +------- + +**Replace instead of remove** + +If a replacement dataset is provided, failed elements are swapped rather than dropped, preserving the collection structure:: + + List: "Aligned reads (filtered failed datasets)" + + sample1 → aligned.bam + sample2 → replacement.bam + sample3 → aligned.bam + +------- .. image:: ${static_path}/images/tools/collection_ops/filter_error.svg - :alt: Filter failed datasets + :alt: Filter failed datasets from collection :width: 500 ------ +------- .. class:: infomark diff --git a/lib/galaxy/tools/filter_null_1.1.0.xml b/lib/galaxy/tools/filter_null_1.1.0.xml index 7f074e520a5..215d1666a0c 100644 --- a/lib/galaxy/tools/filter_null_1.1.0.xml +++ b/lib/galaxy/tools/filter_null_1.1.0.xml @@ -44,11 +44,52 @@ Removes null (missing) elements from a collection. Description =========== -This tool takes a dataset collection and filters out null elements. Null elements can appear when workflows use conditional steps -- if a step is skipped, it may produce a null placeholder instead of a real dataset. +This tool takes a dataset collection and filters out null elements. A null element is a placeholder that contains no actual dataset — it appears when a workflow uses conditional steps and a step is skipped, or when an expression tool produces a ``null`` value. -Optionally, you can provide a replacement dataset. When a replacement is specified, null elements are replaced with that dataset instead of being removed. +This is useful for cleaning up collections produced by conditional workflows. Downstream tools cannot process null elements, so removing them ensures the rest of the analysis can proceed. ------ +Optionally, you can provide a **replacement dataset**. When specified, null elements are replaced with that dataset instead of being removed. This preserves the collection size, which is important when element-wise pairing with another collection is needed. + +======== +Examples +======== + +**Remove null elements** + +Given a list collection produced by a conditional workflow where one branch was skipped:: + + List: "Workflow outputs" + + sample1 → result.txt (ok) + sample2 → (null) + sample3 → result.txt (ok) + +The output keeps only real elements:: + + List: "Workflow outputs (without null datasets)" + + sample1 → result.txt + sample3 → result.txt + +------- + +**Replace instead of remove** + +If a replacement dataset is provided, null elements are swapped rather than dropped, preserving the collection structure:: + + List: "Workflow outputs (without null datasets)" + + sample1 → result.txt + sample2 → replacement.txt + sample3 → result.txt + +------- + +.. image:: ${static_path}/images/tools/collection_ops/filter_null.svg + :alt: Filter null elements from collection + :width: 500 + +------- .. class:: infomark diff --git a/static/images/tools/collection_ops/filter_empty.svg b/static/images/tools/collection_ops/filter_empty.svg index 6e4be27b030..3d4b8dbcb05 100644 --- a/static/images/tools/collection_ops/filter_empty.svg +++ b/static/images/tools/collection_ops/filter_empty.svg @@ -1,542 +1,54 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + List + + + sample1 + 1.2 MB + + + + sample2 + 0 bytes + + + sample3 + 800 KB + + sample2 is empty + + + + Filter + + + + List (filtered) + + + sample1 + + + sample3 + + empty element removed + diff --git a/static/images/tools/collection_ops/filter_error.svg b/static/images/tools/collection_ops/filter_error.svg index fb4d01ad199..114dbaed524 100644 --- a/static/images/tools/collection_ops/filter_error.svg +++ b/static/images/tools/collection_ops/filter_error.svg @@ -1,927 +1,54 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + List + + + sample1 + ok + + + + sample2 + error + + + sample3 + ok + + sample2 failed + + + + Filter + + + + List (filtered) + + + sample1 + + + sample3 + + failed element removed + diff --git a/static/images/tools/collection_ops/filter_null.svg b/static/images/tools/collection_ops/filter_null.svg new file mode 100644 index 00000000000..aa63f945f21 --- /dev/null +++ b/static/images/tools/collection_ops/filter_null.svg @@ -0,0 +1,54 @@ + + + + + + + + + + + + + + + + + + + + + List + + + sample1 + ok + + + + sample2 + null + + + sample3 + ok + + sample2 is null + + + + Filter + + + + List (filtered) + + + sample1 + + + sample3 + + null element removed + + From cdbf13a202fe721b619a721f2a9ed5cf1e76fff3 Mon Sep 17 00:00:00 2001 From: nekrut Date: Mon, 2 Mar 2026 01:18:37 +0000 Subject: [PATCH 09/15] Expand help text and replace SVG for flatten collection tool Co-Authored-By: Claude Opus 4.6 --- lib/galaxy/tools/flatten_collection.xml | 68 +- .../images/tools/collection_ops/flatten.svg | 1042 +---------------- 2 files changed, 128 insertions(+), 982 deletions(-) diff --git a/lib/galaxy/tools/flatten_collection.xml b/lib/galaxy/tools/flatten_collection.xml index 1340e863d28..0f08f86d323 100644 --- a/lib/galaxy/tools/flatten_collection.xml +++ b/lib/galaxy/tools/flatten_collection.xml @@ -48,23 +48,81 @@ Synopsis ======== -Flattens nested collection into a simple list. +Flattens a nested collection into a simple list. =========== Description =========== -This tool takes nested collections such as a list of lists or a list of dataset pairs and produces a flat list from the inputs. It effectively "flattens" the hierarchy. The collection identifiers are merged together (using "_" as default) to create new collection identifiers in the flattened result: +This tool takes a nested collection — such as a list of lists (``list:list``), a list of dataset pairs (``list:paired``), or even deeper structures — and collapses it into a single flat ``list``. Every dataset in the hierarchy becomes a top-level element in the output. + +Because the hierarchy is removed, the element identifiers from each nesting level are joined together to form new unique names. By default the separator is an underscore (``_``), but you can choose a colon (``:``) or dash (``-``) instead. + +======== +Examples +======== + +**Flatten a list of pairs** + +A ``list:paired`` collection with two samples:: + + list:paired: "Sequencing reads" + + sample1/ + forward → reads_R1.fastq + reverse → reads_R2.fastq + sample2/ + forward → reads_R1.fastq + reverse → reads_R2.fastq + +After flattening with separator ``_``:: + + list: "Sequencing reads (flattened)" + + sample1_forward → reads_R1.fastq + sample1_reverse → reads_R2.fastq + sample2_forward → reads_R1.fastq + sample2_reverse → reads_R2.fastq + +------- + +**Flatten a list of lists** + +A ``list:list`` collection:: + + list:list: "Batches" + + batch1/ + Dataset 1 + Dataset 2 + batch2/ + Dataset 3 + +After flattening:: + + list: "Batches (flattened)" + + batch1_Dataset 1 + batch1_Dataset 2 + batch2_Dataset 3 + +------- + +**Choosing a different separator** + +With separator set to ``-`` instead of ``_``, the same example would produce ``batch1-Dataset 1``, ``batch1-Dataset 2``, etc. + +------- .. image:: ${static_path}/images/tools/collection_ops/flatten.svg - :alt: Flattening operation + :alt: Flatten a nested collection into a simple list :width: 500 ----- +------- .. class:: infomark @QUOTA_USAGE_NOTE@ - ]]> + ]]> diff --git a/static/images/tools/collection_ops/flatten.svg b/static/images/tools/collection_ops/flatten.svg index 664dff4f552..0d4b5de0ada 100644 --- a/static/images/tools/collection_ops/flatten.svg +++ b/static/images/tools/collection_ops/flatten.svg @@ -1,978 +1,66 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + list : paired + + + + sample1 + + + forward + + + reverse + + + + sample2 + + + forward + + + reverse + + + + Flatten + join with "_" + + + + List (flattened) + + + sample1_forward + + + sample1_reverse + + + sample2_forward + + + sample2_reverse + + hierarchy removed, identifiers joined + From db11f526bd0435537f848b06f40ea38f37b391d8 Mon Sep 17 00:00:00 2001 From: nekrut Date: Mon, 2 Mar 2026 01:34:17 +0000 Subject: [PATCH 10/15] Expand help text and add SVG for harmonize two collections tool Co-Authored-By: Claude Opus 4.6 --- .../tools/harmonize_two_collections_list.xml | 63 +++++++++++----- .../images/tools/collection_ops/harmonize.svg | 75 +++++++++++++++++++ 2 files changed, 120 insertions(+), 18 deletions(-) create mode 100644 static/images/tools/collection_ops/harmonize.svg diff --git a/lib/galaxy/tools/harmonize_two_collections_list.xml b/lib/galaxy/tools/harmonize_two_collections_list.xml index 238b4ce63bb..f984684669d 100644 --- a/lib/galaxy/tools/harmonize_two_collections_list.xml +++ b/lib/galaxy/tools/harmonize_two_collections_list.xml @@ -218,31 +218,58 @@ Harmonizes two collections so they contain the same element identifiers in the s Description =========== -This tool takes two collections and produces two output collections that share exactly the same set of identifiers in the same order. Any identifiers that are present in only one of the two input collections are removed from both outputs. The order of the first input collection is preserved. +This tool takes two collections and produces two output collections that share exactly the same set of identifiers in the same order. It performs two operations: -======= -Example -======= +1. **Intersection** — any identifier that appears in only one of the two input collections is removed from both outputs. +2. **Reordering** — the output elements follow the order of the **first** input collection. -If the inputs are:: +This is useful when two collections were produced by independent analyses and you need to pair them element-wise for a downstream tool. Without harmonization, mismatched identifiers or different ordering would cause errors. - Collection 1: [Horse123] - [Donkey543] - [Mule176] +The two input collections do not need to have the same collection type. For example, you can harmonize a ``list`` with a ``list:paired`` — each output preserves the type of its corresponding input. - Collection 2: [Horse] - [Mule176] - [Donkey543] +======== +Examples +======== -The tool will output:: +**Keep only shared identifiers** - Collection 1: [Donkey543] - [Mule176] +Given two collections with partially overlapping identifiers:: - Collection 2: [Donkey543] - [Mule176] + Collection 1 (reference order): -Note that ``Horse123`` and ``Horse`` are removed because they do not match between collections. + sampleA → Dataset 1 + sampleB → Dataset 2 + sampleC → Dataset 3 + + Collection 2 (different order, extra element): + + sampleD → Dataset 4 + sampleC → Dataset 5 + sampleA → Dataset 6 + +Output 1 (from Collection 1, matching order preserved):: + + sampleA → Dataset 1 + sampleC → Dataset 3 + +Output 2 (from Collection 2, reordered to match Collection 1):: + + sampleA → Dataset 6 + sampleC → Dataset 5 + +``sampleB`` and ``sampleD`` are removed because they are not present in both collections. + +------- + +**No matching identifiers** + +If the two collections share no identifiers at all, both outputs will be empty collections. + +------- + +.. image:: ${static_path}/images/tools/collection_ops/harmonize.svg + :alt: Harmonize two collections to match identifiers and order + :width: 620 ------- @@ -250,5 +277,5 @@ Note that ``Horse123`` and ``Horse`` are removed because they do not match betwe @QUOTA_USAGE_NOTE@ - ]]> + ]]> diff --git a/static/images/tools/collection_ops/harmonize.svg b/static/images/tools/collection_ops/harmonize.svg new file mode 100644 index 00000000000..0f5724eec23 --- /dev/null +++ b/static/images/tools/collection_ops/harmonize.svg @@ -0,0 +1,75 @@ + + + + + + + + + + + + + + + + + + + + + Collection 1 + + + sampleA + + + sampleB + only here + + + sampleC + + + + Collection 2 + + + sampleD + only here + + + sampleC + + + sampleA + + + + Harmonize + keep shared, match order + + + + Output 1 + + + sampleA + + + sampleC + + + + Output 2 + + + sampleA + + + sampleC + + + same identifiers, same order + + From 28ded5c80fe3124e5252a49c7df6d80fcd8b42bc Mon Sep 17 00:00:00 2001 From: nekrut Date: Mon, 2 Mar 2026 01:53:44 +0000 Subject: [PATCH 11/15] Expand help text and add SVG for keep success datasets tool Co-Authored-By: Claude Opus 4.6 --- .../tools/keep_success_collection_1.1.0.xml | 51 ++++++++++++++-- .../tools/collection_ops/keep_success.svg | 60 +++++++++++++++++++ 2 files changed, 106 insertions(+), 5 deletions(-) create mode 100644 static/images/tools/collection_ops/keep_success.svg diff --git a/lib/galaxy/tools/keep_success_collection_1.1.0.xml b/lib/galaxy/tools/keep_success_collection_1.1.0.xml index 5e2bad5573e..22a89f92e44 100644 --- a/lib/galaxy/tools/keep_success_collection_1.1.0.xml +++ b/lib/galaxy/tools/keep_success_collection_1.1.0.xml @@ -52,19 +52,60 @@ Keeps only datasets in the success (green) state from a collection. Description =========== -This tool takes a dataset collection and retains only datasets in the success (green) state, removing any that are failed, paused, or in other non-success states. This is useful for continuing a multi-sample analysis when one or more of the samples fails or is in a paused state. +This tool takes a dataset collection and retains only elements whose datasets are in the success (green/ok) state. Elements in any other state — failed (red), paused (gray), or still running — are removed from the output collection. -Optionally, you can provide a replacement dataset. When a replacement is specified, unsuccessful elements are replaced with that dataset instead of being removed. +This is stricter than **Filter failed datasets**, which only removes elements in the error state. **Keep success** removes *anything* that is not ok, including paused datasets that have not yet completed. + +This is useful when a multi-sample workflow has partially completed and you want to continue with just the samples that succeeded, discarding any that failed or stalled. + +Optionally, you can provide a **replacement dataset**. When specified, unsuccessful elements are replaced with that dataset instead of being removed. This preserves the collection size, which is important when element-wise pairing with another collection is needed. + +======== +Examples +======== + +**Remove unsuccessful elements** + +Given a collection where one sample failed and another is paused:: + + List: "Aligned reads" + + sample1 → aligned.bam (ok) + sample2 → aligned.bam (error) + sample3 → aligned.bam (ok) + sample4 → aligned.bam (paused) + +The output keeps only the successful elements:: + + List: "Aligned reads (only successful datasets)" + + sample1 → aligned.bam + sample3 → aligned.bam + +------- + +**Replace instead of remove** + +If a replacement dataset is provided, unsuccessful elements are swapped rather than dropped:: + + List: "Aligned reads (only successful datasets)" + + sample1 → aligned.bam + sample2 → replacement.bam + sample3 → aligned.bam + sample4 → replacement.bam + +------- .. image:: ${static_path}/images/tools/collection_ops/keep_success.svg - :alt: Keep success datasets + :alt: Keep only successful datasets in a collection :width: 500 ------ +------- .. class:: infomark @QUOTA_USAGE_NOTE@ - ]]> + ]]> diff --git a/static/images/tools/collection_ops/keep_success.svg b/static/images/tools/collection_ops/keep_success.svg new file mode 100644 index 00000000000..c5c4b9466fc --- /dev/null +++ b/static/images/tools/collection_ops/keep_success.svg @@ -0,0 +1,60 @@ + + + + + + + + + + + + + + + + + + + + + List + + + sample1 + ok + + + + sample2 + error + + + sample3 + ok + + + + sample4 + paused + + only ok elements kept + + + + Keep + success only + + + + List (success only) + + + sample1 + + + sample3 + + failed and paused removed + + From 185fb88dd101f92f7d07c25e3c253a3423c3eaaa Mon Sep 17 00:00:00 2001 From: nekrut Date: Mon, 2 Mar 2026 01:59:53 +0000 Subject: [PATCH 12/15] Expand help text and add/replace SVGs for merge and nest collection tools Co-Authored-By: Claude Opus 4.6 --- lib/galaxy/tools/merge_collection.xml | 141 +-- lib/galaxy/tools/nest_collection.xml | 57 +- static/images/tools/collection_ops/merge.svg | 61 ++ static/images/tools/collection_ops/nest.svg | 1020 ++---------------- 4 files changed, 211 insertions(+), 1068 deletions(-) create mode 100644 static/images/tools/collection_ops/merge.svg diff --git a/lib/galaxy/tools/merge_collection.xml b/lib/galaxy/tools/merge_collection.xml index ec2def8f7f6..2abfe01f206 100644 --- a/lib/galaxy/tools/merge_collection.xml +++ b/lib/galaxy/tools/merge_collection.xml @@ -280,55 +280,44 @@ Synopsis ======== -Takes two or more collections and creates a single collection from them. +Concatenates two or more collections into a single collection. =========== Description =========== -By default the tool assumes that collections that are being merged have unique dataset names. If it is not the case only one (the first) of the datasets with a repeated name will be included in the merged collection. For example, suppose you have two collections with some overlapping dataset names:: +This tool takes two or more collections and combines their elements into one output collection. The elements from the first collection appear first, followed by the elements from the second, and so on. - Collection 1: [Dataset A] - [Dataset B] - [Dataset X] - Collection 2: [Dataset A] - [Dataset B] - [Dataset Y] +When the input collections contain elements with the same identifier (name), the tool must decide how to handle the duplicates. The default behavior (``keep first``) retains only the first occurrence and silently drops later duplicates. You can change this under **Advanced Options**. -Merging them will produce a single collection with only two datasets:: +======== +Examples +======== - Merged Collection: [Dataset A] - [Dataset B] - [Dataset X] - [Dataset Y] +**Basic merge (no duplicates)** -This behavior can be changed by clicking on "*Advanced Options*" link. The following options are available: +Two collections with unique identifiers:: -**Keep first instance (Default behavior)** - -Input:: - - Collection 1: [Dataset A] - [Dataset B] - [Dataset X] - Collection 2: [Dataset A] - [Dataset B] - [Dataset Y] + Collection 1: sampleA, sampleB + Collection 2: sampleC, sampleD Output:: - Merged Collection: [Dataset A] - [Dataset B] - [Dataset X] - [Dataset Y] + Merged: sampleA, sampleB, sampleC, sampleD -Here if two collections have identical dataset names, a dataset is chosen from the *first* collection. +------- ------ +.. image:: ${static_path}/images/tools/collection_ops/merge.svg + :alt: Merge two collections into one + :width: 620 -**Keep last instance** +------- -Input:: +================================== +Handling duplicate identifiers +================================== + +When two collections share element identifiers, the **Advanced Options** control what happens. All examples below use:: Collection 1: [Dataset A] [Dataset B] @@ -337,88 +326,54 @@ Input:: [Dataset B] [Dataset Y] -Output:: +**Keep first instance** (default) - Merged Collection: [Dataset A] - [Dataset B] - [Dataset X] - [Dataset Y] +Duplicates are silently dropped — only the copy from the first collection is kept:: -Here if two collections have identical dataset names, a dataset is chosen from the *last* collection. + Merged: [Dataset A] [Dataset B] [Dataset X] [Dataset Y] ------ +------- + +**Keep last instance** + +The opposite — the copy from the last collection wins:: + + Merged: [Dataset A] [Dataset B] [Dataset X] [Dataset Y] + +------- **Append suffix to conflicted element identifiers** -Input:: +All copies of a duplicated name get a numbered suffix (``_1``, ``_2``, …). Unique names are left unchanged:: - Collection 1: [Dataset A] - [Dataset B] - [Dataset X] - Collection 2: [Dataset A] - [Dataset B] - [Dataset Y] + Merged: [Dataset A_1] [Dataset B_1] [Dataset X] + [Dataset A_2] [Dataset B_2] [Dataset Y] -Output:: +------- - Merged Collection: [Dataset A_1] - [Dataset B_1] - [Dataset A_2] - [Dataset B_2] - [Dataset X] - [Dataset Y] +**Append suffix to conflicted identifiers after first one encountered** ----- +The first copy keeps its original name; later copies get a suffix:: -**Append suffix to conflicted element identifiers after first one encountered** + Merged: [Dataset A] [Dataset B] [Dataset X] + [Dataset A_2] [Dataset B_2] [Dataset Y] -Input:: - - Collection 1: [Dataset A] - [Dataset B] - [Dataset X] - Collection 2: [Dataset A] - [Dataset B] - [Dataset Y] - -Output:: - - Merged Collection: [Dataset A] - [Dataset B] - [Dataset A_2] - [Dataset B_2] - [Dataset X] - [Dataset Y] - ------- +------- **Append suffix to every element identifier** -Input:: +Every element — even unique ones — gets a suffix indicating which input collection it came from:: - Collection 1: [Dataset A] - [Dataset B] - [Dataset X] - Collection 2: [Dataset A] - [Dataset B] - [Dataset Y] + Merged: [Dataset A_1] [Dataset B_1] [Dataset X_1] + [Dataset A_2] [Dataset B_2] [Dataset Y_2] -Output:: - - Merged Collection: [Dataset A_1] - [Dataset B_1] - [Dataset A_2] - [Dataset B_2] - [Dataset X_1] - [Dataset Y_2] - ------ +------- **Fail collection creation** -This option will simply trigger an error. +The tool raises an error if any duplicate identifiers are found. ------- +------- .. class:: infomark diff --git a/lib/galaxy/tools/nest_collection.xml b/lib/galaxy/tools/nest_collection.xml index 4191f8fbe10..fd2e87e5027 100644 --- a/lib/galaxy/tools/nest_collection.xml +++ b/lib/galaxy/tools/nest_collection.xml @@ -87,33 +87,60 @@ Synopsis ======== -Nest a collection, i.e. add a nesting level to a collection. +Wraps each element of a collection inside its own inner list, adding one nesting level. =========== Description =========== -This tool takes a collection and creates a collection of collections where the top level elements are lists each containing one element of the input collection. -For instance, for a simple list (of datasets) the tool will produce a list of lists where the inner lists each contain one of the datasets. +This tool takes a collection and produces a nested collection (``list:list``) where each top-level element becomes a single-element inner list. The inner list and its sole element share the same identifier. + +For example, a ``list`` with elements ``[A, B, C]`` becomes a ``list:list`` where the outer list has three elements, each containing a one-element inner list: ``[A→[A], B→[B], C→[C]]``. + +If the input is already nested (e.g. ``list:paired``), the output gains one more level of nesting (``list:list:paired``), where each inner ``list`` contains a single paired collection. + +======== +Examples +======== + +**Nest a flat list** + +Input ``list``:: + + List: "Samples" + + sample1 → Dataset 1 + sample2 → Dataset 2 + sample3 → Dataset 3 + +Output ``list:list``:: + + Nested list: "Samples (nested)" + + sample1/ + sample1 → Dataset 1 + sample2/ + sample2 → Dataset 2 + sample3/ + sample3 → Dataset 3 + +------- + +**Why is this useful?** + +Some Galaxy tools expect a ``list`` as input — when you feed them a ``list``, Galaxy runs the tool once on the entire list. If you want Galaxy to run the tool separately on each element (map over the collection), you need the input to be a ``list:list`` so that Galaxy maps over the outer list and passes each inner list to the tool individually. Nesting provides that extra level. + +------- .. image:: ${static_path}/images/tools/collection_ops/nest.svg - :alt: Nest operation + :alt: Nest a collection by adding a nesting level :width: 500 -======= -Purpose -======= - -Assume a tool processes an item of type X (where X can be a dataset, a dataset pair, a collection). Galaxy also allows to input a collection of items of type X. -In this case Galaxy will map-over the elements of the collection, i.e. create separate jobs (each processing one item of type X). -If X is a list (of datasets, pairs, or lists) then one needs to add an extra nesting level to the collection, such that Galaxy maps-over the elements of X instead -of processing the elements of X jointly. - ----- +------- .. class:: infomark @QUOTA_USAGE_NOTE@ - ]]> + ]]> diff --git a/static/images/tools/collection_ops/merge.svg b/static/images/tools/collection_ops/merge.svg new file mode 100644 index 00000000000..30a14be3b6a --- /dev/null +++ b/static/images/tools/collection_ops/merge.svg @@ -0,0 +1,61 @@ + + + + + + + + + + + + + + + + + + + + + Collection 1 + + + sampleA + + + sampleB + + + + Collection 2 + + + sampleC + + + sampleD + + + + Merge + + + + Merged collection + + + sampleA + + + sampleB + + + sampleC + + + sampleD + + elements concatenated in order + + diff --git a/static/images/tools/collection_ops/nest.svg b/static/images/tools/collection_ops/nest.svg index 4f98aac6a97..1edc7f73fe9 100644 --- a/static/images/tools/collection_ops/nest.svg +++ b/static/images/tools/collection_ops/nest.svg @@ -1,963 +1,63 @@ - - - - - + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - ... - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - ... - + + + + List + + + sample1 + + + sample2 + + + sample3 + + + + Nest + + + + Nested list + + + + sample1 + + + sample1 + + + + sample2 + + + sample2 + + + + sample3 + + + sample3 + + each element wrapped in its own list + From 54e1ee3e2a0759a42ed57fb779fc2267331bcba0 Mon Sep 17 00:00:00 2001 From: nekrut Date: Mon, 2 Mar 2026 02:22:06 +0000 Subject: [PATCH 13/15] Expand help text and add/replace SVGs for relabel, sort, split, tag, unzip, and zip tools Co-Authored-By: Claude Opus 4.6 --- lib/galaxy/tools/relabel_from_file.xml | 70 +- lib/galaxy/tools/sort_collection_list.xml | 54 +- .../tools/split_paired_and_unpaired.xml | 44 +- lib/galaxy/tools/tag_collection_from_file.xml | 80 +- lib/galaxy/tools/unzip_collection.xml | 30 +- lib/galaxy/tools/zip_collection.xml | 31 +- .../images/tools/collection_ops/relabel.svg | 55 + static/images/tools/collection_ops/sort.svg | 54 + .../collection_ops/split_paired_unpaired.svg | 70 + .../tools/collection_ops/tag_from_file.svg | 61 + static/images/tools/collection_ops/unzip.svg | 1164 ++--------------- static/images/tools/collection_ops/zip.svg | 1164 ++--------------- 12 files changed, 623 insertions(+), 2254 deletions(-) create mode 100644 static/images/tools/collection_ops/relabel.svg create mode 100644 static/images/tools/collection_ops/sort.svg create mode 100644 static/images/tools/collection_ops/split_paired_unpaired.svg create mode 100644 static/images/tools/collection_ops/tag_from_file.svg diff --git a/lib/galaxy/tools/relabel_from_file.xml b/lib/galaxy/tools/relabel_from_file.xml index 7fff171e77a..156737d26cb 100644 --- a/lib/galaxy/tools/relabel_from_file.xml +++ b/lib/galaxy/tools/relabel_from_file.xml @@ -182,71 +182,71 @@ Synopsis ======== -Changes identifiers of datasets within a collection using identifiers from a supplied file. +Renames collection element identifiers using a mapping from a file. =========== Description =========== -New identifiers can be supplied as either a simple list or a tab-delimited file mapping old identifiers to new ones. This is controlled using **How should the new identifiers be specified?** drop-down: +This tool changes the element identifiers (names) in a collection without modifying the datasets themselves. The new identifiers are read from a file you provide. -**Use lines in a simple text file as new identifiers** +Three mapping modes are available: -Given a collection:: +======== +Examples +======== - Collection: [Dataset A] - [Dataset B] - [Dataset X] +**Simple text file — positional renaming** -and a simple text file:: +Each line in the text file replaces the identifier at the same position in the collection. Given a collection:: - Alpha - Beta - Gamma + List: sampleA, sampleB, sampleC -the tool will return:: +and a text file:: - Collection: [Dataset Alpha] - [Dataset Beta] - [Dataset Gamma] + Alpha + Beta + Gamma + +the output is:: + + List: Alpha, Beta, Gamma .. class:: infomark -**Note** that the order and number of entries in the text file must match the order of the items you want to rename in your dataset collection. +The text file must have exactly as many lines as the collection has elements (when **Ensure strict mapping** is enabled). ------- -**Map original identifiers to new ones using a two-column table** +**Two-column tabular file — lookup renaming** -Given a collection:: +The first column contains the current identifier, the second column the new one. Order does not matter:: - Collection: [Dataset A] - [Dataset B] - [Dataset X] + sampleB Beta + sampleC Gamma + sampleA Alpha -and a simple tabular file (you can see that entries do not have to be in order here):: +Output:: - B Beta - X Gamma - A Alpha - -the tool will return:: - - Collection: [Dataset Alpha] - [Dataset Beta] - [Dataset Gamma] + List: Alpha, Beta, Gamma ------- -**Use any two columns of a table for identifier mapping** +**Extended tabular — choose which columns to use** -This mode works exactly as the previous one, but the tabular mapping file is allowed to have more than two columns, and you can specify which of them holds the original and new element identifiers, respectively. +Works like the two-column mode, but the file may have additional columns. You select which column holds the original identifiers and which holds the new ones. ------ +------- + +.. image:: ${static_path}/images/tools/collection_ops/relabel.svg + :alt: Relabel collection element identifiers from a file + :width: 500 + +------- .. class:: warningmark -Valid identifiers must contain only characters (a-z, A-Z), numbers (0-9), dash (-), underscore (_), dot (.), space ( ) and comma (,). Other characters are not allowed. +Valid identifiers must contain only letters (a-z, A-Z), numbers (0-9), dash (-), underscore (_), dot (.), space ( ) and comma (,). Other characters are not allowed. .. class:: infomark diff --git a/lib/galaxy/tools/sort_collection_list.xml b/lib/galaxy/tools/sort_collection_list.xml index e39b2ce6962..630c684b258 100644 --- a/lib/galaxy/tools/sort_collection_list.xml +++ b/lib/galaxy/tools/sort_collection_list.xml @@ -149,53 +149,53 @@ Synopsis ======== -Sorts dataset collection alphabetically, numerically, or using predetermined order from a supplied file. +Reorders elements of a collection alphabetically, numerically, or using a custom order from a file. =========== Description =========== +This tool takes a collection and produces a new collection with the same elements in a different order. The datasets themselves are not modified — only the element ordering changes. + +Three sort modes are available: + +======== +Examples +======== + **Alphabetical sort** -When *alphabetical* sort is chosen, the tool sorts elements by their identifiers in ascending alphabetical order. +Elements are sorted by their identifiers in ascending alphabetical (lexicographic) order:: + + Input: sampleC, sampleA, sampleB + Output: sampleA, sampleB, sampleC ------- **Numeric sort** -When *numeric* sort is chosen, the tool sorts elements in ascending order and ignores non-numeric characters. For example, if a collection contains the following elements:: +Non-numeric characters are stripped from each identifier, and elements are sorted by the resulting number in ascending order:: - Collection: [Horse123] - [Donkey543] - [Mule176] - -The tool will output:: - - Collection: [Horse123] - [Mule176] - [Donkey543] + Input: sample100, sample3, sample20 + Output: sample3, sample20, sample100 ------- -**Sorting from file** +**Sort from file** -Alternatively, you can supply a single-column text file containing element identifiers in the desired sort order. For example, suppose there is a collection:: +You supply a single-column text file listing identifiers in the desired order. The file must contain every element identifier exactly once:: - Collection: [Horse123] - [Donkey543] - [Mule176] + Text file: Output: -and a file specifying sort order:: + sampleB sampleB + sampleC sampleC + sampleA sampleA - Donkey543 - Horse123 - Mule176 - -the output will predictably look like this:: +------- - Collection: [Donkey543] - [Horse123] - [Mule176] +.. image:: ${static_path}/images/tools/collection_ops/sort.svg + :alt: Sort collection elements + :width: 500 ------- @@ -203,5 +203,5 @@ the output will predictably look like this:: @QUOTA_USAGE_NOTE@ - ]]> + ]]> diff --git a/lib/galaxy/tools/split_paired_and_unpaired.xml b/lib/galaxy/tools/split_paired_and_unpaired.xml index 10b7f0e4934..d1d05c88158 100644 --- a/lib/galaxy/tools/split_paired_and_unpaired.xml +++ b/lib/galaxy/tools/split_paired_and_unpaired.xml @@ -121,33 +121,51 @@ Synopsis ======== -Splits a collection into two output collections: one containing paired elements and one containing unpaired (single) elements. +Splits a collection into separate paired and unpaired output collections. =========== Description =========== -This tool is designed to separate mixed collections that may contain both paired-end and single-end datasets. It accepts a ``list``, ``list:paired``, or ``list:paired_or_unpaired`` collection and produces two outputs: +This tool separates a collection that may contain a mix of paired-end and single-end datasets into two output collections: -- **Paired output**: A ``list:paired`` collection containing all elements that have both forward and reverse components. -- **Unpaired output**: A ``list`` collection containing all single (unpaired) elements. +- **Paired output** (``list:paired``) — elements that have both forward and reverse components. +- **Unpaired output** (``list``) — single (unpaired) elements. -For example, given a ``list:paired_or_unpaired`` collection:: +The tool accepts ``list``, ``list:paired``, or ``list:paired_or_unpaired`` input collections. If the input is a plain ``list`` (all unpaired), all elements go to the unpaired output and the paired output is empty. If the input is ``list:paired`` (all paired), all elements go to the paired output and the unpaired output is empty. - Input: [el1: forward + reverse] - [el2: single dataset] - [el3: single dataset] +======== +Examples +======== + +**Split a mixed collection** + +Given a ``list:paired_or_unpaired`` collection:: + + Input: + + sample1 → forward + reverse (paired) + sample2 → single dataset (unpaired) + sample3 → single dataset (unpaired) The tool produces:: - Paired: [el1: forward + reverse] + Paired output (list:paired): - Unpaired: [el2] - [el3] + sample1 → forward + reverse -If the input is a plain ``list`` (all unpaired), all elements go to the unpaired output. If the input is a ``list:paired`` (all paired), all elements go to the paired output. + Unpaired output (list): ------ + sample2 → Dataset 1 + sample3 → Dataset 2 + +------- + +.. image:: ${static_path}/images/tools/collection_ops/split_paired_unpaired.svg + :alt: Split paired and unpaired elements + :width: 620 + +------- .. class:: infomark diff --git a/lib/galaxy/tools/tag_collection_from_file.xml b/lib/galaxy/tools/tag_collection_from_file.xml index 880a49209bf..888846091d2 100644 --- a/lib/galaxy/tools/tag_collection_from_file.xml +++ b/lib/galaxy/tools/tag_collection_from_file.xml @@ -87,27 +87,87 @@ Synopsis ======== -Adds tags (including name: and group: tags) to collection elements. +Assigns tags to collection elements based on a tabular mapping file. =========== Description =========== -The relationship between element names and tags is specified in a two column tab-delimited file. This file may contain fewer entries than elements in the collection. In that case only matching list identifiers will be tagged. +This tool reads a two-column tab-delimited file and applies tags to the matching elements of a collection. The first column contains the element identifier, the second column contains one or more comma-separated tags. -To create name: or group: tags prepend them with ``#`` (you can also use ``name:``) or ``group:``, respectively. +The file does not need to list every element — only elements with a matching identifier will be tagged. Elements not listed keep their existing tags unchanged. -=============== -More about tags -=============== +Three tag update modes are available: -Galaxy allows tagging datasets to facilitate analyses. There are several types of tags including simple tags, name tags, and group tags. **Simple** tags allow you to attach an alternative label to a dataset, which will make it easier to find it later. **Name** tags allow you to track propagation of a dataset through the analyses: all datasets derived from the initial dataset labeled with a name tag will inherit it. Finally, **group** tags allow you to label groups of datasets. This is useful, for example, for differential expression analysis where you can have two groups of datasets labeled as "treatment" and "control". +- **Add** — new tags are added alongside any existing tags. +- **Set** — existing tags are removed and replaced with the new tags. +- **Remove** — the listed tags are removed from matching elements. -To learn more about tags go to `our training site`_. +======== +Examples +======== -.. _our training site: https://training.galaxyproject.org/training-material/search?query=tags +**Add group tags for differential expression** ------ +Given a collection with three samples, and a tag file:: + + sample1 group:treatment + sample2 group:control + sample3 group:treatment + +After running with mode **Add**, each element will carry the specified group tag while keeping any tags it already had. + +------- + +.. image:: ${static_path}/images/tools/collection_ops/tag_from_file.svg + :alt: Tag collection elements from a tabular file + :width: 500 + +========= +Tag types +========= + +Galaxy supports three types of tags: + +**Simple tags** + +A plain keyword such as ``mytag`` or ``important``. Simple tags act as labels that help you find and organize datasets in your history. They do not have any special behavior — they are just searchable labels. In the tag file, write the tag name directly:: + + sample1 important + sample2 batch2,rerun + +------- + +**Name tags** + +A tag prefixed with ``#`` or ``name:`` (e.g. ``#patient42`` or ``name:patient42``). Name tags are *hereditary* — when a dataset with a name tag is used as input to a tool, the output dataset automatically inherits the same name tag. This lets you trace the origin of a dataset through a long analysis. In the tag file:: + + sample1 #patient42 + sample2 name:patient99 + +------- + +**Group tags** + +A tag prefixed with ``group:`` (e.g. ``group:treatment`` or ``group:control``). Group tags are used by tools that compare conditions — for example, **DESeq2** reads group tags to determine which samples belong to which experimental condition. In the tag file:: + + sample1 group:treatment + sample2 group:control + sample3 group:treatment + +------- + +You can combine multiple tags of any type by separating them with commas:: + + sample1 #patient42,group:treatment,batch1 + +------- + +To learn more about tags, visit `the Galaxy Training Network`_. + +.. _the Galaxy Training Network: https://training.galaxyproject.org/training-material/search?query=tags + +------- .. class:: infomark diff --git a/lib/galaxy/tools/unzip_collection.xml b/lib/galaxy/tools/unzip_collection.xml index d3e332c1616..b574f31349e 100644 --- a/lib/galaxy/tools/unzip_collection.xml +++ b/lib/galaxy/tools/unzip_collection.xml @@ -25,19 +25,41 @@ Synopsis ======== -Takes a paired collection and "unzips" it into two simple dataset collections (lists of datasets). +Splits a paired dataset into two individual datasets (forward and reverse). =========== Description =========== -Given a paired collection of forward and reverse reads this tool will "unzip" it into two collections containing forward and reverse reads, respectively: +This tool takes a single ``paired`` collection and extracts its two components — the **forward** and the **reverse** dataset — as separate output datasets. + +This is the inverse of **Zip collections**, which combines two datasets into a pair. + +======== +Examples +======== + +**Unzip a paired collection** + +Input ``paired`` collection:: + + Paired: "Sample 1" + + forward → reads_R1.fastq + reverse → reads_R2.fastq + +Outputs:: + + Forward dataset: reads_R1.fastq + Reverse dataset: reads_R2.fastq + +------- .. image:: ${static_path}/images/tools/collection_ops/unzip.svg - :alt: Unzipping operation + :alt: Unzip a paired collection into forward and reverse datasets :width: 500 ------ +------- .. class:: infomark diff --git a/lib/galaxy/tools/zip_collection.xml b/lib/galaxy/tools/zip_collection.xml index fd2d3f035b1..ee6bc96bfba 100644 --- a/lib/galaxy/tools/zip_collection.xml +++ b/lib/galaxy/tools/zip_collection.xml @@ -28,21 +28,42 @@ Synopsis ======== -Takes two collections and creates a paired collection from them. +Combines two datasets into a single paired collection (forward + reverse). =========== Description =========== -If you have one collection containing only forward reads and one containing only reverse, this tool will "zip" them together into a simple paired collection. For example, given two collections with `forward` and `reverse` reads they can be "zipped" into a single paired collection: +This tool takes two individual datasets and packages them into a ``paired`` collection, labeling the first as **forward** and the second as **reverse**. + +This is the inverse of **Unzip collection**, which splits a pair back into two separate datasets. + +======== +Examples +======== + +**Zip two datasets into a pair** + +Inputs:: + + Input 1 (forward): reads_R1.fastq + Input 2 (reverse): reads_R2.fastq + +Output ``paired`` collection:: + + Paired: "reads (zipped)" + + forward → reads_R1.fastq + reverse → reads_R2.fastq + +------- .. image:: ${static_path}/images/tools/collection_ops/zip.svg - :alt: Zipping operation + :alt: Zip two datasets into a paired collection :width: 500 +------- ------ - .. class:: infomark @QUOTA_USAGE_NOTE@ diff --git a/static/images/tools/collection_ops/relabel.svg b/static/images/tools/collection_ops/relabel.svg new file mode 100644 index 00000000000..bdec46fa689 --- /dev/null +++ b/static/images/tools/collection_ops/relabel.svg @@ -0,0 +1,55 @@ + + + + + + + + + + + + + + + + + + + + + List + + + sampleA + + + sampleB + + + sampleC + + + + sampleA Alpha + sampleB Beta + sampleC Gamma + + + + Relabel + + + + List (relabelled) + + + Alpha + + + Beta + + + Gamma + + diff --git a/static/images/tools/collection_ops/sort.svg b/static/images/tools/collection_ops/sort.svg new file mode 100644 index 00000000000..e189ee649b6 --- /dev/null +++ b/static/images/tools/collection_ops/sort.svg @@ -0,0 +1,54 @@ + + + + + + + + + + + + + + + + + + + + + List + + + sampleC + + + sampleA + + + sampleB + + unsorted + + + + Sort + alphabetical + + + + List (sorted) + + + sampleA + + + sampleB + + + sampleC + + A, B, C order + + diff --git a/static/images/tools/collection_ops/split_paired_unpaired.svg b/static/images/tools/collection_ops/split_paired_unpaired.svg new file mode 100644 index 00000000000..63de001f9ea --- /dev/null +++ b/static/images/tools/collection_ops/split_paired_unpaired.svg @@ -0,0 +1,70 @@ + + + + + + + + + + + + + + + + + + + + + list : paired_or_unpaired + + + + sample1 (paired) + + + forward + + + reverse + + + + sample2 (unpaired) + + + + sample3 (unpaired) + + mixed paired + unpaired + + + + Split + + + + Paired output + + + sample1 + + + forward + + + reverse + + + + Unpaired output + + + sample2 + + + sample3 + + diff --git a/static/images/tools/collection_ops/tag_from_file.svg b/static/images/tools/collection_ops/tag_from_file.svg new file mode 100644 index 00000000000..6d14379b82a --- /dev/null +++ b/static/images/tools/collection_ops/tag_from_file.svg @@ -0,0 +1,61 @@ + + + + + + + + + + + + + + + + + + + + + List + + + sample1 + + + sample2 + + + sample3 + + + + sample1 group:treat + sample2 group:ctrl + sample3 group:treat + + + + Tag + + + + List (tagged) + + + sample1 + + treat + + + sample2 + + ctrl + + + sample3 + + treat + + diff --git a/static/images/tools/collection_ops/unzip.svg b/static/images/tools/collection_ops/unzip.svg index 25047d07b37..6f93499de90 100644 --- a/static/images/tools/collection_ops/unzip.svg +++ b/static/images/tools/collection_ops/unzip.svg @@ -1,1081 +1,85 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + Paired collection + + + + sample1 + + + forward + + + reverse + + + + sample2 + + + forward + + + reverse + + + + sample3 + + + forward + + + reverse + + 3 paired elements + + + + Unzip + + + + Forward datasets + + + sample1_R1.fastq + + + sample2_R1.fastq + + + sample3_R1.fastq + + + + Reverse datasets + + + sample1_R2.fastq + + + sample2_R2.fastq + + + sample3_R2.fastq + diff --git a/static/images/tools/collection_ops/zip.svg b/static/images/tools/collection_ops/zip.svg index 1b037278606..b2a49b8fdc1 100644 --- a/static/images/tools/collection_ops/zip.svg +++ b/static/images/tools/collection_ops/zip.svg @@ -1,1081 +1,85 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + Forward datasets + + + sample1_R1.fastq + + + sample2_R1.fastq + + + sample3_R1.fastq + + + + Reverse datasets + + + sample1_R2.fastq + + + sample2_R2.fastq + + + sample3_R2.fastq + + + + Zip + + + + Paired collection + + + + sample1 + + + forward + + + reverse + + + + sample2 + + + forward + + + reverse + + + + sample3 + + + forward + + + reverse + + matched by position + From ce2d59d6045c110d65f05f45217cef8951784154 Mon Sep 17 00:00:00 2001 From: nekrut Date: Mon, 2 Mar 2026 02:34:28 +0000 Subject: [PATCH 14/15] Expand help text and add SVGs for apply rules, flat cross product, and nested cross product tools Co-Authored-By: Claude Opus 4.6 --- lib/galaxy/tools/apply_rules.xml | 45 ++++-- lib/galaxy/tools/cross_product_flat.xml | 106 +++++++-------- lib/galaxy/tools/cross_product_nested.xml | 128 ++++++++++-------- .../tools/collection_ops/apply_rules.svg | 64 +++++++++ .../collection_ops/cross_product_flat.svg | 91 +++++++++++++ .../collection_ops/cross_product_nested.svg | 102 ++++++++++++++ 6 files changed, 416 insertions(+), 120 deletions(-) create mode 100644 static/images/tools/collection_ops/apply_rules.svg create mode 100644 static/images/tools/collection_ops/cross_product_flat.svg create mode 100644 static/images/tools/collection_ops/cross_product_nested.svg diff --git a/lib/galaxy/tools/apply_rules.xml b/lib/galaxy/tools/apply_rules.xml index 0f158e36041..a0677fe7d59 100644 --- a/lib/galaxy/tools/apply_rules.xml +++ b/lib/galaxy/tools/apply_rules.xml @@ -27,28 +27,53 @@ Synopsis ======== -This tool allows one to process an existing Galaxy dataset collection's metadata as tabular data, -apply a series of rules to it, and generate a new collection. +Reorganizes a collection by applying a chain of rules (filter, sort, rename, nest, flatten, tag) through an interactive rule builder. =========== Description =========== -When used interactively in the tool form, a dynamic preview of the processing will be available in a tabular data viewer but this tool -may be used in workflows as well where no such preview can be generated. +This tool opens an interactive rule builder that lets you manipulate the element identifiers (and nesting structure) of a collection as if they were columns in a spreadsheet. You can chain multiple operations together — for example, split an identifier on a delimiter, use one part as a new nesting level and another as the element name, filter rows by a pattern, and sort the result — all in a single step. -This tool is an advanced feature but has a lot of flexibility - it can be used to process collections with arbitrary nesting and can do many kinds of filtering, re-sorting, nesting, flattening, and arbitrary combinations thereof not possible with Galaxy's other, more simple -collection operation tools. +This is an advanced tool that combines the capabilities of many simpler collection operations (sort, relabel, nest, flatten, filter) into one flexible interface. Use it when the simpler tools cannot achieve the reorganization you need. -More information about the rule processor in general can be found at `our training site`_. +======== +Examples +======== -.. _our training site: https://training.galaxyproject.org/training-material/search?query=rule+builder +**Flatten and re-nest by a different criterion** ------ +Given a ``list:list`` collection where the outer identifiers are sequencing lanes and inner identifiers are sample names, you could use the rule builder to re-nest so that the outer level is sample name and the inner level is lane — effectively transposing the collection structure. + +------- + +**Extract metadata from identifiers** + +If your element identifiers encode multiple pieces of information (e.g. ``patient01_treatment_rep2``), you can split on ``_`` to create columns, then use those columns to define new nesting levels, group tags, or filters. + +------- + +**Assign tags based on identifier patterns** + +You can add a column derived from an identifier (e.g. extract "treatment" or "control" from the name) and assign it as a group tag, all within the rule builder. + +------- + +.. image:: ${static_path}/images/tools/collection_ops/apply_rules.svg + :alt: Apply rules to reorganize a collection + :width: 500 + +------- + +For detailed tutorials on the rule builder, visit `the Galaxy Training Network`_. + +.. _the Galaxy Training Network: https://training.galaxyproject.org/training-material/search?query=rule+builder + +------- .. class:: infomark @QUOTA_USAGE_NOTE@ - ]]> + ]]> diff --git a/lib/galaxy/tools/cross_product_flat.xml b/lib/galaxy/tools/cross_product_flat.xml index a4cf09edb01..6f31505b439 100644 --- a/lib/galaxy/tools/cross_product_flat.xml +++ b/lib/galaxy/tools/cross_product_flat.xml @@ -72,72 +72,72 @@ Synopsis ======== -@CROSS_PRODUCT_INTRO@ +Creates two flat lists that, when processed element-wise by a downstream tool, produce every pairwise combination (all-vs-all) of the two input collections. -==================== -How to use this tool -==================== +=========== +Description +=========== -@GALAXY_DOT_PRODUCT_SEMANTICS@ +This tool takes two list collections and produces two output lists that encode the **Cartesian product** (cross product) of the inputs. Both output lists have the same length (``n * m`` elements, where ``n`` and ``m`` are the sizes of the inputs) and the same element identifiers. -Running input lists through this tool produces new dataset lists (described in detail below) that when using -the same natural element-wise matching "map over" semantics described above produce every combination of the -elements of the two lists compared against each other. Running a tool with these two outputs instead of the initial -two inputs produces a list of the comparison of each combination of pairs from the respective inputs. +When you feed these two outputs into a tool that operates on pairs of datasets, Galaxy's element-wise matching ensures that every combination of one element from Input A and one element from Input B is compared exactly once. -.. image:: ${static_path}/images/tools/collection_ops/flat_crossproduct_output.png - :alt: The Flat Cartesian Product of Two Collections +======== +Examples +======== + +**All-vs-all comparison** + +Given:: + + Input A: [a1] [a2] + Input B: [b1] [b2] + +The tool produces two flat lists of length 4:: + + Output A: [a1_b1 → a1] [a1_b2 → a1] [a2_b1 → a2] [a2_b2 → a2] + Output B: [a1_b1 → b1] [a1_b2 → b2] [a2_b1 → b1] [a2_b2 → b2] + +Feeding both outputs to a comparison tool produces:: + + Result: [a1_b1 → compare(a1,b1)] + [a1_b2 → compare(a1,b2)] + [a2_b1 → compare(a2,b1)] + [a2_b2 → compare(a2,b2)] + +The identifier separator (default ``_``) can be changed to ``:`` or ``-``. + +------- + +.. image:: ${static_path}/images/tools/collection_ops/cross_product_flat.svg + :alt: Flat cross product of two collections + :width: 620 + +------- + +================================== +How element-wise matching works +================================== + +Normally, when you connect two lists to a tool that takes two inputs, Galaxy matches them element-wise (dot product): the first element of list 1 is paired with the first element of list 2, and so on. This requires both lists to have the same identifiers. + +.. image:: ${static_path}/images/tools/collection_ops/dot_product.png + :alt: Element-wise (dot product) matching :width: 500 -The result of running a subsequent tool with the outputs produced by this tool will be a much larger list -whose element identifiers are the concatenation of the combinations of the elements identifiers from the -two input lists. +This tool rearranges the inputs so that the same element-wise matching mechanism produces every combination instead. -.. image:: ${static_path}/images/tools/collection_ops/flat_crossproduct_separator.png - :alt: Flat Cross Product Identifier Separator - :width: 500 +------- -============================================ -What this tool does (technical details) -============================================ +.. class:: infomark -This tool consumes two lists - we will call them ``input_a`` and ``input_b``. If ``input_a`` -has length ``n`` and dataset elements identified as ``a1``, ``a2``, ... ``an`` and ``input_b`` -has length ``m`` and dataset elements identified as ``b1``, ``b2``, ... ``bm``, then this tool -produces a pair of larger lists - each of size ``n*m``. +For a nested (``list:list``) output structure instead of a flat list, use **Nested Cross Product**. -Both output lists will be the same length and contain the same set of element identifiers in the -same order. If the kth input can be described as ``(i-1)*n + (j-1)`` where ``1 <= i <= m`` and ``1 <= j <= n`` -then the element identifier for this kth element is the concatenation of the element identifier for -the ith item of ``input_a`` and the jth item of ``input_b``. - -In the first output list, this kth element will be the ith element of ``input_a``. In the second -output list, the kth element will be the jth element of ``input_b``. - -.. image:: ${static_path}/images/tools/collection_ops/flat_cross_product_outputs.png - :alt: Flat Cross Product Outputs - :width: 500 - -These list structures might appear to be a little odd, but they have the very useful property -that if you match up corresponding elements of the lists the result is each combination of -elements in ``input_a`` and ``input_b`` are matched up once. - -.. image:: ${static_path}/images/tools/collection_ops/flat_cross_product_matched.png - :alt: Flat Cross Product Matching Datasets - :width: 500 - -Running a downstream comparison tool that compares two datasets with these two lists produces a -new list with every combination of comparisons. - -.. image:: ${static_path}/images/tools/collection_ops/flat_cross_product_downstream.png - :alt: Flat Cross Product All-vs-All Result - :width: 500 - ----- +------- .. class:: infomark @QUOTA_USAGE_NOTE@ - ]]> + ]]> diff --git a/lib/galaxy/tools/cross_product_nested.xml b/lib/galaxy/tools/cross_product_nested.xml index 46f0bef7bd6..f389d9a18ff 100644 --- a/lib/galaxy/tools/cross_product_nested.xml +++ b/lib/galaxy/tools/cross_product_nested.xml @@ -76,75 +76,89 @@ Synopsis ======== -@CROSS_PRODUCT_INTRO@ +Creates two nested lists (``list:list``) that, when processed element-wise by a downstream tool, produce every pairwise combination (all-vs-all) of the two input collections, organized by the first input. -==================== -How to use this tool -==================== +=========== +Description +=========== -@GALAXY_DOT_PRODUCT_SEMANTICS@ +This tool takes two flat list collections and produces two ``list:list`` output collections that encode the **Cartesian product** (cross product) of the inputs. The outer list has one element per element of Input A, and each inner list has one element per element of Input B. -Running input lists through this tool produces new list structures (described in detail below) that when using -the same natural element-wise matching "map over" semantics described above produce every combination of the -elements of the two lists compared against each other. Running a tool with these two outputs instead of the initial -two inputs produces a nested list structure where the jth element of the inner list of the ith element of the outer -list is a comparison of the ith element of the first list to the jth element of the second list. -Put more simply, the result is a nested list where the identifiers of an element describe which inputs were -matched to produce the comparison output found at that element. +When you feed these two outputs into a tool that takes two inputs, Galaxy matches elements at each nesting level. The result is a nested list where the outer identifier tells you which element from Input A was used, and the inner identifier tells you which element from Input B was used. -.. image:: ${static_path}/images/tools/collection_ops/nested_crossproduct_output.png - :alt: The Cartesian Product of Two Collections +This is similar to **Flat Cross Product**, but the output preserves structure: results are grouped by the first input, making it easier to see all comparisons for a given element of Input A. + +======== +Examples +======== + +**All-vs-all comparison with nested output** + +Given:: + + Input A: [a1] [a2] + Input B: [b1] [b2] + +The tool produces two ``list:list`` collections:: + + Output A (list:list): + + a1/ + b1 → a1 + b2 → a1 + a2/ + b1 → a2 + b2 → a2 + + Output B (list:list): + + a1/ + b1 → b1 + b2 → b2 + a2/ + b1 → b1 + b2 → b2 + +Feeding both outputs to a comparison tool produces:: + + Result (list:list): + + a1/ + b1 → compare(a1, b1) + b2 → compare(a1, b2) + a2/ + b1 → compare(a2, b1) + b2 → compare(a2, b2) + +------- + +.. image:: ${static_path}/images/tools/collection_ops/cross_product_nested.svg + :alt: Nested cross product of two collections + :width: 620 + +------- + +================================== +How element-wise matching works +================================== + +Normally, when you connect two lists to a tool, Galaxy matches them element-wise (dot product). This tool rearranges the inputs so that the same matching mechanism produces every combination instead, with results grouped into a nested structure. + +.. image:: ${static_path}/images/tools/collection_ops/dot_product.png + :alt: Element-wise (dot product) matching :width: 500 -============================================ -What this tool does (technical details) -============================================ +------- -This tool consumes two flat lists. We will call the input collections ``input_a`` and ``input_b``. If ``input_a`` -has length ``n`` and dataset elements identified as ``a1``, ``a2``, ... ``an`` and ``input_b`` -has length ``m`` and dataset elements identified as ``b1``, ``b2``, ... ``bm``, then this tool -produces a pair of output nested lists (specifically of the ``list:list`` collection type) where -the outer list is of length ``n`` and each inner list has a length of ``m`` (a ``n X m`` nested list). The jth element -inside the outer list's ith element is a pseudo copy of the ith dataset of ``input_a``. One -way to think about the output nested lists is as matrices. Here is a diagram of the first output -showing the element identifiers of the outer and inner lists along with dataset identifiers being -"copied" into this new collection. +.. class:: infomark -.. image:: ${static_path}/images/tools/collection_ops/nested_cross_product_out_1.png - :alt: Nested Cross Product First Output - :width: 500 +For a flat output list instead of nested, use **Flat Cross Product**. -The second output is a nested list of pseudo copies of the elements of ``input_b`` instead of -``input_a``. In particular the outer list is again of length ``n`` and each inner list is again -of length ``m`` but this time the jth element inside the outer list's ith element is a pseudo copy -of the jth dataset of ``input_b``. Here is the matrix of these outputs. - -.. image:: ${static_path}/images/tools/collection_ops/nested_cross_product_out_2.png - :alt: Nested Cross Product Second Output - :width: 500 - -These nested list structures might appear to be a little odd, but they have the very useful property -that if you match up corresponding elements of the nested lists the result is each combination of -elements in ``input_a`` and ``input_b`` are matched up once. The following diagram describes these matching -datasets. - -.. image:: ${static_path}/images/tools/collection_ops/nested_cross_product_matching.png - :alt: Matching Inputs - :width: 500 - -Running a tool that compares two datasets with these two nested lists produces a new nested list -as described above. The following diagram shows the structure of this output and how the element -identifiers are preserved and indicate what comparison was performed. - -.. image:: ${static_path}/images/tools/collection_ops/nested_cross_product_output.png - :alt: Matching Inputs - :width: 500 - ----- +------- .. class:: infomark @QUOTA_USAGE_NOTE@ - ]]> + ]]> diff --git a/static/images/tools/collection_ops/apply_rules.svg b/static/images/tools/collection_ops/apply_rules.svg new file mode 100644 index 00000000000..402aadaf88f --- /dev/null +++ b/static/images/tools/collection_ops/apply_rules.svg @@ -0,0 +1,64 @@ + + + + + + + + + + + + + + + + + + + + + List + + + patientA_treat_r1 + + + patientA_ctrl_r1 + + + patientB_treat_r1 + + + + Rule Builder + + + + Apply rules + split, nest, tag + + + + Nested list (re-organized) + + + + patientA + + + treat_r1 + + + ctrl_r1 + + + + patientB + + + treat_r1 + + split on "_", nest by patient + + diff --git a/static/images/tools/collection_ops/cross_product_flat.svg b/static/images/tools/collection_ops/cross_product_flat.svg new file mode 100644 index 00000000000..485ebcc7714 --- /dev/null +++ b/static/images/tools/collection_ops/cross_product_flat.svg @@ -0,0 +1,91 @@ + + + + + + + + + + + + + + + + + + + + + Input A + + + a1 + + + a2 + + + + Input B + + + b1 + + + b2 + + + + Cross + product + + + + Output A + + + a1_b1 → a1 + + + a1_b2 → a1 + + + a2_b1 → a2 + + + a2_b2 → a2 + + + + Output B + + + a1_b1 → b1 + + + a1_b2 → b2 + + + a2_b1 → b1 + + + a2_b2 → b2 + + + Match element-wise + to get all-vs-all + + + + + + + + a1_b1: compare(a1,b1) + a1_b2: compare(a1,b2) + a2_b1: compare(a2,b1) + a2_b2: compare(a2,b2) + + diff --git a/static/images/tools/collection_ops/cross_product_nested.svg b/static/images/tools/collection_ops/cross_product_nested.svg new file mode 100644 index 00000000000..bed7697efa6 --- /dev/null +++ b/static/images/tools/collection_ops/cross_product_nested.svg @@ -0,0 +1,102 @@ + + + + + + + + + + + + + + + + + + + + + Input A + + + a1 + + + a2 + + + + Input B + + + b1 + + + b2 + + + + Cross + product + + + + Output A (list:list) + + + + a1 + + + b1 → a1 + + + b2 → a1 + + + + a2 + + + b1 → a2 + + + b2 → a2 + + + + Output B (list:list) + + + + a1 + + + b1 → b1 + + + b2 → b2 + + + + a2 + + + b1 → b1 + + + b2 → b2 + + + Results grouped + by Input A element + + + a1/b1: compare(a1,b1) + a1/b2: compare(a1,b2) + a2/b1: compare(a2,b1) + a2/b2: compare(a2,b2) + + From ae93a58d9aef08d4a3c3936e396772b7d89a0f09 Mon Sep 17 00:00:00 2001 From: nekrut Date: Mon, 2 Mar 2026 02:43:52 +0000 Subject: [PATCH 15/15] Add subcategory labels to Collection Operations tool section Organize the flat list of ~30 collection tools into 6 labeled subcategories (Build, Restructure, Filter, Pair and Combine, Sort and Label, Extract) to help users find tools faster. Co-Authored-By: Claude Opus 4.6 --- lib/galaxy/config/sample/tool_conf.xml.sample | 41 ++++++++++++------- 1 file changed, 26 insertions(+), 15 deletions(-) diff --git a/lib/galaxy/config/sample/tool_conf.xml.sample b/lib/galaxy/config/sample/tool_conf.xml.sample index 5f6368a262e..80d349011ef 100644 --- a/lib/galaxy/config/sample/tool_conf.xml.sample +++ b/lib/galaxy/config/sample/tool_conf.xml.sample @@ -29,8 +29,19 @@
- - + + + + + + + + + + + + + @@ -39,25 +50,25 @@ - - - - - - + + + + + + + + + - - - - - + + - - + +