diff --git a/doc/source/dev/collection_semantics.md b/doc/source/dev/collection_semantics.md index 1e3ad18695d..55ef137a190 100644 --- a/doc/source/dev/collection_semantics.md +++ b/doc/source/dev/collection_semantics.md @@ -28,21 +28,22 @@ mapped over. Each element of the implicit collections correspond to their own jo Galaxy very naturally and intuitively parallelizes jobs without extra work from the user and without any knowledge of the tool. + (BASIC_MAPPING_PAIRED)= (BASIC_MAPPING_PAIRED_OR_UNPAIRED_PAIRED)= (BASIC_MAPPING_PAIRED_OR_UNPAIRED_UNPAIRED)= (BASIC_MAPPING_LIST)= -
Examples -:::{admonition} Example: `BASIC_MAPPING_PAIRED` +:::{admonition} Example: `BASIC_MAPPING_PAIRED` :class: note Assuming, -- $ d_f $, $ d_r $ are datasets -- $ tool \text{ is } (i: \text{ dataset }) \Rightarrow \{ o: \text{ dataset } \} $ -- $ C $ is $ \text{CollectionInstance<}paired,\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\}\text{>} $ +* $ d_f $, $ d_r $ are datasets +* $ tool \text{ is } (i: \text{ dataset }) \Rightarrow \{ o: \text{ dataset } \} $ +* $ C $ is $ \text{CollectionInstance<}\text{paired},\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\}\text{>} $ + then @@ -50,14 +51,17 @@ $$tool(i=\text{mapOver}(C)) \mapsto \left\{o: \text{collection}<\text{paired},\l ::: -:::{admonition} Example: `BASIC_MAPPING_PAIRED_OR_UNPAIRED_PAIRED` + + +:::{admonition} Example: `BASIC_MAPPING_PAIRED_OR_UNPAIRED_PAIRED` :class: note Assuming, -- $ d_f $, $ d_r $ are datasets -- $ tool \text{ is } (i: \text{ dataset }) \Rightarrow \{ o: \text{ dataset } \} $ -- $ C $ is $ \text{CollectionInstance<}paired_or_unpaired,\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\}\text{>} $ +* $ d_f $, $ d_r $ are datasets +* $ tool \text{ is } (i: \text{ dataset }) \Rightarrow \{ o: \text{ dataset } \} $ +* $ C $ is $ \text{CollectionInstance<}\text{paired\_or\_unpaired},\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\}\text{>} $ + then @@ -65,14 +69,17 @@ $$tool(i=\text{mapOver}(C)) \mapsto \left\{o: \text{collection}<\text{paired\_or ::: -:::{admonition} Example: `BASIC_MAPPING_PAIRED_OR_UNPAIRED_UNPAIRED` + + +:::{admonition} Example: `BASIC_MAPPING_PAIRED_OR_UNPAIRED_UNPAIRED` :class: note Assuming, -- $ d_u $ is a dataset -- $ tool \text{ is } (i: \text{ dataset }) \Rightarrow \{ o: \text{ dataset } \} $ -- $ C $ is $ \text{CollectionInstance<}paired_or_unpaired,\left\{ \text{ unpaired }=d_u \right\}\text{>} $ +* $ d_u $ is a dataset +* $ tool \text{ is } (i: \text{ dataset }) \Rightarrow \{ o: \text{ dataset } \} $ +* $ C $ is $ \text{CollectionInstance<}\text{paired\_or\_unpaired},\left\{ \text{ unpaired }=d_u \right\}\text{>} $ + then @@ -80,14 +87,17 @@ $$tool(i=\text{mapOver}(C)) \mapsto \left\{o: \text{collection}<\text{paired\_or ::: -:::{admonition} Example: `BASIC_MAPPING_LIST` + + +:::{admonition} Example: `BASIC_MAPPING_LIST` :class: note Assuming, -- $ d_1,...,d_n $ are datasets -- $ tool \text{ is } (i: \text{ dataset }) \Rightarrow \{ o: \text{ dataset } \} $ -- $ C $ is $ \text{CollectionInstance<}list,\left\{ \text{ i1 }=d_1, ..., \text{ in }=d_n \right\}\text{>} $ +* $ d_1,...,d_n $ are datasets +* $ tool \text{ is } (i: \text{ dataset }) \Rightarrow \{ o: \text{ dataset } \} $ +* $ C $ is $ \text{CollectionInstance<}\text{list},\left\{ \text{ i1 }=d_1, ..., \text{ in }=d_n \right\}\text{>} $ + then @@ -95,24 +105,29 @@ $$tool(i=\text{mapOver}(C)) \mapsto \left\{o: \text{collection}<\text{list},\lef ::: + +

+ + The above description of mapping over inputs works naturally and as expected for nested collections. + (NESTED_LIST_MAPPING)= (BASIC_MAPPING_LIST_PAIRED_OR_UNPAIRED)= -
Examples -:::{admonition} Example: `NESTED_LIST_MAPPING` +:::{admonition} Example: `NESTED_LIST_MAPPING` :class: note Assuming, -- $ d_1,...,d_n $ are datasets -- $ tool \text{ is } (i: \text{ dataset }) \Rightarrow \{ o: \text{ dataset } \} $ -- $ C $ is $ \text{CollectionInstance<}list:list,\left\{ \text{ o1 }=\left\{ \text{ inner }=d_1 \right\}, ..., \text{ on }=\left\{ \text{ inner }=d_n \right\} \right\}\text{>} $ +* $ d_1,...,d_n $ are datasets +* $ tool \text{ is } (i: \text{ dataset }) \Rightarrow \{ o: \text{ dataset } \} $ +* $ C $ is $ \text{CollectionInstance<}\text{list}:\text{list},\left\{ \text{ o1 }=\left\{ \text{ inner }=d_1 \right\}, ..., \text{ on }=\left\{ \text{ inner }=d_n \right\} \right\}\text{>} $ + then @@ -120,14 +135,17 @@ $$tool(i=\text{mapOver}(C)) \mapsto \left\{o: \text{collection}<\text{list}:\tex ::: -:::{admonition} Example: `BASIC_MAPPING_LIST_PAIRED_OR_UNPAIRED` + + +:::{admonition} Example: `BASIC_MAPPING_LIST_PAIRED_OR_UNPAIRED` :class: note Assuming, -- $ d_f $, $ d_r $ are datasets -- $ tool \text{ is } (i: \text{ dataset }) \Rightarrow \{ o: \text{ dataset } \} $ -- $ C $ is $ \text{CollectionInstance<}list:paired_or_unpaired,\left\{ \text{ el1 }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\} \right\}\text{>} $ +* $ d_f $, $ d_r $ are datasets +* $ tool \text{ is } (i: \text{ dataset }) \Rightarrow \{ o: \text{ dataset } \} $ +* $ C $ is $ \text{CollectionInstance<}\text{list}:\text{paired\_or\_unpaired},\left\{ \text{ el1 }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\} \right\}\text{>} $ + then @@ -135,24 +153,30 @@ $$tool(i=\text{mapOver}(C)) \mapsto \left\{o: \text{collection}<\text{list}:\tex ::: + +

+ + + For tools with multiple data inputs, the tool can be executed with individual datasets for the non-mapped over input and each tool execution will just be executed with that dataset. The dataset not mapped over serves as the input for each execution. -(BASIC_MAPPING_INCLUDING_SINGLE_DATASET)= +(BASIC_MAPPING_INCLUDING_SINGLE_DATASET)=
Examples -:::{admonition} Example: `BASIC_MAPPING_INCLUDING_SINGLE_DATASET` +:::{admonition} Example: `BASIC_MAPPING_INCLUDING_SINGLE_DATASET` :class: note Assuming, -- $ d_1,...,d_n $, $ d_o $ are datasets -- $ tool \text{ is } (i: \text{ dataset }, i2: \text{ dataset }) \Rightarrow \{ o: \text{ dataset } \} $ -- $ C $ is $ \text{CollectionInstance<}list,\left\{ \text{ i1 }=d_1, ..., \text{ in }=d_n \right\}\text{>} $ +* $ d_1,...,d_n $, $ d_o $ are datasets +* $ tool \text{ is } (i: \text{ dataset }, i2: \text{ dataset }) \Rightarrow \{ o: \text{ dataset } \} $ +* $ C $ is $ \text{CollectionInstance<}\text{list},\left\{ \text{ i1 }=d_1, ..., \text{ in }=d_n \right\}\text{>} $ + then @@ -160,8 +184,12 @@ $$tool(i=\text{mapOver}(C), i2=d_o) \mapsto \left\{o: \text{collection}<\text{li ::: + +

+ + If a tool consumes two input datasets and produces one output dataset, you can map two collections with identical structure (same element identifiers in the same order) over the respective inputs and the result is an implicit collection with the same structure @@ -178,19 +206,20 @@ of map over operations on tools - the results will all continue to match and wor very naturally - again without extra work by the user and without extra knowledge by the tool author. -(BASIC_MAPPING_TWO_INPUTS_WITH_IDENTICAL_STRUCTURE)= +(BASIC_MAPPING_TWO_INPUTS_WITH_IDENTICAL_STRUCTURE)=
Examples -:::{admonition} Example: `BASIC_MAPPING_TWO_INPUTS_WITH_IDENTICAL_STRUCTURE` +:::{admonition} Example: `BASIC_MAPPING_TWO_INPUTS_WITH_IDENTICAL_STRUCTURE` :class: note Assuming, -- $ d1_1,...,d1_n $, $ d2_1,...,d2_n $ are datasets -- $ tool \text{ is } (i: \text{ dataset }, i2: \text{ dataset }) \Rightarrow \{ o: \text{ dataset } \} $ -- $ C1 $ is $ \text{CollectionInstance<}list,\left\{ \text{ i1 }=d1_1, ..., \text{ in }=d1_n \right\}\text{>} $ -- $ C2 $ is $ \text{CollectionInstance<}list,\left\{ \text{ i1 }=d2_1, ..., \text{ in }=d2_n \right\}\text{>} $ +* $ d1_1,...,d1_n $, $ d2_1,...,d2_n $ are datasets +* $ tool \text{ is } (i: \text{ dataset }, i2: \text{ dataset }) \Rightarrow \{ o: \text{ dataset } \} $ +* $ C1 $ is $ \text{CollectionInstance<}\text{list},\left\{ \text{ i1 }=d1_1, ..., \text{ in }=d1_n \right\}\text{>} $ +* $ C2 $ is $ \text{CollectionInstance<}\text{list},\left\{ \text{ i1 }=d2_1, ..., \text{ in }=d2_n \right\}\text{>} $ + then @@ -198,33 +227,38 @@ $$tool(i=\text{mapOver}(C1), i2=\text{mapOver}(C2)) \mapsto \left\{o: \text{coll ::: + +

+ + ## Reduction Not all tool executions result in implicit collections and mapping -over inputs. Tool inputs of `type` `data_collection` can consume +over inputs. Tool inputs of ``type`` ``data_collection`` can consume collections directly and do not necessarily result in mapping over. Tools that consume collections and output datasets effectively reduce the dimension of the Galaxy data structure. When used at runtime this is often referred to as a "reduction" in the code. + (COLLECTION_INPUT_PAIRED)= (COLLECTION_INPUT_LIST)= (COLLECTION_INPUT_PAIRED_OR_UNPAIRED)= (COLLECTION_INPUT_LIST_PAIRED_OR_UNPAIRED)= -
Examples -:::{admonition} Example: `COLLECTION_INPUT_PAIRED` +:::{admonition} Example: `COLLECTION_INPUT_PAIRED` :class: note Assuming, -- $ d_f $, $ d_r $ are datasets -- $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ -- $ C $ is $ \text{CollectionInstance<}paired,\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\}\text{>} $ +* $ d_f $, $ d_r $ are datasets +* $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ +* $ C $ is $ \text{CollectionInstance<}\text{paired},\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\}\text{>} $ + then @@ -232,14 +266,17 @@ $$tool(i=C) \rightarrow \left\{o: \text{dataset}\right\}$$ ::: -:::{admonition} Example: `COLLECTION_INPUT_LIST` + + +:::{admonition} Example: `COLLECTION_INPUT_LIST` :class: note Assuming, -- $ d_1,...,d_n $ are datasets -- $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ -- $ C $ is $ \text{CollectionInstance<}list,\left\{ \text{ el1 }=d_1, ..., \text{ eln }=d_n \right\}\text{>} $ +* $ d_1,...,d_n $ are datasets +* $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ +* $ C $ is $ \text{CollectionInstance<}\text{list},\left\{ \text{ el1 }=d_1, ..., \text{ eln }=d_n \right\}\text{>} $ + then @@ -247,14 +284,17 @@ $$tool(i=C) \rightarrow \left\{o: \text{dataset}\right\}$$ ::: -:::{admonition} Example: `COLLECTION_INPUT_PAIRED_OR_UNPAIRED` + + +:::{admonition} Example: `COLLECTION_INPUT_PAIRED_OR_UNPAIRED` :class: note Assuming, -- $ d_f $, $ d_r $ are datasets -- $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ -- $ C $ is $ \text{CollectionInstance<}paired_or_unpaired,\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\}\text{>} $ +* $ d_f $, $ d_r $ are datasets +* $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ +* $ C $ is $ \text{CollectionInstance<}\text{paired\_or\_unpaired},\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\}\text{>} $ + then @@ -262,14 +302,17 @@ $$tool(i=C) \rightarrow \left\{o: \text{dataset}\right\}$$ ::: -:::{admonition} Example: `COLLECTION_INPUT_LIST_PAIRED_OR_UNPAIRED` + + +:::{admonition} Example: `COLLECTION_INPUT_LIST_PAIRED_OR_UNPAIRED` :class: note Assuming, -- $ d_f $, $ d_r $ are datasets -- $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ -- $ C $ is $ \text{CollectionInstance<}list:paired_or_unpaired,\left\{ \text{ el1 }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\} \right\}\text{>} $ +* $ d_f $, $ d_r $ are datasets +* $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ +* $ C $ is $ \text{CollectionInstance<}\text{list}:\text{paired\_or\_unpaired},\left\{ \text{ el1 }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\} \right\}\text{>} $ + then @@ -277,26 +320,31 @@ $$tool(i=C) \rightarrow \left\{o: \text{dataset}\right\}$$ ::: + +

-For nested collections where each rank is a `list` or a `paired` collection, + + +For nested collections where each rank is a ``list`` or a ``paired`` collection, then collection inputs must match every part of the collection type input definition. + (COLLECTION_INPUT_LIST_NOT_CONSUMES_PAIRS)= (COLLECTION_INPUT_PAIRED_NOT_CONSUMES_LIST)= (COLLECTION_INPUT_LIST_PAIRED_NOT_CONSUMES_PAIRED_PAIRED)= (COLLECTION_INPUT_LIST_PAIRED_OR_NOT_PAIRED_NOT_CONSUMES_PAIRED_PAIRED)= -
Examples -:::{admonition} Example: `COLLECTION_INPUT_LIST_NOT_CONSUMES_PAIRS` +:::{admonition} Example: `COLLECTION_INPUT_LIST_NOT_CONSUMES_PAIRS` :class: note Assuming, -- $ d_f $, $ d_r $ are datasets -- $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ -- $ C $ is $ \text{CollectionInstance<}paired,\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\}\text{>} $ +* $ d_f $, $ d_r $ are datasets +* $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ +* $ C $ is $ \text{CollectionInstance<}\text{paired},\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\}\text{>} $ + then @@ -304,14 +352,17 @@ $$tool(i=C)\text{ is invalid}$$ ::: -:::{admonition} Example: `COLLECTION_INPUT_PAIRED_NOT_CONSUMES_LIST` + + +:::{admonition} Example: `COLLECTION_INPUT_PAIRED_NOT_CONSUMES_LIST` :class: note Assuming, -- $ d_1,...,d_n $ are datasets -- $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ -- $ C $ is $ \text{CollectionInstance<}list,\left\{ \text{ i1 }=d_1, ..., \text{ in }=d_n \right\}\text{>} $ +* $ d_1,...,d_n $ are datasets +* $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ +* $ C $ is $ \text{CollectionInstance<}\text{list},\left\{ \text{ i1 }=d_1, ..., \text{ in }=d_n \right\}\text{>} $ + then @@ -319,14 +370,17 @@ $$tool(i=C)\text{ is invalid}$$ ::: -:::{admonition} Example: `COLLECTION_INPUT_LIST_PAIRED_NOT_CONSUMES_PAIRED_PAIRED` + + +:::{admonition} Example: `COLLECTION_INPUT_LIST_PAIRED_NOT_CONSUMES_PAIRED_PAIRED` :class: note Assuming, -- $ d_f $, $ d_r $ are datasets -- $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ -- $ C $ is $ \text{CollectionInstance<}paired:paired,\left\{ \text{ forward }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\}, \text{ reverse }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\} \right\}\text{>} $ +* $ d_f $, $ d_r $ are datasets +* $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ +* $ C $ is $ \text{CollectionInstance<}\text{paired}:\text{paired},\left\{ \text{ forward }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\}, \text{ reverse }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\} \right\}\text{>} $ + then @@ -334,14 +388,17 @@ $$tool(i=C)\text{ is invalid}$$ ::: -:::{admonition} Example: `COLLECTION_INPUT_LIST_PAIRED_OR_NOT_PAIRED_NOT_CONSUMES_PAIRED_PAIRED` + + +:::{admonition} Example: `COLLECTION_INPUT_LIST_PAIRED_OR_NOT_PAIRED_NOT_CONSUMES_PAIRED_PAIRED` :class: note Assuming, -- $ d_f $, $ d_r $ are datasets -- $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ -- $ C $ is $ \text{CollectionInstance<}paired:paired,\left\{ \text{ forward }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\}, \text{ reverse }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\} \right\}\text{>} $ +* $ d_f $, $ d_r $ are datasets +* $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ +* $ C $ is $ \text{CollectionInstance<}\text{paired}:\text{paired},\left\{ \text{ forward }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\}, \text{ reverse }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\} \right\}\text{>} $ + then @@ -349,24 +406,29 @@ $$tool(i=C)\text{ is invalid}$$ ::: + +

-In addition to explicit collection inputs, tool inputs of `type` `data` -where `multiple="true"` can consume lists directly. This is likewise a + + +In addition to explicit collection inputs, tool inputs of ``type`` ``data`` +where ``multiple="true"`` can consume lists directly. This is likewise a "reduction" and does not result in implicit collection creation. -(LIST_REDUCTION)= +(LIST_REDUCTION)=
Examples -:::{admonition} Example: `LIST_REDUCTION` +:::{admonition} Example: `LIST_REDUCTION` :class: note Assuming, -- $ d_1,...,d_n $ are datasets -- $ tool \text{ is } (i: \text{ dataset }) \Rightarrow \{ o: \text{ dataset } \} $ -- $ C $ is $ \text{CollectionInstance<}list,\left\{ \text{ i1 }=d_1, ..., \text{ in }=d_n \right\}\text{>} $ +* $ d_1,...,d_n $ are datasets +* $ tool \text{ is } (i: \text{ dataset }) \Rightarrow \{ o: \text{ dataset } \} $ +* $ C $ is $ \text{CollectionInstance<}\text{list},\left\{ \text{ i1 }=d_1, ..., \text{ in }=d_n \right\}\text{>} $ + then @@ -374,24 +436,29 @@ $$tool(i=C) == tool(i=[d_1,...,d_n])$$ ::: + +

-Paired collections cannot be reduced this way. `paired` is not meant + + +Paired collections cannot be reduced this way. ``paired`` is not meant to represent a list/array/vector data structure - it is more like a tuple. + (PAIRED_REDUCTION_INVALID)= (PAIRED_OR_UNPAIRED_REDUCTION_INVALID)= -
Examples -:::{admonition} Example: `PAIRED_REDUCTION_INVALID` +:::{admonition} Example: `PAIRED_REDUCTION_INVALID` :class: note Assuming, -- $ d_f $, $ d_r $ are datasets -- $ tool \text{ is } (i: \text{ dataset }) \Rightarrow \{ o: \text{ dataset } \} $ -- $ C $ is $ \text{CollectionInstance<}paired,\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\}\text{>} $ +* $ d_f $, $ d_r $ are datasets +* $ tool \text{ is } (i: \text{ dataset }) \Rightarrow \{ o: \text{ dataset } \} $ +* $ C $ is $ \text{CollectionInstance<}\text{paired},\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\}\text{>} $ + then @@ -399,14 +466,17 @@ $$tool(i=C)\text{ is invalid}$$ ::: -:::{admonition} Example: `PAIRED_OR_UNPAIRED_REDUCTION_INVALID` + + +:::{admonition} Example: `PAIRED_OR_UNPAIRED_REDUCTION_INVALID` :class: note Assuming, -- $ d_f $, $ d_r $ are datasets -- $ tool \text{ is } (i: \text{ dataset }) \Rightarrow \{ o: \text{ dataset } \} $ -- $ C $ is $ \text{CollectionInstance<}paired_or_unpaired,\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\}\text{>} $ +* $ d_f $, $ d_r $ are datasets +* $ tool \text{ is } (i: \text{ dataset }) \Rightarrow \{ o: \text{ dataset } \} $ +* $ C $ is $ \text{CollectionInstance<}\text{paired\_or\_unpaired},\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\}\text{>} $ + then @@ -414,25 +484,30 @@ $$tool(i=C)\text{ is invalid}$$ ::: + +

+ + ## Sub-collection Mapping ![](https://planemo.readthedocs.io/en/master/_images/subcollection_mapping_identifiers.svg) -(MAPPING_LIST_PAIRED_OVER_PAIRED)= +(MAPPING_LIST_PAIRED_OVER_PAIRED)=
Examples -:::{admonition} Example: `MAPPING_LIST_PAIRED_OVER_PAIRED` +:::{admonition} Example: `MAPPING_LIST_PAIRED_OVER_PAIRED` :class: note Assuming, -- $ d_f $, $ d_r $ are datasets -- $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ -- $ C $ is $ \text{CollectionInstance<}list:paired,\left\{ \text{ el1 }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\} \right\}\text{>} $ -- $ C_PAIRED $ is $ \text{CollectionInstance<}paired,\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\}\text{>} $ +* $ d_f $, $ d_r $ are datasets +* $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ +* $ C $ is $ \text{CollectionInstance<}\text{list}:\text{paired},\left\{ \text{ el1 }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\} \right\}\text{>} $ +* $ C\_PAIRED $ is $ \text{CollectionInstance<}\text{paired},\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\}\text{>} $ + then @@ -440,26 +515,31 @@ $$tool(i=\text{mapOver}(C, '\text{paired}')) \mapsto \left\{o: \text{collection} ::: + +

+ + The natural extension of multiple data input parameters consuming list collections as described -above when discussing reductions is that nested lists of lists (`list:list`) can be mapped +above when discussing reductions is that nested lists of lists (``list:list``) can be mapped over a multiple data input parameter. Each nested list will be reduced by this operation but the results will be mapped over. The result will be a list with the same structure as the outer list of the input collection. -(NESTED_LIST_REDUCTION)= +(NESTED_LIST_REDUCTION)=
Examples -:::{admonition} Example: `NESTED_LIST_REDUCTION` +:::{admonition} Example: `NESTED_LIST_REDUCTION` :class: note Assuming, -- $ d_1,...,d_n $ are datasets -- $ tool \text{ is } (i: \text{ dataset }) \Rightarrow \{ o: \text{ dataset } \} $ -- $ C $ is $ \text{CollectionInstance<}list:list,\left\{ \text{ o1 }=\left\{ \text{ inner }=d_1 \right\}, ..., \text{ on }=\left\{ \text{ inner }=d_n \right\} \right\}\text{>} $ +* $ d_1,...,d_n $ are datasets +* $ tool \text{ is } (i: \text{ dataset }) \Rightarrow \{ o: \text{ dataset } \} $ +* $ C $ is $ \text{CollectionInstance<}\text{list}:\text{list},\left\{ \text{ o1 }=\left\{ \text{ inner }=d_1 \right\}, ..., \text{ on }=\left\{ \text{ inner }=d_n \right\} \right\}\text{>} $ + then @@ -467,25 +547,30 @@ $$tool(i=\text{mapOver}(C, '\text{list}')) \mapsto \left\{o: \text{collection}<\ ::: + +

+ + Just as a paired collection won't be reduced by a multiple data input, any sort of nested collection ending in a paired collection cannot be mapped over such an input. So a multiple -data input parameter cannot be mapped over by a list of pairs (`list:paired`) for instance. +data input parameter cannot be mapped over by a list of pairs (``list:paired``) for instance. + (LIST_PAIRED_REDUCTION_INVALID)= (LIST_PAIRED_OR_UNPAIRED_REDUCTION_INVALID)= -
Examples -:::{admonition} Example: `LIST_PAIRED_REDUCTION_INVALID` +:::{admonition} Example: `LIST_PAIRED_REDUCTION_INVALID` :class: note Assuming, -- $ d_f $, $ d_r $ are datasets -- $ tool \text{ is } (i: \text{ dataset }) \Rightarrow \{ o: \text{ dataset } \} $ -- $ C $ is $ \text{CollectionInstance<}list:paired,\left\{ \text{ el1 }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\} \right\}\text{>} $ +* $ d_f $, $ d_r $ are datasets +* $ tool \text{ is } (i: \text{ dataset }) \Rightarrow \{ o: \text{ dataset } \} $ +* $ C $ is $ \text{CollectionInstance<}\text{list}:\text{paired},\left\{ \text{ el1 }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\} \right\}\text{>} $ + then @@ -493,14 +578,17 @@ $$tool(i=\text{mapOver}(C, '\text{paired}'))\text{ is invalid}$$ ::: -:::{admonition} Example: `LIST_PAIRED_OR_UNPAIRED_REDUCTION_INVALID` + + +:::{admonition} Example: `LIST_PAIRED_OR_UNPAIRED_REDUCTION_INVALID` :class: note Assuming, -- $ d_f $, $ d_r $ are datasets -- $ tool \text{ is } (i: \text{ dataset }) \Rightarrow \{ o: \text{ dataset } \} $ -- $ C $ is $ \text{CollectionInstance<}list:paired_or_unpaired,\left\{ \text{ el1 }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\} \right\}\text{>} $ +* $ d_f $, $ d_r $ are datasets +* $ tool \text{ is } (i: \text{ dataset }) \Rightarrow \{ o: \text{ dataset } \} $ +* $ C $ is $ \text{CollectionInstance<}\text{list}:\text{paired\_or\_unpaired},\left\{ \text{ el1 }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\} \right\}\text{>} $ + then @@ -508,32 +596,37 @@ $$tool(i=\text{mapOver}(C, '\text{paired\_or\_unpaired}'))\text{ is invalid}$$ ::: + +

+ + ## paired_or_unpaired Collections -The collection type `paired_or_unpaired` is meant to serve as a stand-in for -an entity that can be either a single dataset or what is effectively a `paired` +The collection type ``paired_or_unpaired`` is meant to serve as a stand-in for +an entity that can be either a single dataset or what is effectively a ``paired`` dataset collection. These collections either have one element with identifier -`unpaired` or two elements with identifiers `forward` and `reverse`. +``unpaired`` or two elements with identifiers ``forward`` and ``reverse``. + +Tools can declare a data_collection input with collection type ``paired_or_unpaired`` +and that input will consume either an explicit ``paired_or_unpaired`` collection +normally or can consume a ``paired`` input. -Tools can declare a data_collection input with collection type `paired_or_unpaired` -and that input will consume either an explicit `paired_or_unpaired` collection -normally or can consume a `paired` input. (PAIRED_OR_UNPAIRED_CONSUMES_PAIRED)= -
Examples -:::{admonition} Example: `PAIRED_OR_UNPAIRED_CONSUMES_PAIRED` +:::{admonition} Example: `PAIRED_OR_UNPAIRED_CONSUMES_PAIRED` :class: note Assuming, -- $ d_f $, $ d_r $ are datasets -- $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ -- $ C $ is $ \text{CollectionInstance<}paired,\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\}\text{>} $ -- $ C_AS_MIXED = CollectionInstance<\text{paired_or_unpaired}, \left\{\text{forward}: d_f, \text{reverse}: d_r\right\}> $ +* $ d_f $, $ d_r $ are datasets +* $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ +* $ C $ is $ \text{CollectionInstance<}\text{paired},\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\}\text{>} $ +* $ C_AS_MIXED = CollectionInstance<\text{paired\_or\_unpaired}, \left\{\text{forward}: d_f, \text{reverse}: d_r\right\}> $ + then @@ -541,26 +634,32 @@ $$tool(i=C) == tool(i=C_AS_MIXED)$$ ::: + +

-The inverse of this doesn't work intentionally. In some ways a `paired` collection -acts as a `paired_or_unpaired` collection but a `paired_or_unpaired` is not a paired -collection. This makes a lot of sense in terms of tools - a tool consuming a `paired` -dataset expects to find both a `forward` and `reverse` element but these may not exist -in `paired_or_unpaired` collection. + + + +The inverse of this doesn't work intentionally. In some ways a ``paired`` collection +acts as a ``paired_or_unpaired`` collection but a ``paired_or_unpaired`` is not a paired +collection. This makes a lot of sense in terms of tools - a tool consuming a ``paired`` +dataset expects to find both a ``forward`` and ``reverse`` element but these may not exist +in ``paired_or_unpaired`` collection. + (PAIRED_OR_UNPAIRED_NOT_CONSUMED_BY_PAIRED)= -
Examples -:::{admonition} Example: `PAIRED_OR_UNPAIRED_NOT_CONSUMED_BY_PAIRED` +:::{admonition} Example: `PAIRED_OR_UNPAIRED_NOT_CONSUMED_BY_PAIRED` :class: note Assuming, -- $ d_f $, $ d_r $ are datasets -- $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ -- $ C $ is $ \text{CollectionInstance<}paired_or_unpaired,\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\}\text{>} $ +* $ d_f $, $ d_r $ are datasets +* $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ +* $ C $ is $ \text{CollectionInstance<}\text{paired\_or\_unpaired},\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\}\text{>} $ + then @@ -568,28 +667,34 @@ $$tool(i=C)\text{ is invalid}$$ ::: + +

-The same logic holds for mapping, lists of paired datasets (`list:paired`) can be mapped over these -`paired_or_unpaired` inputs and mixed lists of pairs (`list:paired_or_unpaired`) cannot -be mapped over a `paired` input. Following the same logic, `list:paired_or_unpaired` cannot -be mapped over a `list` input or multiple data input. + + + +The same logic holds for mapping, lists of paired datasets (``list:paired``) can be mapped over these +``paired_or_unpaired`` inputs and mixed lists of pairs (``list:paired_or_unpaired``) cannot +be mapped over a ``paired`` input. Following the same logic, ``list:paired_or_unpaired`` cannot +be mapped over a ``list`` input or multiple data input. + (MAPPING_LIST_PAIRED_OVER_PAIRED_OR_UNPAIRED)= (PAIRED_OR_UNPAIRED_NOT_CONSUMED_BY_PAIRED_WHEN_MAPPING)= (PAIRED_OR_UNPAIRED_NOT_CONSUMED_BY_LIST_WHEN_MAPPING)= -
Examples -:::{admonition} Example: `MAPPING_LIST_PAIRED_OVER_PAIRED_OR_UNPAIRED` +:::{admonition} Example: `MAPPING_LIST_PAIRED_OVER_PAIRED_OR_UNPAIRED` :class: note Assuming, -- $ d_f $, $ d_r $ are datasets -- $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ -- $ C $ is $ \text{CollectionInstance<}list:paired,\left\{ \text{ el }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\} \right\}\text{>} $ -- $ C_AS_MIXED $ is $ \text{CollectionInstance<}list:paired_or_unpaired,\left\{ \text{ el }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\} \right\}\text{>} $ +* $ d_f $, $ d_r $ are datasets +* $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ +* $ C $ is $ \text{CollectionInstance<}\text{list}:\text{paired},\left\{ \text{ el }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\} \right\}\text{>} $ +* $ C_AS_MIXED $ is $ \text{CollectionInstance<}\text{list}:\text{paired\_or\_unpaired},\left\{ \text{ el }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\} \right\}\text{>} $ + then @@ -597,14 +702,17 @@ $$tool(i=\text{mapOver}(C)) == tool(i=\text{mapOver}(C_AS_MIXED))$$ ::: -:::{admonition} Example: `PAIRED_OR_UNPAIRED_NOT_CONSUMED_BY_PAIRED_WHEN_MAPPING` + + +:::{admonition} Example: `PAIRED_OR_UNPAIRED_NOT_CONSUMED_BY_PAIRED_WHEN_MAPPING` :class: note Assuming, -- $ d_f $, $ d_r $ are datasets -- $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ -- $ C $ is $ \text{CollectionInstance<}list:paired_or_unpaired,\left\{ \text{ el }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\} \right\}\text{>} $ +* $ d_f $, $ d_r $ are datasets +* $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ +* $ C $ is $ \text{CollectionInstance<}\text{list}:\text{paired\_or\_unpaired},\left\{ \text{ el }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\} \right\}\text{>} $ + then @@ -612,14 +720,17 @@ $$tool(i=\text{mapOver}(C))\text{ is invalid}$$ ::: -:::{admonition} Example: `PAIRED_OR_UNPAIRED_NOT_CONSUMED_BY_LIST_WHEN_MAPPING` + + +:::{admonition} Example: `PAIRED_OR_UNPAIRED_NOT_CONSUMED_BY_LIST_WHEN_MAPPING` :class: note Assuming, -- $ d_f $, $ d_r $ are datasets -- $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ -- $ C $ is $ \text{CollectionInstance<}list:paired_or_unpaired,\left\{ \text{ el }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\} \right\}\text{>} $ +* $ d_f $, $ d_r $ are datasets +* $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ +* $ C $ is $ \text{CollectionInstance<}\text{list}:\text{paired\_or\_unpaired},\left\{ \text{ el }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\} \right\}\text{>} $ + then @@ -627,25 +738,30 @@ $$tool(i=\text{mapOver}(C))\text{ is invalid}$$ ::: + +

-This logic extends naturally into higher dimensional collections. A `list:list:paired` -can be mapped over either a `paired_or_unpaired` input to produce a nested list (`list:list`) -or a `list:paired_or_unpaired` input to produce a flat list (`list`). + + +This logic extends naturally into higher dimensional collections. A ``list:list:paired`` +can be mapped over either a ``paired_or_unpaired`` input to produce a nested list (``list:list``) +or a ``list:paired_or_unpaired`` input to produce a flat list (``list``). + (MAPPING_LIST_LIST_PAIRED_OVER_PAIRED_OR_UNPAIRED)= -
Examples -:::{admonition} Example: `MAPPING_LIST_LIST_PAIRED_OVER_PAIRED_OR_UNPAIRED` +:::{admonition} Example: `MAPPING_LIST_LIST_PAIRED_OVER_PAIRED_OR_UNPAIRED` :class: note Assuming, -- $ d_f $, $ d_r $ are datasets -- $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ -- $ C $ is $ \text{CollectionInstance<}list:list:paired,\left\{ \text{ o1 }=\left\{ \text{ el1 }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\} \right\} \right\}\text{>} $ -- $ C_AS_MIXED $ is $ \text{CollectionInstance<}list:list:paired_or_unpaired,\left\{ \text{ o1 }=\left\{ \text{ el1 }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\} \right\} \right\}\text{>} $ +* $ d_f $, $ d_r $ are datasets +* $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ +* $ C $ is $ \text{CollectionInstance<}\text{list}:\text{list}:\text{paired},\left\{ \text{ o1 }=\left\{ \text{ el1 }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\} \right\} \right\}\text{>} $ +* $ C_AS_MIXED $ is $ \text{CollectionInstance<}\text{list}:\text{list}:\text{paired\_or\_unpaired},\left\{ \text{ o1 }=\left\{ \text{ el1 }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\} \right\} \right\}\text{>} $ + then @@ -653,25 +769,31 @@ $$tool(i=\text{mapOver}(C)) == tool(i=\text{mapOver}(C_AS_MIXED))$$ ::: + +

-In order for `paired_or_unpaired` collections to also act as a single dataset, + + + +In order for ``paired_or_unpaired`` collections to also act as a single dataset, a flat list can be mapped over a such an input with a special sub collection mapping type of 'single_datasets'. -(MAPPING_LIST_OVER_PAIRED_OR_UNPAIRED)= +(MAPPING_LIST_OVER_PAIRED_OR_UNPAIRED)=
Examples -:::{admonition} Example: `MAPPING_LIST_OVER_PAIRED_OR_UNPAIRED` +:::{admonition} Example: `MAPPING_LIST_OVER_PAIRED_OR_UNPAIRED` :class: note Assuming, -- $ d_1,...,d_n $ are datasets -- $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ -- $ C $ is $ \text{CollectionInstance<}list,\left\{ \text{ i1 }=d_1, ..., \text{ in }=d_n \right\}\text{>} $ -- $ C_AS_UNPAIRED_i = CollectionInstance<\text{paired_or_unpaired},\left\{\text{un\text{paired}}=di\right\}> for i from 1...n $ +* $ d_1,...,d_n $ are datasets +* $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ +* $ C $ is $ \text{CollectionInstance<}\text{list},\left\{ \text{ i1 }=d_1, ..., \text{ in }=d_n \right\}\text{>} $ +* $ C_AS_UNPAIRED_i = CollectionInstance<\text{paired\_or\_unpaired},\left\{\text{unpaired}=di\right\}> for i from 1...n $ + then @@ -679,29 +801,34 @@ $$tool(i=\text{mapOver}(C, '\text{single\_datasets}')) \mapsto \left\{o: \text{c ::: + +

+ + This treatment of lists without pairing extends to nested structures naturally. -For instance, a list of list of datasets (`list:list`) can be mapped over a -`paired_or_unpaired` input to produce a nested list of lists (`list:list`) +For instance, a list of list of datasets (``list:list``) can be mapped over a +``paired_or_unpaired`` input to produce a nested list of lists (``list:list``) with a structure matching the input. Likewise, the nested list can be mapped over -a `list:paired_or_unpaired` input to produce a flat list with the same structure +a ``list:paired_or_unpaired`` input to produce a flat list with the same structure as the outer list of the input. + (MAPPING_LIST_LIST_OVER_PAIRED_OR_UNPAIRED)= (MAPPING_LIST_LIST_OVER_LIST_PAIRED_OR_UNPAIRED)= -
Examples -:::{admonition} Example: `MAPPING_LIST_LIST_OVER_PAIRED_OR_UNPAIRED` +:::{admonition} Example: `MAPPING_LIST_LIST_OVER_PAIRED_OR_UNPAIRED` :class: note Assuming, -- $ d_1,...,d_n $ are datasets -- $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ -- $ C $ is $ \text{CollectionInstance<}list:list,\left\{ \text{ o1 }=\left\{ \text{ inner }=d_1 \right\}, ..., \text{ on }=\left\{ \text{ inner }=d_n \right\} \right\}\text{>} $ -- $ C_AS_UNPAIRED_j = CollectionInstance<\text{paired_or_unpaired},\left\{\text{un\text{paired}}=dj\right\}> for each \text{dataset} dj in C $ +* $ d_1,...,d_n $ are datasets +* $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ +* $ C $ is $ \text{CollectionInstance<}\text{list}:\text{list},\left\{ \text{ o1 }=\left\{ \text{ inner }=d_1 \right\}, ..., \text{ on }=\left\{ \text{ inner }=d_n \right\} \right\}\text{>} $ +* $ C_AS_UNPAIRED_j = CollectionInstance<\text{paired\_or\_unpaired},\left\{\text{unpaired}=dj\right\}> for each \text{dataset} dj in C $ + then @@ -709,15 +836,18 @@ $$tool(i=\text{mapOver}(C, '\text{single\_datasets}')) \mapsto \left\{o: \text{c ::: -:::{admonition} Example: `MAPPING_LIST_LIST_OVER_LIST_PAIRED_OR_UNPAIRED` + + +:::{admonition} Example: `MAPPING_LIST_LIST_OVER_LIST_PAIRED_OR_UNPAIRED` :class: note Assuming, -- $ d_1,...,d_n $ are datasets -- $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ -- $ C $ is $ \text{CollectionInstance<}list:list,\left\{ \text{ o1 }=\left\{ \text{ inner }=d_1 \right\}, ..., \text{ on }=\left\{ \text{ inner }=d_n \right\} \right\}\text{>} $ -- $ C_AS_LPU_i = \text{inner} \text{list} of C at position i, treated as \text{list}:\text{paired_or_unpaired} via \text{single\_\text{dataset}s} $ +* $ d_1,...,d_n $ are datasets +* $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ +* $ C $ is $ \text{CollectionInstance<}\text{list}:\text{list},\left\{ \text{ o1 }=\left\{ \text{ inner }=d_1 \right\}, ..., \text{ on }=\left\{ \text{ inner }=d_n \right\} \right\}\text{>} $ +* $ C_AS_LPU_i = \text{inner} \text{list} of C at position i, treated as \text{list}:\text{paired\_or\_unpaired} via \text{single\_datasets} $ + then @@ -725,8 +855,12 @@ $$tool(i=\text{mapOver}(C, '\text{list}:\text{paired\_or\_unpaired}')) \mapsto \ ::: + +

+ + Due only to implementation time, the special casing of allowing paired_or_unpaired act as both datasets and paired collections only works when it is the deepest collection type. So while list:paired can be consumed by a list:paired_or_unpaired @@ -735,45 +869,50 @@ it should be able to for consistency. We have focused our time on data structure more likely to be used in actual Galaxy analyses given current and guessed future usage. + ## sample_sheet Collections -The collection type `sample_sheet` attaches typed, columnar metadata to each +The collection type ``sample_sheet`` attaches typed, columnar metadata to each element of a dataset collection. For mapping and type matching purposes, -`sample_sheet` behaves identically to `list` - it can be mapped over tool -inputs, matched against `list` collection inputs, and composed with inner types -(`sample_sheet:paired`, `sample_sheet:paired_or_unpaired`). The key asymmetry -is that while a `sample_sheet` output can satisfy a `list` input, a `list` -output cannot satisfy a `sample_sheet` input - sample sheets carry metadata +``sample_sheet`` behaves identically to ``list`` - it can be mapped over tool +inputs, matched against ``list`` collection inputs, and composed with inner types +(``sample_sheet:paired``, ``sample_sheet:paired_or_unpaired``). The key asymmetry +is that while a ``sample_sheet`` output can satisfy a ``list`` input, a ``list`` +output cannot satisfy a ``sample_sheet`` input - sample sheets carry metadata that plain lists do not. + (SAMPLE_SHEET_MAPPING)= (SAMPLE_SHEET_MATCHES_LIST)= -
Examples -:::{admonition} Example: `SAMPLE_SHEET_MAPPING` +:::{admonition} Example: `SAMPLE_SHEET_MAPPING` :class: note Assuming, -- $ d_1,...,d_n $ are datasets -- $ tool \text{ is } (i: \text{ dataset }) \Rightarrow \{ o: \text{ dataset } \} $ -- $ C $ is $ \text{CollectionInstance<}sample_sheet,\left\{ \text{ i1 }=d_1, ..., \text{ in }=d_n \right\}\text{>} $ +* $ d_1,...,d_n $ are datasets +* $ tool \text{ is } (i: \text{ dataset }) \Rightarrow \{ o: \text{ dataset } \} $ +* $ C $ is $ \text{CollectionInstance<}\text{sample\_sheet},\left\{ \text{ i1 }=d_1, ..., \text{ in }=d_n \right\}\text{>} $ + then -$$tool(i=\text{mapOver}(C)) \mapsto \left\{o: \text{collection}<\text{sample_sheet},\left\{\text{i1}=tool(i=d_1)[o],...,\text{in}=tool(i=d_n)[o]\right\}>\right\}$$ +$$tool(i=\text{mapOver}(C)) \mapsto \left\{o: \text{collection}<\text{sample\_sheet},\left\{\text{i1}=tool(i=d_1)[o],...,\text{in}=tool(i=d_n)[o]\right\}>\right\}$$ ::: -:::{admonition} Example: `SAMPLE_SHEET_MATCHES_LIST` + + +:::{admonition} Example: `SAMPLE_SHEET_MATCHES_LIST` :class: note Assuming, -- $ d_1,...,d_n $ are datasets -- $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ -- $ C $ is $ \text{CollectionInstance<}sample_sheet,\left\{ \text{ i1 }=d_1, ..., \text{ in }=d_n \right\}\text{>} $ +* $ d_1,...,d_n $ are datasets +* $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ +* $ C $ is $ \text{CollectionInstance<}\text{sample\_sheet},\left\{ \text{ i1 }=d_1, ..., \text{ in }=d_n \right\}\text{>} $ + then @@ -781,41 +920,49 @@ $$tool(i=C) \rightarrow \left\{o: \text{dataset}\right\}$$ ::: + +

-Sub-collection mapping works the same as for `list` composites. A -`sample_sheet:paired` can be mapped over a `paired` collection input, -extracting each inner pair and producing a `sample_sheet` implicit output. + + +Sub-collection mapping works the same as for ``list`` composites. A +``sample_sheet:paired`` can be mapped over a ``paired`` collection input, +extracting each inner pair and producing a ``sample_sheet`` implicit output. + (SAMPLE_SHEET_PAIRED_MAPPING_OVER_PAIRED)= (SAMPLE_SHEET_PAIRED_MATCHES_LIST_PAIRED)= -
Examples -:::{admonition} Example: `SAMPLE_SHEET_PAIRED_MAPPING_OVER_PAIRED` +:::{admonition} Example: `SAMPLE_SHEET_PAIRED_MAPPING_OVER_PAIRED` :class: note Assuming, -- $ d_f $, $ d_r $ are datasets -- $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ -- $ C $ is $ \text{CollectionInstance<}sample_sheet:paired,\left\{ \text{ el1 }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\} \right\}\text{>} $ -- $ C_PAIRED $ is $ \text{CollectionInstance<}paired,\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\}\text{>} $ +* $ d_f $, $ d_r $ are datasets +* $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ +* $ C $ is $ \text{CollectionInstance<}\text{sample\_sheet}:\text{paired},\left\{ \text{ el1 }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\} \right\}\text{>} $ +* $ C\_PAIRED $ is $ \text{CollectionInstance<}\text{paired},\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\}\text{>} $ + then -$$tool(i=\text{mapOver}(C, '\text{paired}')) \mapsto \left\{o: \text{collection}<\text{sample_sheet},\left\{\text{el1}=tool(i=C\_PAIRED)[o]\right\}>\right\}$$ +$$tool(i=\text{mapOver}(C, '\text{paired}')) \mapsto \left\{o: \text{collection}<\text{sample\_sheet},\left\{\text{el1}=tool(i=C\_PAIRED)[o]\right\}>\right\}$$ ::: -:::{admonition} Example: `SAMPLE_SHEET_PAIRED_MATCHES_LIST_PAIRED` + + +:::{admonition} Example: `SAMPLE_SHEET_PAIRED_MATCHES_LIST_PAIRED` :class: note Assuming, -- $ d_f $, $ d_r $ are datasets -- $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ -- $ C $ is $ \text{CollectionInstance<}sample_sheet:paired,\left\{ \text{ el1 }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\} \right\}\text{>} $ +* $ d_f $, $ d_r $ are datasets +* $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ +* $ C $ is $ \text{CollectionInstance<}\text{sample\_sheet}:\text{paired},\left\{ \text{ el1 }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\} \right\}\text{>} $ + then @@ -823,45 +970,53 @@ $$tool(i=C) \rightarrow \left\{o: \text{dataset}\right\}$$ ::: + +

-The `paired_or_unpaired` integration rules carry over from `list` to -`sample_sheet`. A flat `sample_sheet` can be mapped over a -`paired_or_unpaired` input via `single_datasets` sub-collection mapping, -and `sample_sheet:paired` can be mapped over `paired_or_unpaired` just as -`list:paired` can. + + +The ``paired_or_unpaired`` integration rules carry over from ``list`` to +``sample_sheet``. A flat ``sample_sheet`` can be mapped over a +``paired_or_unpaired`` input via ``single_datasets`` sub-collection mapping, +and ``sample_sheet:paired`` can be mapped over ``paired_or_unpaired`` just as +``list:paired`` can. + (SAMPLE_SHEET_MAPPING_OVER_PAIRED_OR_UNPAIRED)= (SAMPLE_SHEET_PAIRED_MAPPING_OVER_PAIRED_OR_UNPAIRED)= (SAMPLE_SHEET_PAIRED_OR_UNPAIRED_MATCHES_LIST_PAIRED_OR_UNPAIRED)= -
Examples -:::{admonition} Example: `SAMPLE_SHEET_MAPPING_OVER_PAIRED_OR_UNPAIRED` +:::{admonition} Example: `SAMPLE_SHEET_MAPPING_OVER_PAIRED_OR_UNPAIRED` :class: note Assuming, -- $ d_1,...,d_n $ are datasets -- $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ -- $ C $ is $ \text{CollectionInstance<}sample_sheet,\left\{ \text{ i1 }=d_1, ..., \text{ in }=d_n \right\}\text{>} $ -- $ C_AS_UNPAIRED_i = CollectionInstance<\text{paired_or_unpaired},\left\{\text{un\text{paired}}=di\right\}> for i from 1...n $ +* $ d_1,...,d_n $ are datasets +* $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ +* $ C $ is $ \text{CollectionInstance<}\text{sample\_sheet},\left\{ \text{ i1 }=d_1, ..., \text{ in }=d_n \right\}\text{>} $ +* $ C_AS_UNPAIRED_i = CollectionInstance<\text{paired\_or\_unpaired},\left\{\text{unpaired}=di\right\}> for i from 1...n $ + then -$$tool(i=\text{mapOver}(C, '\text{single\_datasets}')) \mapsto \left\{o: \text{collection}<\text{sample_sheet},\left\{\text{i1}=tool(i=C_AS_UNPAIRED_1)[o],...,\text{in}=tool(i=C_AS_UNPAIRED_n)[o]\right\}>\right\}$$ +$$tool(i=\text{mapOver}(C, '\text{single\_datasets}')) \mapsto \left\{o: \text{collection}<\text{sample\_sheet},\left\{\text{i1}=tool(i=C_AS_UNPAIRED_1)[o],...,\text{in}=tool(i=C_AS_UNPAIRED_n)[o]\right\}>\right\}$$ ::: -:::{admonition} Example: `SAMPLE_SHEET_PAIRED_MAPPING_OVER_PAIRED_OR_UNPAIRED` + + +:::{admonition} Example: `SAMPLE_SHEET_PAIRED_MAPPING_OVER_PAIRED_OR_UNPAIRED` :class: note Assuming, -- $ d_f $, $ d_r $ are datasets -- $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ -- $ C $ is $ \text{CollectionInstance<}sample_sheet:paired,\left\{ \text{ el1 }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\} \right\}\text{>} $ -- $ C_AS_MIXED $ is $ \text{CollectionInstance<}sample_sheet:paired_or_unpaired,\left\{ \text{ el1 }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\} \right\}\text{>} $ +* $ d_f $, $ d_r $ are datasets +* $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ +* $ C $ is $ \text{CollectionInstance<}\text{sample\_sheet}:\text{paired},\left\{ \text{ el1 }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\} \right\}\text{>} $ +* $ C_AS_MIXED $ is $ \text{CollectionInstance<}\text{sample\_sheet}:\text{paired\_or\_unpaired},\left\{ \text{ el1 }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\} \right\}\text{>} $ + then @@ -869,14 +1024,17 @@ $$tool(i=\text{mapOver}(C)) == tool(i=\text{mapOver}(C_AS_MIXED))$$ ::: -:::{admonition} Example: `SAMPLE_SHEET_PAIRED_OR_UNPAIRED_MATCHES_LIST_PAIRED_OR_UNPAIRED` + + +:::{admonition} Example: `SAMPLE_SHEET_PAIRED_OR_UNPAIRED_MATCHES_LIST_PAIRED_OR_UNPAIRED` :class: note Assuming, -- $ d_f $, $ d_r $ are datasets -- $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ -- $ C $ is $ \text{CollectionInstance<}sample_sheet:paired_or_unpaired,\left\{ \text{ el1 }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\} \right\}\text{>} $ +* $ d_f $, $ d_r $ are datasets +* $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ +* $ C $ is $ \text{CollectionInstance<}\text{sample\_sheet}:\text{paired\_or\_unpaired},\left\{ \text{ el1 }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\} \right\}\text{>} $ + then @@ -884,28 +1042,33 @@ $$tool(i=C) \rightarrow \left\{o: \text{dataset}\right\}$$ ::: + +

-The type matching is asymmetric: a `sample_sheet` output can satisfy a `list` + + +The type matching is asymmetric: a ``sample_sheet`` output can satisfy a ``list`` input because sample sheets carry all the structural information lists have (plus -metadata). However, a `list` output cannot satisfy a `sample_sheet` input because -lists lack the `column_definitions` and per-element `columns` metadata that +metadata). However, a ``list`` output cannot satisfy a ``sample_sheet`` input because +lists lack the ``column_definitions`` and per-element ``columns`` metadata that sample sheet consumers expect. + (LIST_NOT_MATCHES_SAMPLE_SHEET)= (LIST_PAIRED_NOT_MATCHES_SAMPLE_SHEET_PAIRED)= (SAMPLE_SHEET_MATCHES_SAMPLE_SHEET)= -
Examples -:::{admonition} Example: `LIST_NOT_MATCHES_SAMPLE_SHEET` +:::{admonition} Example: `LIST_NOT_MATCHES_SAMPLE_SHEET` :class: note Assuming, -- $ d_1,...,d_n $ are datasets -- $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ -- $ C $ is $ \text{CollectionInstance<}list,\left\{ \text{ i1 }=d_1, ..., \text{ in }=d_n \right\}\text{>} $ +* $ d_1,...,d_n $ are datasets +* $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ +* $ C $ is $ \text{CollectionInstance<}\text{list},\left\{ \text{ i1 }=d_1, ..., \text{ in }=d_n \right\}\text{>} $ + then @@ -913,14 +1076,17 @@ $$tool(i=C)\text{ is invalid}$$ ::: -:::{admonition} Example: `LIST_PAIRED_NOT_MATCHES_SAMPLE_SHEET_PAIRED` + + +:::{admonition} Example: `LIST_PAIRED_NOT_MATCHES_SAMPLE_SHEET_PAIRED` :class: note Assuming, -- $ d_f $, $ d_r $ are datasets -- $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ -- $ C $ is $ \text{CollectionInstance<}list:paired,\left\{ \text{ el1 }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\} \right\}\text{>} $ +* $ d_f $, $ d_r $ are datasets +* $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ +* $ C $ is $ \text{CollectionInstance<}\text{list}:\text{paired},\left\{ \text{ el1 }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\} \right\}\text{>} $ + then @@ -928,14 +1094,17 @@ $$tool(i=C)\text{ is invalid}$$ ::: -:::{admonition} Example: `SAMPLE_SHEET_MATCHES_SAMPLE_SHEET` + + +:::{admonition} Example: `SAMPLE_SHEET_MATCHES_SAMPLE_SHEET` :class: note Assuming, -- $ d_1,...,d_n $ are datasets -- $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ -- $ C $ is $ \text{CollectionInstance<}sample_sheet,\left\{ \text{ i1 }=d_1, ..., \text{ in }=d_n \right\}\text{>} $ +* $ d_1,...,d_n $ are datasets +* $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ +* $ C $ is $ \text{CollectionInstance<}\text{sample\_sheet},\left\{ \text{ i1 }=d_1, ..., \text{ in }=d_n \right\}\text{>} $ + then @@ -943,4 +1112,9 @@ $$tool(i=C) \rightarrow \left\{o: \text{dataset}\right\}$$ ::: + +

+ + + diff --git a/lib/galaxy/model/dataset_collections/types/semantics.py b/lib/galaxy/model/dataset_collections/types/semantics.py index 42f947b8b20..5e42b02894e 100644 --- a/lib/galaxy/model/dataset_collections/types/semantics.py +++ b/lib/galaxy/model/dataset_collections/types/semantics.py @@ -72,8 +72,8 @@ class ToolDefinition(BaseModel): outputs: dict[str, str] = Field(alias="out") def as_latex(self) -> str: - inputs = ", ".join([f"{k}: \\text{{ {v} }}" for (k, v) in self.inputs.items()]) - outputs = ", ".join([f"{k}: \\text{{ {v} }}" for (k, v) in self.outputs.items()]) + inputs = ", ".join([f"{k}: \\text{{ {_escape_underscores(v)} }}" for (k, v) in self.inputs.items()]) + outputs = ", ".join([f"{k}: \\text{{ {_escape_underscores(v)} }}" for (k, v) in self.outputs.items()]) return f"tool \\text{{ is }} ({inputs}) \\Rightarrow \\{{ {outputs} \\}}" @@ -104,10 +104,8 @@ class CollectionDefinition(NamedTuple): elements: dict[str, Any] def as_latex(self) -> str: - collection_type = self.collection_type.replace("_", "\\_") - return ( - f"\\text{{CollectionInstance<}}{collection_type},{_assumption_elements_to_latex(self.elements)}\\text{{>}}" - ) + ct = _latex_collection_type(self.collection_type) + return f"\\text{{CollectionInstance<}}{ct},{_assumption_elements_to_latex(self.elements)}\\text{{>}}" class CollectionDeclarations(BaseModel): @@ -120,11 +118,14 @@ Expression = Union[str, DatasetsDeclaration, ToolDeclaration, CollectionDeclarat # --- Structured Then Expression Models --- +def _escape_underscores(text: str) -> str: + return text.replace("_", "\\_") + + def _latex_type_word(word: str) -> str: - if word == "paired_or_unpaired": - return "\\text{paired\\_or\\_unpaired}" - if word == "single_datasets": - return "\\text{single\\_datasets}" + if "_" in word: + escaped = word.replace("_", "\\_") + return "\\text{" + escaped + "}" return "\\text{" + word + "}" @@ -313,6 +314,9 @@ YAMLRootModel = RootModel[list[Union[DocEntry, ExampleEntry]]] WORDS_TO_TEXTIFY = ["list", "forward", "reverse", "mapOver", "collection", "dataset", "inner"] _POU_PLACEHOLDER = "\x00POU\x00" +_UNPAIRED_PLACEHOLDER = "\x00UNPAIRED\x00" +_SINGLE_DATASETS_PLACEHOLDER = "\x00SINGLEDATASETS\x00" +_SAMPLE_SHEET_PLACEHOLDER = "\x00SAMPLESHEET\x00" def expression_to_latex(expression: str, wrap: bool = True): @@ -320,13 +324,19 @@ def expression_to_latex(expression: str, wrap: bool = True): expression = expression.replace("~>", "\\mapsto") expression = expression.replace("{", "\\left\\{") expression = expression.replace("}", "\\right\\}") - expression = expression.replace("single_datasets", "\\text{single\\_datasets}") + # Replace multi-word identifiers with placeholders before WORDS_TO_TEXTIFY + # to prevent double-replacement (e.g. "dataset" inside "single_datasets") + expression = expression.replace("single_datasets", _SINGLE_DATASETS_PLACEHOLDER) + expression = expression.replace("sample_sheet", _SAMPLE_SHEET_PLACEHOLDER) expression = expression.replace("paired_or_unpaired", _POU_PLACEHOLDER) - expression = expression.replace("unpaired", "\\text{unpaired}") + expression = expression.replace("unpaired", _UNPAIRED_PLACEHOLDER) expression = expression.replace("paired", "\\text{paired}") + expression = expression.replace(_UNPAIRED_PLACEHOLDER, "\\text{unpaired}") expression = expression.replace(_POU_PLACEHOLDER, "\\text{paired\\_or\\_unpaired}") for word in WORDS_TO_TEXTIFY: expression = expression.replace(word, "\\text{" + word + "}") + expression = expression.replace(_SINGLE_DATASETS_PLACEHOLDER, "\\text{single\\_datasets}") + expression = expression.replace(_SAMPLE_SHEET_PLACEHOLDER, "\\text{sample\\_sheet}") if wrap: return f"$ {expression} $" else: