diff --git a/doc/source/dev/collection_semantics.md b/doc/source/dev/collection_semantics.md index 2386e289c88..55ef137a190 100644 --- a/doc/source/dev/collection_semantics.md +++ b/doc/source/dev/collection_semantics.md @@ -8,7 +8,7 @@ In particular it describes how they operate within Galaxy tools and workflows. Any significantly sophisticated workflow language will have ways to collect data into arrays or vectors or dictionaries and apply operations across this data (mapping) -or reduce the dimensionality of this data (reductions). Typically, this explicitly +or reduce the dimensionality of this data (reductions). Typically, this is explicitly annotated with map functions or for loops. Galaxy however is designed to be a point and click interface for connecting steps and running tools. It is important that steps just connect and just do the most natural thing - and this is what Galaxy does. @@ -42,12 +42,12 @@ Assuming, * $ d_f $, $ d_r $ are datasets * $ tool \text{ is } (i: \text{ dataset }) \Rightarrow \{ o: \text{ dataset } \} $ -* $ C $ is $ \text{CollectionInstance<}paired,\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\}\text{>} $ +* $ C $ is $ \text{CollectionInstance<}\text{paired},\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\}\text{>} $ then -$$tool(i=\text{mapOver}(C)) \mapsto \left\{o: collection \right\}$$ +$$tool(i=\text{mapOver}(C)) \mapsto \left\{o: \text{collection}<\text{paired},\left\{\text{forward}=tool(i=d_f)[o], \text{reverse}=tool(i=d_r)[o]\right\}>\right\}$$ ::: @@ -60,12 +60,12 @@ Assuming, * $ d_f $, $ d_r $ are datasets * $ tool \text{ is } (i: \text{ dataset }) \Rightarrow \{ o: \text{ dataset } \} $ -* $ C $ is $ \text{CollectionInstance<}paired\_or\_unpaired,\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\}\text{>} $ +* $ C $ is $ \text{CollectionInstance<}\text{paired\_or\_unpaired},\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\}\text{>} $ then -$$tool(i=\text{mapOver}(C)) \mapsto \left\{o: collection\right\}$$ +$$tool(i=\text{mapOver}(C)) \mapsto \left\{o: \text{collection}<\text{paired\_or\_unpaired},\left\{\text{forward}=tool(i=d_f)[o], \text{reverse}=tool(i=d_r)[o]\right\}>\right\}$$ ::: @@ -78,12 +78,12 @@ Assuming, * $ d_u $ is a dataset * $ tool \text{ is } (i: \text{ dataset }) \Rightarrow \{ o: \text{ dataset } \} $ -* $ C $ is $ \text{CollectionInstance<}paired\_or\_unpaired,\left\{ \text{ unpaired }=d_u \right\}\text{>} $ +* $ C $ is $ \text{CollectionInstance<}\text{paired\_or\_unpaired},\left\{ \text{ unpaired }=d_u \right\}\text{>} $ then -$$tool(i=\text{mapOver}(C)) \mapsto \left\{o: collection\right\}$$ +$$tool(i=\text{mapOver}(C)) \mapsto \left\{o: \text{collection}<\text{paired\_or\_unpaired},\left\{\text{unpaired}=tool(i=d_u)[o]\right\}>\right\}$$ ::: @@ -96,12 +96,12 @@ Assuming, * $ d_1,...,d_n $ are datasets * $ tool \text{ is } (i: \text{ dataset }) \Rightarrow \{ o: \text{ dataset } \} $ -* $ C $ is $ \text{CollectionInstance<}list,\left\{ \text{ i1 }=d_1, ..., \text{ in }=d_n \right\}\text{>} $ +* $ C $ is $ \text{CollectionInstance<}\text{list},\left\{ \text{ i1 }=d_1, ..., \text{ in }=d_n \right\}\text{>} $ then -$$tool(i=\text{mapOver}(C)) \mapsto \left\{o: collection<\text{list},\left\{i1=tool(i=d_1)[o],...,in=tool(i=d_n)[o]]\right\}>\right\}$$ +$$tool(i=\text{mapOver}(C)) \mapsto \left\{o: \text{collection}<\text{list},\left\{\text{i1}=tool(i=d_1)[o],...,\text{in}=tool(i=d_n)[o]\right\}>\right\}$$ ::: @@ -126,12 +126,12 @@ Assuming, * $ d_1,...,d_n $ are datasets * $ tool \text{ is } (i: \text{ dataset }) \Rightarrow \{ o: \text{ dataset } \} $ -* $ C $ is $ \text{CollectionInstance<}list:list,\left\{ \text{ o1 }=\left\{ \text{ inner }=d_1 \right\}, ..., \text{ on }=\left\{ \text{ inner }=d_n \right\} \right\}\text{>} $ +* $ C $ is $ \text{CollectionInstance<}\text{list}:\text{list},\left\{ \text{ o1 }=\left\{ \text{ inner }=d_1 \right\}, ..., \text{ on }=\left\{ \text{ inner }=d_n \right\} \right\}\text{>} $ then -$$tool(i=\text{mapOver}(C)) \mapsto \left\{o: collection<\text{list}:\text{list},\left\{o1=\left\{inner=tool(i=d_1)[o]\right\}\right\},...,\left\{on=\left\{inner=tool(i=d_n)[o]\right\}\right\}>\right\}$$ +$$tool(i=\text{mapOver}(C)) \mapsto \left\{o: \text{collection}<\text{list}:\text{list},\left\{\text{o1}=\left\{\text{inner}=tool(i=d_1)[o]\right\},...,\text{on}=\left\{\text{inner}=tool(i=d_n)[o]\right\}\right\}>\right\}$$ ::: @@ -144,12 +144,12 @@ Assuming, * $ d_f $, $ d_r $ are datasets * $ tool \text{ is } (i: \text{ dataset }) \Rightarrow \{ o: \text{ dataset } \} $ -* $ C $ is $ \text{CollectionInstance<}list:paired\_or\_unpaired,\left\{ \text{ el1 }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\} \right\}\text{>} $ +* $ C $ is $ \text{CollectionInstance<}\text{list}:\text{paired\_or\_unpaired},\left\{ \text{ el1 }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\} \right\}\text{>} $ then -$$tool(i=\text{mapOver}(C)) \mapsto \left\{o: collection<\text{list}:paired\_or\_unpaired,\left\{el1=\left\{\text{forward}=tool(i=d_f)[o],\text{reverse}=tool(i=d_r)[o]\right\}\right\}>\right\}$$ +$$tool(i=\text{mapOver}(C)) \mapsto \left\{o: \text{collection}<\text{list}:\text{paired\_or\_unpaired},\left\{\text{el1}=\left\{\text{forward}=tool(i=d_f)[o], \text{reverse}=tool(i=d_r)[o]\right\}\right\}>\right\}$$ ::: @@ -175,12 +175,12 @@ Assuming, * $ d_1,...,d_n $, $ d_o $ are datasets * $ tool \text{ is } (i: \text{ dataset }, i2: \text{ dataset }) \Rightarrow \{ o: \text{ dataset } \} $ -* $ C $ is $ \text{CollectionInstance<}list,\left\{ \text{ i1 }=d_1, ..., \text{ in }=d_n \right\}\text{>} $ +* $ C $ is $ \text{CollectionInstance<}\text{list},\left\{ \text{ i1 }=d_1, ..., \text{ in }=d_n \right\}\text{>} $ then -$$tool(i=\text{mapOver}(C),i2=d_o) \mapsto \left\{o: collection<\text{list},\left\{i1=tool(i=d_1, i2=d_o)[o],...,in=tool(i=d_n, i2=d_o)[o]\right\}>\right\}$$ +$$tool(i=\text{mapOver}(C), i2=d_o) \mapsto \left\{o: \text{collection}<\text{list},\left\{\text{i1}=tool(i=d_1, i2=d_o)[o],...,\text{in}=tool(i=d_n, i2=d_o)[o]\right\}>\right\}$$ ::: @@ -217,13 +217,13 @@ Assuming, * $ d1_1,...,d1_n $, $ d2_1,...,d2_n $ are datasets * $ tool \text{ is } (i: \text{ dataset }, i2: \text{ dataset }) \Rightarrow \{ o: \text{ dataset } \} $ -* $ C1 $ is $ \text{CollectionInstance<}list,\left\{ \text{ i1 }=d1_1, ..., \text{ in }=d1_n \right\}\text{>} $ -* $ C2 $ is $ \text{CollectionInstance<}list,\left\{ \text{ i1 }=d2_1, ..., \text{ in }=d2_n \right\}\text{>} $ +* $ C1 $ is $ \text{CollectionInstance<}\text{list},\left\{ \text{ i1 }=d1_1, ..., \text{ in }=d1_n \right\}\text{>} $ +* $ C2 $ is $ \text{CollectionInstance<}\text{list},\left\{ \text{ i1 }=d2_1, ..., \text{ in }=d2_n \right\}\text{>} $ then -$$tool(i=\text{mapOver}(C1), i2=\text{mapOver}(C2)) \mapsto \left\{o: collection<\text{list},\left\{i1=tool(i=d1_1, i2=d2_1)[o],...,in=tool(i=d1_n, i2=d2_n)[o]]\right\}>\right\}$$ +$$tool(i=\text{mapOver}(C1), i2=\text{mapOver}(C2)) \mapsto \left\{o: \text{collection}<\text{list},\left\{\text{i1}=tool(i=d1_1, i2=d2_1)[o],...,\text{in}=tool(i=d1_n, i2=d2_n)[o]\right\}>\right\}$$ ::: @@ -241,7 +241,7 @@ collections directly and do not necessarily result in mapping over. Tools that consume collections and output datasets effectively reduce the dimension of the Galaxy data structure. When used at runtime -this is often referred to a "reduction" in the code. +this is often referred to as a "reduction" in the code. (COLLECTION_INPUT_PAIRED)= @@ -257,12 +257,12 @@ Assuming, * $ d_f $, $ d_r $ are datasets * $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ -* $ C $ is $ \text{CollectionInstance<}paired,\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\}\text{>} $ +* $ C $ is $ \text{CollectionInstance<}\text{paired},\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\}\text{>} $ then -$$tool(i=C) \rightarrow \left\{o: dataset\right\}$$ +$$tool(i=C) \rightarrow \left\{o: \text{dataset}\right\}$$ ::: @@ -273,14 +273,14 @@ $$tool(i=C) \rightarrow \left\{o: dataset\right\}$$ Assuming, -* $ d1,...,dn $ are datasets +* $ d_1,...,d_n $ are datasets * $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ -* $ C $ is $ \text{CollectionInstance<}list,\left\{ \text{ el1 }=d_1, ..., \text{ eln }=d_n \right\}\text{>} $ +* $ C $ is $ \text{CollectionInstance<}\text{list},\left\{ \text{ el1 }=d_1, ..., \text{ eln }=d_n \right\}\text{>} $ then -$$tool(i=C) \rightarrow \left\{o: dataset\right\}$$ +$$tool(i=C) \rightarrow \left\{o: \text{dataset}\right\}$$ ::: @@ -292,13 +292,13 @@ $$tool(i=C) \rightarrow \left\{o: dataset\right\}$$ Assuming, * $ d_f $, $ d_r $ are datasets -* $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ -* $ C $ is $ \text{CollectionInstance<}paired\_or\_unpaired,\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\}\text{>} $ +* $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ +* $ C $ is $ \text{CollectionInstance<}\text{paired\_or\_unpaired},\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\}\text{>} $ then -$$tool(i=C) \rightarrow \left\{o: dataset\right\}$$ +$$tool(i=C) \rightarrow \left\{o: \text{dataset}\right\}$$ ::: @@ -310,13 +310,13 @@ $$tool(i=C) \rightarrow \left\{o: dataset\right\}$$ Assuming, * $ d_f $, $ d_r $ are datasets -* $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ -* $ C $ is $ \text{CollectionInstance<}list:paired\_or\_unpaired,\left\{ \text{ el1 }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\} \right\}\text{>} $ +* $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ +* $ C $ is $ \text{CollectionInstance<}\text{list}:\text{paired\_or\_unpaired},\left\{ \text{ el1 }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\} \right\}\text{>} $ then -$$tool(i=C) \rightarrow \left\{o: dataset\right\}$$ +$$tool(i=C) \rightarrow \left\{o: \text{dataset}\right\}$$ ::: @@ -332,6 +332,8 @@ then collection inputs must match every part of the collection type input defini (COLLECTION_INPUT_LIST_NOT_CONSUMES_PAIRS)= (COLLECTION_INPUT_PAIRED_NOT_CONSUMES_LIST)= +(COLLECTION_INPUT_LIST_PAIRED_NOT_CONSUMES_PAIRED_PAIRED)= +(COLLECTION_INPUT_LIST_PAIRED_OR_NOT_PAIRED_NOT_CONSUMES_PAIRED_PAIRED)=
Examples :::{admonition} Example: `COLLECTION_INPUT_LIST_NOT_CONSUMES_PAIRS` @@ -341,7 +343,7 @@ Assuming, * $ d_f $, $ d_r $ are datasets * $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ -* $ C $ is $ \text{CollectionInstance<}paired,\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\}\text{>} $ +* $ C $ is $ \text{CollectionInstance<}\text{paired},\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\}\text{>} $ then @@ -359,7 +361,43 @@ Assuming, * $ d_1,...,d_n $ are datasets * $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ -* $ C $ is $ \text{CollectionInstance<}list,\left\{ \text{ i1 }=d_1, ..., \text{ in }=d_n \right\}\text{>} $ +* $ C $ is $ \text{CollectionInstance<}\text{list},\left\{ \text{ i1 }=d_1, ..., \text{ in }=d_n \right\}\text{>} $ + + +then + +$$tool(i=C)\text{ is invalid}$$ + +::: + + + +:::{admonition} Example: `COLLECTION_INPUT_LIST_PAIRED_NOT_CONSUMES_PAIRED_PAIRED` +:class: note + +Assuming, + +* $ d_f $, $ d_r $ are datasets +* $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ +* $ C $ is $ \text{CollectionInstance<}\text{paired}:\text{paired},\left\{ \text{ forward }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\}, \text{ reverse }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\} \right\}\text{>} $ + + +then + +$$tool(i=C)\text{ is invalid}$$ + +::: + + + +:::{admonition} Example: `COLLECTION_INPUT_LIST_PAIRED_OR_NOT_PAIRED_NOT_CONSUMES_PAIRED_PAIRED` +:class: note + +Assuming, + +* $ d_f $, $ d_r $ are datasets +* $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ +* $ C $ is $ \text{CollectionInstance<}\text{paired}:\text{paired},\left\{ \text{ forward }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\}, \text{ reverse }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\} \right\}\text{>} $ then @@ -389,7 +427,7 @@ Assuming, * $ d_1,...,d_n $ are datasets * $ tool \text{ is } (i: \text{ dataset }) \Rightarrow \{ o: \text{ dataset } \} $ -* $ C $ is $ \text{CollectionInstance<}list,\left\{ \text{ i1 }=d_1, ..., \text{ in }=d_n \right\}\text{>} $ +* $ C $ is $ \text{CollectionInstance<}\text{list},\left\{ \text{ i1 }=d_1, ..., \text{ in }=d_n \right\}\text{>} $ then @@ -419,7 +457,7 @@ Assuming, * $ d_f $, $ d_r $ are datasets * $ tool \text{ is } (i: \text{ dataset }) \Rightarrow \{ o: \text{ dataset } \} $ -* $ C $ is $ \text{CollectionInstance<}paired,\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\}\text{>} $ +* $ C $ is $ \text{CollectionInstance<}\text{paired},\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\}\text{>} $ then @@ -437,7 +475,7 @@ Assuming, * $ d_f $, $ d_r $ are datasets * $ tool \text{ is } (i: \text{ dataset }) \Rightarrow \{ o: \text{ dataset } \} $ -* $ C $ is $ \text{CollectionInstance<}paired\_or\_unpaired,\left\{ forward=d_f, reverse=d_r \right\}\text{>} $ +* $ C $ is $ \text{CollectionInstance<}\text{paired\_or\_unpaired},\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\}\text{>} $ then @@ -467,13 +505,13 @@ Assuming, * $ d_f $, $ d_r $ are datasets * $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ -* $ C $ is $ \text{CollectionInstance<}list:paired,\left\{ \text{ el1 }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\} \right\}\text{>} $ -* $ C\_PAIRED $ is $ \text{CollectionInstance<}paired,\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\}\text{>} $ +* $ C $ is $ \text{CollectionInstance<}\text{list}:\text{paired},\left\{ \text{ el1 }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\} \right\}\text{>} $ +* $ C\_PAIRED $ is $ \text{CollectionInstance<}\text{paired},\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\}\text{>} $ then -$$tool(i=\text{mapOver}(C, 'paired')) \mapsto \left\{o: collection<\text{list}, \left\{el1: tool(i=C\_PAIRED)[o]\right\}>\right\}$$ +$$tool(i=\text{mapOver}(C, '\text{paired}')) \mapsto \left\{o: \text{collection}<\text{list},\left\{\text{el1}=tool(i=C\_PAIRED)[o]\right\}>\right\}$$ ::: @@ -483,7 +521,7 @@ $$tool(i=\text{mapOver}(C, 'paired')) \mapsto \left\{o: collection<\text{list}, -The natural extension of multiple data input parameters consuming list collections as describe +The natural extension of multiple data input parameters consuming list collections as described above when discussing reductions is that nested lists of lists (``list:list``) can be mapped over a multiple data input parameter. Each nested list will be reduced by this operation but the results will be mapped over. The result will be a list with the same structure as the outer list @@ -500,12 +538,12 @@ Assuming, * $ d_1,...,d_n $ are datasets * $ tool \text{ is } (i: \text{ dataset }) \Rightarrow \{ o: \text{ dataset } \} $ -* $ C $ is $ \text{CollectionInstance<}list:list,\left\{ \text{ o1 }=\left\{ \text{ inner }=d_1 \right\}, ..., \text{ on }=\left\{ \text{ inner }=d_n \right\} \right\}\text{>} $ +* $ C $ is $ \text{CollectionInstance<}\text{list}:\text{list},\left\{ \text{ o1 }=\left\{ \text{ inner }=d_1 \right\}, ..., \text{ on }=\left\{ \text{ inner }=d_n \right\} \right\}\text{>} $ then -$$tool(i=\text{mapOver}(C, '\text{list}')) \mapsto \left\{o: collection<\text{list},\left\{o1: tool(i=[d_1])[o]\right\},...,on: tool(i=[d_n])[o]\right\}>\right\}$$ +$$tool(i=\text{mapOver}(C, '\text{list}')) \mapsto \left\{o: \text{collection}<\text{list},\left\{\text{o1}=tool(i=[d_1])[o],...,\text{on}=tool(i=[d_n])[o]\right\}>\right\}$$ ::: @@ -531,12 +569,12 @@ Assuming, * $ d_f $, $ d_r $ are datasets * $ tool \text{ is } (i: \text{ dataset }) \Rightarrow \{ o: \text{ dataset } \} $ -* $ C $ is $ \text{CollectionInstance<}list:paired,\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\}\text{>} $ +* $ C $ is $ \text{CollectionInstance<}\text{list}:\text{paired},\left\{ \text{ el1 }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\} \right\}\text{>} $ then -$$tool(i=\text{mapOver}(C, 'paired'))\text{ is invalid}$$ +$$tool(i=\text{mapOver}(C, '\text{paired}'))\text{ is invalid}$$ ::: @@ -549,12 +587,12 @@ Assuming, * $ d_f $, $ d_r $ are datasets * $ tool \text{ is } (i: \text{ dataset }) \Rightarrow \{ o: \text{ dataset } \} $ -* $ C $ is $ \text{CollectionInstance<}list:paired\_or\_unpaired,\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\}\text{>} $ +* $ C $ is $ \text{CollectionInstance<}\text{list}:\text{paired\_or\_unpaired},\left\{ \text{ el1 }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\} \right\}\text{>} $ then -$$tool(i=\text{mapOver}(C, 'paired\_or\_unpaired'))\text{ is invalid}$$ +$$tool(i=\text{mapOver}(C, '\text{paired\_or\_unpaired}'))\text{ is invalid}$$ ::: @@ -585,9 +623,9 @@ normally or can consume a ``paired`` input. Assuming, * $ d_f $, $ d_r $ are datasets -* $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ -* $ C $ is $ \text{CollectionInstance<}paired,\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\}\text{>} $ -* $ C_AS_MIXED = CollectionInstance $ +* $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ +* $ C $ is $ \text{CollectionInstance<}\text{paired},\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\}\text{>} $ +* $ C_AS_MIXED = CollectionInstance<\text{paired\_or\_unpaired}, \left\{\text{forward}: d_f, \text{reverse}: d_r\right\}> $ then @@ -603,7 +641,7 @@ $$tool(i=C) == tool(i=C_AS_MIXED)$$ -This inverse of this doesn't work intentionally. In some ways a ``paired`` collection +The inverse of this doesn't work intentionally. In some ways a ``paired`` collection acts as a ``paired_or_unpaired`` collection but a ``paired_or_unpaired`` is not a paired collection. This makes a lot of sense in terms of tools - a tool consuming a ``paired`` dataset expects to find both a ``forward`` and ``reverse`` element but these may not exist @@ -620,12 +658,12 @@ Assuming, * $ d_f $, $ d_r $ are datasets * $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ -* $ C $ is $ \text{CollectionInstance<}paired\_or\_unpaired,\left\{ forward=d_f, \text{ reverse }=d_r \right\}\text{>} $ +* $ C $ is $ \text{CollectionInstance<}\text{paired\_or\_unpaired},\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\}\text{>} $ then -$$tool(i=C) is invalid$$ +$$tool(i=C)\text{ is invalid}$$ ::: @@ -653,9 +691,9 @@ be mapped over a ``list`` input or multiple data input. Assuming, * $ d_f $, $ d_r $ are datasets -* $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ -* $ C $ is $ \text{CollectionInstance<}list:paired,\left\{ \text{ el }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\} \right\}\text{>} $ -* $ C_AS_MIXED $ is $ \text{CollectionInstance<}list:paired\_or\_unpaired,\left\{ \text{ el }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\} \right\}\text{>} $ +* $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ +* $ C $ is $ \text{CollectionInstance<}\text{list}:\text{paired},\left\{ \text{ el }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\} \right\}\text{>} $ +* $ C_AS_MIXED $ is $ \text{CollectionInstance<}\text{list}:\text{paired\_or\_unpaired},\left\{ \text{ el }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\} \right\}\text{>} $ then @@ -673,12 +711,12 @@ Assuming, * $ d_f $, $ d_r $ are datasets * $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ -* $ C $ is $ \text{CollectionInstance<}list:paired\_or\_unpaired,\left\{ \text{ el }=\left\{ \text{ forward }=f, \text{ reverse }=r \right\} \right\}\text{>} $ +* $ C $ is $ \text{CollectionInstance<}\text{list}:\text{paired\_or\_unpaired},\left\{ \text{ el }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\} \right\}\text{>} $ then -$$tool(i=\text{mapOver}(C)) is invalid$$ +$$tool(i=\text{mapOver}(C))\text{ is invalid}$$ ::: @@ -691,12 +729,12 @@ Assuming, * $ d_f $, $ d_r $ are datasets * $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ -* $ C $ is $ \text{CollectionInstance<}list:paired\_or\_unpaired,\left\{ \text{ el }=\left\{ \text{ forward }=f, \text{ reverse }=r \right\} \right\}\text{>} $ +* $ C $ is $ \text{CollectionInstance<}\text{list}:\text{paired\_or\_unpaired},\left\{ \text{ el }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\} \right\}\text{>} $ then -$$tool(i=\text{mapOver}(C)) is invalid$$ +$$tool(i=\text{mapOver}(C))\text{ is invalid}$$ ::: @@ -711,6 +749,32 @@ can be mapped over either a ``paired_or_unpaired`` input to produce a nested lis or a ``list:paired_or_unpaired`` input to produce a flat list (``list``). +(MAPPING_LIST_LIST_PAIRED_OVER_PAIRED_OR_UNPAIRED)= +
Examples + +:::{admonition} Example: `MAPPING_LIST_LIST_PAIRED_OVER_PAIRED_OR_UNPAIRED` +:class: note + +Assuming, + +* $ d_f $, $ d_r $ are datasets +* $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ +* $ C $ is $ \text{CollectionInstance<}\text{list}:\text{list}:\text{paired},\left\{ \text{ o1 }=\left\{ \text{ el1 }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\} \right\} \right\}\text{>} $ +* $ C_AS_MIXED $ is $ \text{CollectionInstance<}\text{list}:\text{list}:\text{paired\_or\_unpaired},\left\{ \text{ o1 }=\left\{ \text{ el1 }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\} \right\} \right\}\text{>} $ + + +then + +$$tool(i=\text{mapOver}(C)) == tool(i=\text{mapOver}(C_AS_MIXED))$$ + +::: + + + +

+ + + In order for ``paired_or_unpaired`` collections to also act as a single dataset, a flat list can be mapped over a such an input with a special sub collection mapping @@ -726,14 +790,14 @@ type of 'single_datasets'. Assuming, * $ d_1,...,d_n $ are datasets -* $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ -* $ C $ is $ \text{CollectionInstance<}list,\left\{ \text{ i1 }=d_1, ..., \text{ in }=d_n \right\}\text{>} $ -* $ C_AS_UNPAIRED_i = CollectionInstance for i from 1...n $ +* $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ +* $ C $ is $ \text{CollectionInstance<}\text{list},\left\{ \text{ i1 }=d_1, ..., \text{ in }=d_n \right\}\text{>} $ +* $ C_AS_UNPAIRED_i = CollectionInstance<\text{paired\_or\_unpaired},\left\{\text{unpaired}=di\right\}> for i from 1...n $ then -$$tool(i=\text{mapOver}(C, 'single_datasets')) \mapsto \left\{o: collection<\text{list},\left\{i1=tool(i=C_AS_UNPAIRED_1)[o],...,in=tool(i=C_AS_UNPAIRED_n)[o]]\right\}>\right\}$$ +$$tool(i=\text{mapOver}(C, '\text{single\_datasets}')) \mapsto \left\{o: \text{collection}<\text{list},\left\{\text{i1}=tool(i=C_AS_UNPAIRED_1)[o],...,\text{in}=tool(i=C_AS_UNPAIRED_n)[o]\right\}>\right\}$$ ::: @@ -751,7 +815,53 @@ a ``list:paired_or_unpaired`` input to produce a flat list with the same structu as the outer list of the input. -Due only implementation time, the special casing of allowing paired_or_unpaired +(MAPPING_LIST_LIST_OVER_PAIRED_OR_UNPAIRED)= +(MAPPING_LIST_LIST_OVER_LIST_PAIRED_OR_UNPAIRED)= +
Examples + +:::{admonition} Example: `MAPPING_LIST_LIST_OVER_PAIRED_OR_UNPAIRED` +:class: note + +Assuming, + +* $ d_1,...,d_n $ are datasets +* $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ +* $ C $ is $ \text{CollectionInstance<}\text{list}:\text{list},\left\{ \text{ o1 }=\left\{ \text{ inner }=d_1 \right\}, ..., \text{ on }=\left\{ \text{ inner }=d_n \right\} \right\}\text{>} $ +* $ C_AS_UNPAIRED_j = CollectionInstance<\text{paired\_or\_unpaired},\left\{\text{unpaired}=dj\right\}> for each \text{dataset} dj in C $ + + +then + +$$tool(i=\text{mapOver}(C, '\text{single\_datasets}')) \mapsto \left\{o: \text{collection}<\text{list}:\text{list},\left\{\text{o1}=\left\{\text{inner}=tool(i=C_AS_UNPAIRED_1)[o]\right\},...,\text{on}=\left\{\text{inner}=tool(i=C_AS_UNPAIRED_n)[o]\right\}\right\}>\right\}$$ + +::: + + + +:::{admonition} Example: `MAPPING_LIST_LIST_OVER_LIST_PAIRED_OR_UNPAIRED` +:class: note + +Assuming, + +* $ d_1,...,d_n $ are datasets +* $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ +* $ C $ is $ \text{CollectionInstance<}\text{list}:\text{list},\left\{ \text{ o1 }=\left\{ \text{ inner }=d_1 \right\}, ..., \text{ on }=\left\{ \text{ inner }=d_n \right\} \right\}\text{>} $ +* $ C_AS_LPU_i = \text{inner} \text{list} of C at position i, treated as \text{list}:\text{paired\_or\_unpaired} via \text{single\_datasets} $ + + +then + +$$tool(i=\text{mapOver}(C, '\text{list}:\text{paired\_or\_unpaired}')) \mapsto \left\{o: \text{collection}<\text{list},\left\{\text{o1}=tool(i=C_AS_LPU_1)[o],...,\text{on}=tool(i=C_AS_LPU_n)[o]\right\}>\right\}$$ + +::: + + + +

+ + + +Due only to implementation time, the special casing of allowing paired_or_unpaired act as both datasets and paired collections only works when it is the deepest collection type. So while list:paired can be consumed by a list:paired_or_unpaired input, a paired:list cannot be consumed by a paired_or_unpaired:list input though @@ -760,3 +870,251 @@ more likely to be used in actual Galaxy analyses given current and guessed futur usage. +## sample_sheet Collections + +The collection type ``sample_sheet`` attaches typed, columnar metadata to each +element of a dataset collection. For mapping and type matching purposes, +``sample_sheet`` behaves identically to ``list`` - it can be mapped over tool +inputs, matched against ``list`` collection inputs, and composed with inner types +(``sample_sheet:paired``, ``sample_sheet:paired_or_unpaired``). The key asymmetry +is that while a ``sample_sheet`` output can satisfy a ``list`` input, a ``list`` +output cannot satisfy a ``sample_sheet`` input - sample sheets carry metadata +that plain lists do not. + + +(SAMPLE_SHEET_MAPPING)= +(SAMPLE_SHEET_MATCHES_LIST)= +
Examples + +:::{admonition} Example: `SAMPLE_SHEET_MAPPING` +:class: note + +Assuming, + +* $ d_1,...,d_n $ are datasets +* $ tool \text{ is } (i: \text{ dataset }) \Rightarrow \{ o: \text{ dataset } \} $ +* $ C $ is $ \text{CollectionInstance<}\text{sample\_sheet},\left\{ \text{ i1 }=d_1, ..., \text{ in }=d_n \right\}\text{>} $ + + +then + +$$tool(i=\text{mapOver}(C)) \mapsto \left\{o: \text{collection}<\text{sample\_sheet},\left\{\text{i1}=tool(i=d_1)[o],...,\text{in}=tool(i=d_n)[o]\right\}>\right\}$$ + +::: + + + +:::{admonition} Example: `SAMPLE_SHEET_MATCHES_LIST` +:class: note + +Assuming, + +* $ d_1,...,d_n $ are datasets +* $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ +* $ C $ is $ \text{CollectionInstance<}\text{sample\_sheet},\left\{ \text{ i1 }=d_1, ..., \text{ in }=d_n \right\}\text{>} $ + + +then + +$$tool(i=C) \rightarrow \left\{o: \text{dataset}\right\}$$ + +::: + + + +

+ + + +Sub-collection mapping works the same as for ``list`` composites. A +``sample_sheet:paired`` can be mapped over a ``paired`` collection input, +extracting each inner pair and producing a ``sample_sheet`` implicit output. + + +(SAMPLE_SHEET_PAIRED_MAPPING_OVER_PAIRED)= +(SAMPLE_SHEET_PAIRED_MATCHES_LIST_PAIRED)= +
Examples + +:::{admonition} Example: `SAMPLE_SHEET_PAIRED_MAPPING_OVER_PAIRED` +:class: note + +Assuming, + +* $ d_f $, $ d_r $ are datasets +* $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ +* $ C $ is $ \text{CollectionInstance<}\text{sample\_sheet}:\text{paired},\left\{ \text{ el1 }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\} \right\}\text{>} $ +* $ C\_PAIRED $ is $ \text{CollectionInstance<}\text{paired},\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\}\text{>} $ + + +then + +$$tool(i=\text{mapOver}(C, '\text{paired}')) \mapsto \left\{o: \text{collection}<\text{sample\_sheet},\left\{\text{el1}=tool(i=C\_PAIRED)[o]\right\}>\right\}$$ + +::: + + + +:::{admonition} Example: `SAMPLE_SHEET_PAIRED_MATCHES_LIST_PAIRED` +:class: note + +Assuming, + +* $ d_f $, $ d_r $ are datasets +* $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ +* $ C $ is $ \text{CollectionInstance<}\text{sample\_sheet}:\text{paired},\left\{ \text{ el1 }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\} \right\}\text{>} $ + + +then + +$$tool(i=C) \rightarrow \left\{o: \text{dataset}\right\}$$ + +::: + + + +

+ + + +The ``paired_or_unpaired`` integration rules carry over from ``list`` to +``sample_sheet``. A flat ``sample_sheet`` can be mapped over a +``paired_or_unpaired`` input via ``single_datasets`` sub-collection mapping, +and ``sample_sheet:paired`` can be mapped over ``paired_or_unpaired`` just as +``list:paired`` can. + + +(SAMPLE_SHEET_MAPPING_OVER_PAIRED_OR_UNPAIRED)= +(SAMPLE_SHEET_PAIRED_MAPPING_OVER_PAIRED_OR_UNPAIRED)= +(SAMPLE_SHEET_PAIRED_OR_UNPAIRED_MATCHES_LIST_PAIRED_OR_UNPAIRED)= +
Examples + +:::{admonition} Example: `SAMPLE_SHEET_MAPPING_OVER_PAIRED_OR_UNPAIRED` +:class: note + +Assuming, + +* $ d_1,...,d_n $ are datasets +* $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ +* $ C $ is $ \text{CollectionInstance<}\text{sample\_sheet},\left\{ \text{ i1 }=d_1, ..., \text{ in }=d_n \right\}\text{>} $ +* $ C_AS_UNPAIRED_i = CollectionInstance<\text{paired\_or\_unpaired},\left\{\text{unpaired}=di\right\}> for i from 1...n $ + + +then + +$$tool(i=\text{mapOver}(C, '\text{single\_datasets}')) \mapsto \left\{o: \text{collection}<\text{sample\_sheet},\left\{\text{i1}=tool(i=C_AS_UNPAIRED_1)[o],...,\text{in}=tool(i=C_AS_UNPAIRED_n)[o]\right\}>\right\}$$ + +::: + + + +:::{admonition} Example: `SAMPLE_SHEET_PAIRED_MAPPING_OVER_PAIRED_OR_UNPAIRED` +:class: note + +Assuming, + +* $ d_f $, $ d_r $ are datasets +* $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ +* $ C $ is $ \text{CollectionInstance<}\text{sample\_sheet}:\text{paired},\left\{ \text{ el1 }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\} \right\}\text{>} $ +* $ C_AS_MIXED $ is $ \text{CollectionInstance<}\text{sample\_sheet}:\text{paired\_or\_unpaired},\left\{ \text{ el1 }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\} \right\}\text{>} $ + + +then + +$$tool(i=\text{mapOver}(C)) == tool(i=\text{mapOver}(C_AS_MIXED))$$ + +::: + + + +:::{admonition} Example: `SAMPLE_SHEET_PAIRED_OR_UNPAIRED_MATCHES_LIST_PAIRED_OR_UNPAIRED` +:class: note + +Assuming, + +* $ d_f $, $ d_r $ are datasets +* $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ +* $ C $ is $ \text{CollectionInstance<}\text{sample\_sheet}:\text{paired\_or\_unpaired},\left\{ \text{ el1 }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\} \right\}\text{>} $ + + +then + +$$tool(i=C) \rightarrow \left\{o: \text{dataset}\right\}$$ + +::: + + + +

+ + + +The type matching is asymmetric: a ``sample_sheet`` output can satisfy a ``list`` +input because sample sheets carry all the structural information lists have (plus +metadata). However, a ``list`` output cannot satisfy a ``sample_sheet`` input because +lists lack the ``column_definitions`` and per-element ``columns`` metadata that +sample sheet consumers expect. + + +(LIST_NOT_MATCHES_SAMPLE_SHEET)= +(LIST_PAIRED_NOT_MATCHES_SAMPLE_SHEET_PAIRED)= +(SAMPLE_SHEET_MATCHES_SAMPLE_SHEET)= +
Examples + +:::{admonition} Example: `LIST_NOT_MATCHES_SAMPLE_SHEET` +:class: note + +Assuming, + +* $ d_1,...,d_n $ are datasets +* $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ +* $ C $ is $ \text{CollectionInstance<}\text{list},\left\{ \text{ i1 }=d_1, ..., \text{ in }=d_n \right\}\text{>} $ + + +then + +$$tool(i=C)\text{ is invalid}$$ + +::: + + + +:::{admonition} Example: `LIST_PAIRED_NOT_MATCHES_SAMPLE_SHEET_PAIRED` +:class: note + +Assuming, + +* $ d_f $, $ d_r $ are datasets +* $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ +* $ C $ is $ \text{CollectionInstance<}\text{list}:\text{paired},\left\{ \text{ el1 }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\} \right\}\text{>} $ + + +then + +$$tool(i=C)\text{ is invalid}$$ + +::: + + + +:::{admonition} Example: `SAMPLE_SHEET_MATCHES_SAMPLE_SHEET` +:class: note + +Assuming, + +* $ d_1,...,d_n $ are datasets +* $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ +* $ C $ is $ \text{CollectionInstance<}\text{sample\_sheet},\left\{ \text{ i1 }=d_1, ..., \text{ in }=d_n \right\}\text{>} $ + + +then + +$$tool(i=C) \rightarrow \left\{o: \text{dataset}\right\}$$ + +::: + + + +

+ + + diff --git a/lib/galaxy/model/dataset_collections/type_description.py b/lib/galaxy/model/dataset_collections/type_description.py index 77e7cec37e9..7c7f6950a40 100644 --- a/lib/galaxy/model/dataset_collections/type_description.py +++ b/lib/galaxy/model/dataset_collections/type_description.py @@ -71,6 +71,34 @@ class CollectionTypeDescription: if subcollection_type == "single_datasets": return self.collection_type + normalized = _normalize_collection_type(self.collection_type) + normalized_sub = _normalize_collection_type(subcollection_type) + + if subcollection_type == "paired_or_unpaired": + if self.collection_type.endswith(":paired"): + # paired_or_unpaired consumes the :paired suffix + return self.collection_type[: -len(":paired")] + elif normalized.endswith("list"): + # paired_or_unpaired acts like single_datasets for collections + # whose innermost type is list (each element wrapped as unpaired) + return self.collection_type + else: + # strip last rank (paired_or_unpaired consumes it) + return self.collection_type[: self.collection_type.rfind(":")] + + if normalized_sub.endswith(":paired_or_unpaired"): + # Compound :paired_or_unpaired suffix — iterative peel-off matching + # TS effectiveMapOver logic. Strip ranks from both sides, then + # optionally strip one more if :paired was consumed. + current = self.collection_type + current_other = subcollection_type + while ":" in current_other: + current_other = current_other[: current_other.rfind(":")] + current = current[: current.rfind(":")] + if normalized.endswith(":paired"): + current = current[: current.rfind(":")] + return current + return self.collection_type[: -(len(subcollection_type) + 1)] def has_subcollections_of_type(self, other_collection_type) -> bool: @@ -88,12 +116,23 @@ class CollectionTypeDescription: other_collection_type = _normalize_collection_type(other_collection_type) if collection_type == other_collection_type: return False - if collection_type.endswith(other_collection_type): + if collection_type.endswith(f":{other_collection_type}"): return True if other_collection_type == "paired_or_unpaired": # this can be thought of as a subcollection of anything except a pair # since it would match a pair exactly return collection_type != "paired" + if other_collection_type.endswith(":paired_or_unpaired"): + # Compound :paired_or_unpaired suffix — e.g. list:list can map over + # list:paired_or_unpaired. Strip the :paired_or_unpaired to get the + # required higher ranks, optionally strip :paired from self (since + # paired_or_unpaired consumes paired), then check alignment. + higher_ranks_required = other_collection_type[: other_collection_type.rfind(":")] + if collection_type.endswith(":paired"): + higher_ranks = collection_type[: collection_type.rfind(":")] + else: + higher_ranks = collection_type + return higher_ranks.endswith(higher_ranks_required) and higher_ranks != higher_ranks_required if other_collection_type == "single_datasets": # effectively any collection has unpaired subcollections return True diff --git a/lib/galaxy/model/dataset_collections/types/collection_semantics.yml b/lib/galaxy/model/dataset_collections/types/collection_semantics.yml index d044ba11bb5..650e39e8427 100644 --- a/lib/galaxy/model/dataset_collections/types/collection_semantics.yml +++ b/lib/galaxy/model/dataset_collections/types/collection_semantics.yml @@ -9,7 +9,7 @@ Any significantly sophisticated workflow language will have ways to collect data into arrays or vectors or dictionaries and apply operations across this data (mapping) - or reduce the dimensionality of this data (reductions). Typically, this explicitly + or reduce the dimensionality of this data (reductions). Typically, this is explicitly annotated with map functions or for loops. Galaxy however is designed to be a point and click interface for connecting steps and running tools. It is important that steps just connect and just do the most natural thing - and this is what Galaxy does. @@ -38,11 +38,30 @@ out: {o: dataset} - collections: C: [paired, {forward: d_f, reverse: d_r}] - then: "tool(i=mapOver(C)) ~> {o: collection }" + then: + type: map_over + invocation: + inputs: + i: {type: map_over, collection: C} + produces: + o: + type: collection + collection_type: paired + elements: + forward: + type: tool_output_ref + invocation: {inputs: {i: {type: dataset, ref: d_f}}} + output: o + reverse: + type: tool_output_ref + invocation: {inputs: {i: {type: dataset, ref: d_r}}} + output: o tests: tool_runtime: api_test: "test_tool_execute.py::test_map_over_collection" - wf_editor: "accepts paired data -> data connection" + workflow_runtime: + framework_test: "collection_semantics_cat_0" + workflow_editor: "accepts paired data -> data connection" - example: label: BASIC_MAPPING_PAIRED_OR_UNPAIRED_PAIRED @@ -53,10 +72,29 @@ out: {o: dataset} - collections: C: [paired_or_unpaired, {forward: d_f, reverse: d_r}] - then: "tool(i=mapOver(C)) ~> {o: collection}" + then: + type: map_over + invocation: + inputs: + i: {type: map_over, collection: C} + produces: + o: + type: collection + collection_type: paired_or_unpaired + elements: + forward: + type: tool_output_ref + invocation: {inputs: {i: {type: dataset, ref: d_f}}} + output: o + reverse: + type: tool_output_ref + invocation: {inputs: {i: {type: dataset, ref: d_r}}} + output: o tests: tool_runtime: - api_test: "test_tool_execute.py::test_map_over_data_with_paired_or_unpaired_unpaired" + api_test: "test_tool_execute.py::test_map_over_data_with_paired_or_unpaired_paired" + workflow_runtime: + framework_test: "collection_semantics_cat_1" workflow_editor: "accepts paired_or_unpaired data -> data connection" - example: @@ -68,10 +106,25 @@ out: {o: dataset} - collections: C: [paired_or_unpaired, {unpaired: d_u}] - then: "tool(i=mapOver(C)) ~> {o: collection}" + then: + type: map_over + invocation: + inputs: + i: {type: map_over, collection: C} + produces: + o: + type: collection + collection_type: paired_or_unpaired + elements: + unpaired: + type: tool_output_ref + invocation: {inputs: {i: {type: dataset, ref: d_u}}} + output: o tests: tool_runtime: - api_test: "test_tool_execute.py::test_map_over_data_with_paired_or_unpaired_paired" + api_test: "test_tool_execute.py::test_map_over_data_with_paired_or_unpaired_unpaired" + workflow_runtime: + framework_test: "collection_semantics_cat_2" workflow_editor: "accepts paired_or_unpaired data -> data connection" - example: @@ -83,7 +136,25 @@ out: {o: dataset} - collections: C: [list, {i1: "d_1", ..., in: "d_n"}] - then: "tool(i=mapOver(C)) ~> {o: collection}" + then: + type: map_over + invocation: + inputs: + i: {type: map_over, collection: C} + produces: + o: + type: collection + collection_type: list + elements: + i1: + type: tool_output_ref + invocation: {inputs: {i: {type: dataset, ref: d_1}}} + output: o + "...": {type: ellipsis} + in: + type: tool_output_ref + invocation: {inputs: {i: {type: dataset, ref: d_n}}} + output: o tests: workflow_editor: "accepts collection data -> data connection" @@ -100,7 +171,31 @@ out: {o: dataset} - collections: C: ["list:list", {o1: {inner: d_1}, ..., "on": {inner: d_n}}] - then: "tool(i=mapOver(C)) ~> {o: collection}" + then: + type: map_over + invocation: + inputs: + i: {type: map_over, collection: C} + produces: + o: + type: collection + collection_type: "list:list" + elements: + o1: + type: nested_elements + elements: + inner: + type: tool_output_ref + invocation: {inputs: {i: {type: dataset, ref: d_1}}} + output: o + "...": {type: ellipsis} + "on": + type: nested_elements + elements: + inner: + type: tool_output_ref + invocation: {inputs: {i: {type: dataset, ref: d_n}}} + output: o tests: tool_runtime: api_test: "test_tools.py::TestToolsApi::test_map_over_nested_collections" @@ -115,10 +210,32 @@ out: {o: dataset} - collections: C: ["list:paired_or_unpaired", {el1: {forward: d_f, reverse: d_r}}] - then: "tool(i=mapOver(C)) ~> {o: collection}" + then: + type: map_over + invocation: + inputs: + i: {type: map_over, collection: C} + produces: + o: + type: collection + collection_type: "list:paired_or_unpaired" + elements: + el1: + type: nested_elements + elements: + forward: + type: tool_output_ref + invocation: {inputs: {i: {type: dataset, ref: d_f}}} + output: o + reverse: + type: tool_output_ref + invocation: {inputs: {i: {type: dataset, ref: d_r}}} + output: o tests: tool_runtime: api_test: "test_tool_execute.py::test_map_over_data_with_list_paired_or_unpaired" + workflow_runtime: + framework_test: "collection_semantics_cat_3" workflow_editor: "accepts list:paired_or_unpaired data -> data connection" - doc: | @@ -136,10 +253,31 @@ out: {o: dataset} - collections: C: ["list", {i1: d_1, ..., in: d_n}] - then: "tool(i=mapOver(C),i2=d_o) ~> {o: collection}" + then: + type: map_over + invocation: + inputs: + i: {type: map_over, collection: C} + i2: {type: dataset, ref: d_o} + produces: + o: + type: collection + collection_type: list + elements: + i1: + type: tool_output_ref + invocation: {inputs: {i: {type: dataset, ref: d_1}, i2: {type: dataset, ref: d_o}}} + output: o + "...": {type: ellipsis} + in: + type: tool_output_ref + invocation: {inputs: {i: {type: dataset, ref: d_n}, i2: {type: dataset, ref: d_o}}} + output: o tests: tool_runtime: api_test: "test_tools.py::TestToolsApi::test_map_over_collected_and_individual_datasets" + workflow_runtime: + framework_test: "collection_semantics_cat_two_inputs_0" - doc: | If a tool consumes two input datasets and produces one output dataset, you can map two @@ -168,20 +306,41 @@ - collections: C1: ["list", {i1: d1_1, ..., in: d1_n}] C2: ["list", {i1: d2_1, ..., in: d2_n}] - then: "tool(i=mapOver(C1), i2=mapOver(C2)) ~> {o: collection}" + then: + type: map_over + invocation: + inputs: + i: {type: map_over, collection: C1} + i2: {type: map_over, collection: C2} + produces: + o: + type: collection + collection_type: list + elements: + i1: + type: tool_output_ref + invocation: {inputs: {i: {type: dataset, ref: d1_1}, i2: {type: dataset, ref: d2_1}}} + output: o + "...": {type: ellipsis} + in: + type: tool_output_ref + invocation: {inputs: {i: {type: dataset, ref: d1_n}, i2: {type: dataset, ref: d2_n}}} + output: o tests: tool_runtime: api_test: test_tools.py::TestToolsApi::test_map_over_two_collections + workflow_runtime: + framework_test: "collection_semantics_cat_two_inputs_1" - doc: "## Reduction" -- doc: | +- doc: | Not all tool executions result in implicit collections and mapping over inputs. Tool inputs of ``type`` ``data_collection`` can consume collections directly and do not necessarily result in mapping over. Tools that consume collections and output datasets effectively reduce the dimension of the Galaxy data structure. When used at runtime - this is often referred to a "reduction" in the code. + this is often referred to as a "reduction" in the code. - example: label: COLLECTION_INPUT_PAIRED @@ -192,23 +351,37 @@ out: {o: dataset} - collections: C: [paired, {forward: d_f, reverse: d_r}] - then: "tool(i=C) -> {o: dataset}" + then: + type: reduction + invocation: + inputs: + i: {type: collection, ref: C} + produces: + o: {type: dataset} tests: - tool_runtime: + tool_runtime: tool: collection_paired_test workflow_editor: "accepts paired -> paired connection" - example: label: COLLECTION_INPUT_LIST assumptions: - - datasets: ["d1,...,dn"] + - datasets: ["d_1,...,d_n"] - tool: in: {i: "collection"} out: {o: dataset} - collections: C: [list, {el1: d_1, ..., eln: d_n}] - then: "tool(i=C) -> {o: dataset}" + then: + type: reduction + invocation: + inputs: + i: {type: collection, ref: C} + produces: + o: {type: dataset} tests: + workflow_runtime: + framework_test: "collection_semantics_cat_collection_0" workflow_editor: "accepts list -> list connection" - example: @@ -220,10 +393,18 @@ out: {o: dataset} - collections: C: [paired_or_unpaired, {forward: d_f, reverse: d_r}] - then: "tool(i=C) -> {o: dataset}" + then: + type: reduction + invocation: + inputs: + i: {type: collection, ref: C} + produces: + o: {type: dataset} tests: tool_runtime: tool: collection_paired_or_unpaired + workflow_runtime: + framework_test: "collection_semantics_paired_or_unpaired_0" workflow_editor: "accepts paired_or_unpaired data -> paired_or_unpaired connection" - example: @@ -235,10 +416,18 @@ out: {o: dataset} - collections: C: ["list:paired_or_unpaired", {el1: {forward: d_f, reverse: d_r}}] - then: "tool(i=C) -> {o: dataset}" + then: + type: reduction + invocation: + inputs: + i: {type: collection, ref: C} + produces: + o: {type: dataset} tests: tool_runtime: tool: collection_list_paired_or_unpaired + workflow_runtime: + framework_test: "collection_semantics_list_paired_or_unpaired_0" workflow_editor: "accepts list:paired_or_unpaired data -> list:paired_or_unpaired connection" - doc: | @@ -250,11 +439,15 @@ assumptions: - datasets: [d_f, d_r] - tool: - in: {i: "collection"} + in: {i: "collection"} out: {o: dataset} - collections: C: [paired, {forward: d_f, reverse: d_r}] - then: "tool(i=C)" + then: + type: invalid + invocation: + inputs: + i: {type: collection, ref: C} is_valid: false tests: workflow_editor: "rejects connecting paired -> list" @@ -264,22 +457,52 @@ assumptions: - datasets: ["d_1,...,d_n"] - tool: - in: {i: "collection"} + in: {i: "collection"} out: {o: dataset} - collections: C: [list, {i1: d_1, ..., in: d_n}] - then: "tool(i=C)" + then: + type: invalid + invocation: + inputs: + i: {type: collection, ref: C} is_valid: false tests: workflow_editor: "rejects connecting list -> paired" - example: label: COLLECTION_INPUT_LIST_PAIRED_NOT_CONSUMES_PAIRED_PAIRED + assumptions: + - datasets: [d_f, d_r] + - tool: + in: {i: "collection"} + out: {o: dataset} + - collections: + C: ["paired:paired", {forward: {forward: d_f, reverse: d_r}, reverse: {forward: d_f, reverse: d_r}}] + then: + type: invalid + invocation: + inputs: + i: {type: collection, ref: C} + is_valid: false tests: workflow_editor: "rejects paired:paired -> list:paired connection" - example: label: COLLECTION_INPUT_LIST_PAIRED_OR_NOT_PAIRED_NOT_CONSUMES_PAIRED_PAIRED + assumptions: + - datasets: [d_f, d_r] + - tool: + in: {i: "collection"} + out: {o: dataset} + - collections: + C: ["paired:paired", {forward: {forward: d_f, reverse: d_r}, reverse: {forward: d_f, reverse: d_r}}] + then: + type: invalid + invocation: + inputs: + i: {type: collection, ref: C} + is_valid: false tests: workflow_editor: "rejects paired:paired -> list:paired_or_unpaired connection" @@ -292,15 +515,24 @@ label: LIST_REDUCTION assumptions: - datasets: ["d_1,...,d_n"] - - tool: + - tool: in: {i: "dataset"} out: {o: dataset} - collections: C: [list, {i1: d_1, ..., in: d_n}] - then: "tool(i=C) == tool(i=[d_1,...,d_n])" + then: + type: equivalence + left: + inputs: + i: {type: collection, ref: C} + right: + inputs: + i: {type: dataset_list, refs: ["d_1", "...", "d_n"]} tests: tool_runtime: - api_test: "test_tools.py::TestToolsApi::test_tools.py::test_reduce_collections" + api_test: "test_tools.py::TestToolsApi::test_reduce_collections" + workflow_runtime: + framework_test: "collection_semantics_multi_data_optional_0" workflow_editor: "treats multi data input as list input" - doc: | @@ -311,12 +543,16 @@ label: PAIRED_REDUCTION_INVALID assumptions: - datasets: [d_f, d_r] - - tool: + - tool: in: {i: "dataset"} out: {o: dataset} - collections: C: [paired, {forward: d_f, reverse: d_r}] - then: "tool(i=C)" + then: + type: invalid + invocation: + inputs: + i: {type: collection, ref: C} is_valid: false tests: workflow_editor: "rejects paired input on multi-data input" @@ -325,12 +561,16 @@ label: PAIRED_OR_UNPAIRED_REDUCTION_INVALID assumptions: - datasets: [d_f, d_r] - - tool: + - tool: in: {i: "dataset"} out: {o: dataset} - collections: - C: [paired_or_unpaired,{forward=d_f, reverse=d_r}] - then: "tool(i=C)" + C: [paired_or_unpaired, {forward: d_f, reverse: d_r}] + then: + type: invalid + invocation: + inputs: + i: {type: collection, ref: C} is_valid: false tests: workflow_editor: "rejects paired_or_unpaired input on multi-data input" @@ -349,14 +589,27 @@ - collections: C: ["list:paired", {el1: {forward: d_f, reverse: d_r}}] "C\\_PAIRED": [paired, {forward: d_f,reverse: d_r}] - then: "tool(i=mapOver(C, 'paired')) ~> {o: collection}" + then: + type: map_over + invocation: + inputs: + i: {type: map_over, collection: C, sub_collection_type: paired} + produces: + o: + type: collection + collection_type: list + elements: + el1: + type: tool_output_ref + invocation: {inputs: {i: {type: collection, ref: "C\\_PAIRED"}}} + output: o tests: tool_runtime: - api_test: "test_tools.py::TestToolsApi::test_subcollection_mapping" + api_test: "test_tool_execute.py::test_simple_subcollection_mapping" workflow_editor: "accepts list:paired -> paired connection" - doc: | - The natural extension of multiple data input parameters consuming list collections as describe + The natural extension of multiple data input parameters consuming list collections as described above when discussing reductions is that nested lists of lists (``list:list``) can be mapped over a multiple data input parameter. Each nested list will be reduced by this operation but the results will be mapped over. The result will be a list with the same structure as the outer list @@ -371,7 +624,25 @@ out: {o: dataset} - collections: C: ["list:list", {o1: {inner: d_1}, ..., "on": {inner: d_n}}] - then: "tool(i=mapOver(C, 'list')) ~> {o: collection}" + then: + type: map_over + invocation: + inputs: + i: {type: map_over, collection: C, sub_collection_type: list} + produces: + o: + type: collection + collection_type: list + elements: + o1: + type: tool_output_ref + invocation: {inputs: {i: {type: dataset_list, refs: ["d_1"]}}} + output: o + "...": {type: ellipsis} + "on": + type: tool_output_ref + invocation: {inputs: {i: {type: dataset_list, refs: ["d_n"]}}} + output: o tests: workflow_editor: "maps list:list over multi data input" @@ -384,12 +655,16 @@ label: LIST_PAIRED_REDUCTION_INVALID assumptions: - datasets: [d_f, d_r] - - tool: + - tool: in: {i: "dataset"} out: {o: dataset} - collections: - C: [list:paired, {forward: d_f, reverse: d_r}] - then: "tool(i=mapOver(C, 'paired'))" + C: ["list:paired", {el1: {forward: d_f, reverse: d_r}}] + then: + type: invalid + invocation: + inputs: + i: {type: map_over, collection: C, sub_collection_type: paired} is_valid: false tests: workflow_editor: "rejects list:paired input on multi-data input" @@ -398,12 +673,16 @@ label: LIST_PAIRED_OR_UNPAIRED_REDUCTION_INVALID assumptions: - datasets: [d_f, d_r] - - tool: + - tool: in: {i: "dataset"} out: {o: dataset} - collections: - C: ["list:paired_or_unpaired", {forward: d_f, reverse: d_r}] - then: "tool(i=mapOver(C, 'paired_or_unpaired'))" + C: ["list:paired_or_unpaired", {el1: {forward: d_f, reverse: d_r}}] + then: + type: invalid + invocation: + inputs: + i: {type: map_over, collection: C, sub_collection_type: paired_or_unpaired} is_valid: false tests: workflow_editor: "rejects list:paired_or_unpaired input on multi-data input" @@ -429,15 +708,24 @@ - collections: C: [paired, {forward: d_f, reverse: d_r}] - "C_AS_MIXED = CollectionInstance" - then: "tool(i=C) == tool(i=C_AS_MIXED)" + then: + type: equivalence + left: + inputs: + i: {type: collection, ref: C} + right: + inputs: + i: {type: collection, ref: C_AS_MIXED} tests: tool_runtime: tool: collection_paired_or_unpaired + workflow_runtime: + framework_test: "collection_semantics_paired_or_unpaired_1" workflow_editor: "accepts paired -> paired_or_unpaired connection" - doc: | - This inverse of this doesn't work intentionally. In some ways a ``paired`` collection + The inverse of this doesn't work intentionally. In some ways a ``paired`` collection acts as a ``paired_or_unpaired`` collection but a ``paired_or_unpaired`` is not a paired collection. This makes a lot of sense in terms of tools - a tool consuming a ``paired`` dataset expects to find both a ``forward`` and ``reverse`` element but these may not exist @@ -451,8 +739,13 @@ in: {i: "collection"} out: {o: dataset} - collections: - C: [paired_or_unpaired, {forward=d_f, reverse: d_r}] - then: "tool(i=C) is invalid" + C: [paired_or_unpaired, {forward: d_f, reverse: d_r}] + then: + type: invalid + invocation: + inputs: + i: {type: collection, ref: C} + is_valid: false tests: workflow_editor: "rejects paired_or_unpaired -> paired connection" @@ -473,8 +766,17 @@ - collections: C: ["list:paired", {el: {forward: d_f, reverse: d_r}}] C_AS_MIXED: ["list:paired_or_unpaired", {el: {forward: d_f, reverse: d_r}}] - then: "tool(i=mapOver(C)) == tool(i=mapOver(C_AS_MIXED))" + then: + type: equivalence + left: + inputs: + i: {type: map_over, collection: C} + right: + inputs: + i: {type: map_over, collection: C_AS_MIXED} tests: + workflow_runtime: + framework_test: "collection_semantics_paired_or_unpaired_2" workflow_editor: "accepts list:paired -> paired_or_unpaired connection" - example: @@ -485,9 +787,16 @@ in: {i: "collection"} out: {o: dataset} - collections: - C: [list:paired_or_unpaired, {el: {forward: f, reverse: r}}] - then: "tool(i=mapOver(C)) is invalid" + C: ["list:paired_or_unpaired", {el: {forward: d_f, reverse: d_r}}] + then: + type: invalid + invocation: + inputs: + i: {type: map_over, collection: C} + is_valid: false tests: + tool_runtime: + api_test: "test_tool_execute.py::test_paired_or_unpaired_not_consumed_by_paired_when_mapping" workflow_editor: "rejects list:paired_or_unpaired -> paired connection" - example: @@ -498,8 +807,13 @@ in: {i: "collection"} out: {o: dataset} - collections: - C: [list:paired_or_unpaired, {el: {forward: f, reverse: r}}] - then: "tool(i=mapOver(C)) is invalid" + C: ["list:paired_or_unpaired", {el: {forward: d_f, reverse: d_r}}] + then: + type: invalid + invocation: + inputs: + i: {type: map_over, collection: C} + is_valid: false tests: workflow_editor: "rejects list:paired_or_unpaired -> list connection" @@ -510,7 +824,25 @@ - example: label: MAPPING_LIST_LIST_PAIRED_OVER_PAIRED_OR_UNPAIRED + assumptions: + - datasets: [d_f, d_r] + - tool: + in: {i: "collection"} + out: {o: dataset} + - collections: + C: ["list:list:paired", {o1: {el1: {forward: d_f, reverse: d_r}}}] + C_AS_MIXED: ["list:list:paired_or_unpaired", {o1: {el1: {forward: d_f, reverse: d_r}}}] + then: + type: equivalence + left: + inputs: + i: {type: map_over, collection: C} + right: + inputs: + i: {type: map_over, collection: C_AS_MIXED} tests: + workflow_runtime: + framework_test: "collection_semantics_paired_or_unpaired_3" workflow_editor: "accepts list:list:paired -> paired_or_unpaired connection" - doc: | @@ -529,10 +861,30 @@ - collections: C: [list, {i1: d_1, ..., in: d_n}] - "C_AS_UNPAIRED_i = CollectionInstance for i from 1...n" - then: "tool(i=mapOver(C, 'single_datasets')) ~> {o: collection}" + then: + type: map_over + invocation: + inputs: + i: {type: map_over, collection: C, sub_collection_type: single_datasets} + produces: + o: + type: collection + collection_type: list + elements: + i1: + type: tool_output_ref + invocation: {inputs: {i: {type: collection, ref: C_AS_UNPAIRED_1}}} + output: o + "...": {type: ellipsis} + in: + type: tool_output_ref + invocation: {inputs: {i: {type: collection, ref: C_AS_UNPAIRED_n}}} + output: o tests: tool_runtime: api_test: "test_tool_execute.py::test_map_over_paired_or_unpaired_with_list" + workflow_runtime: + framework_test: "collection_semantics_paired_or_unpaired_7" workflow_editor: "accepts list -> paired_or_unpaired connection" - doc: | @@ -545,19 +897,354 @@ - example: label: MAPPING_LIST_LIST_OVER_PAIRED_OR_UNPAIRED + assumptions: + - datasets: ["d_1,...,d_n"] + - tool: + in: {i: "collection"} + out: {o: dataset} + - collections: + C: ["list:list", {o1: {inner: d_1}, ..., "on": {inner: d_n}}] + - "C_AS_UNPAIRED_j = CollectionInstance for each dataset dj in C" + then: + type: map_over + invocation: + inputs: + i: {type: map_over, collection: C, sub_collection_type: single_datasets} + produces: + o: + type: collection + collection_type: "list:list" + elements: + o1: + type: nested_elements + elements: + inner: + type: tool_output_ref + invocation: {inputs: {i: {type: collection, ref: C_AS_UNPAIRED_1}}} + output: o + "...": {type: ellipsis} + "on": + type: nested_elements + elements: + inner: + type: tool_output_ref + invocation: {inputs: {i: {type: collection, ref: C_AS_UNPAIRED_n}}} + output: o tests: workflow_editor: "accepts list:list -> paired_or_unpaired connection" - example: label: MAPPING_LIST_LIST_OVER_LIST_PAIRED_OR_UNPAIRED + assumptions: + - datasets: ["d_1,...,d_n"] + - tool: + in: {i: "collection"} + out: {o: dataset} + - collections: + C: ["list:list", {o1: {inner: d_1}, ..., "on": {inner: d_n}}] + - "C_AS_LPU_i = inner list of C at position i, treated as list:paired_or_unpaired via single_datasets" + then: + type: map_over + invocation: + inputs: + i: {type: map_over, collection: C, sub_collection_type: "list:paired_or_unpaired"} + produces: + o: + type: collection + collection_type: list + elements: + o1: + type: tool_output_ref + invocation: {inputs: {i: {type: collection, ref: C_AS_LPU_1}}} + output: o + "...": {type: ellipsis} + "on": + type: tool_output_ref + invocation: {inputs: {i: {type: collection, ref: C_AS_LPU_n}}} + output: o tests: + tool_runtime: + api_test: "test_tool_execute.py::test_map_over_list_paired_or_unpaired_with_list_of_lists" + workflow_runtime: + framework_test: "collection_semantics_list_paired_or_unpaired_2" workflow_editor: "accepts list:list -> list:paired_or_unpaired connection" - doc: | - Due only implementation time, the special casing of allowing paired_or_unpaired + Due only to implementation time, the special casing of allowing paired_or_unpaired act as both datasets and paired collections only works when it is the deepest collection type. So while list:paired can be consumed by a list:paired_or_unpaired input, a paired:list cannot be consumed by a paired_or_unpaired:list input though it should be able to for consistency. We have focused our time on data structures more likely to be used in actual Galaxy analyses given current and guessed future usage. + +- doc: "## sample_sheet Collections" +- doc: | + The collection type ``sample_sheet`` attaches typed, columnar metadata to each + element of a dataset collection. For mapping and type matching purposes, + ``sample_sheet`` behaves identically to ``list`` - it can be mapped over tool + inputs, matched against ``list`` collection inputs, and composed with inner types + (``sample_sheet:paired``, ``sample_sheet:paired_or_unpaired``). The key asymmetry + is that while a ``sample_sheet`` output can satisfy a ``list`` input, a ``list`` + output cannot satisfy a ``sample_sheet`` input - sample sheets carry metadata + that plain lists do not. + +- example: + label: SAMPLE_SHEET_MAPPING + assumptions: + - datasets: ["d_1,...,d_n"] + - tool: + in: {i: dataset} + out: {o: dataset} + - collections: + C: [sample_sheet, {i1: "d_1", ..., in: "d_n"}] + then: + type: map_over + invocation: + inputs: + i: {type: map_over, collection: C} + produces: + o: + type: collection + collection_type: sample_sheet + elements: + i1: + type: tool_output_ref + invocation: {inputs: {i: {type: dataset, ref: d_1}}} + output: o + "...": {type: ellipsis} + in: + type: tool_output_ref + invocation: {inputs: {i: {type: dataset, ref: d_n}}} + output: o + tests: + workflow_runtime: + framework_test: "collection_semantics_cat_4" + workflow_editor: "accepts sample_sheet data -> data connection (maps like list)" + +- example: + label: SAMPLE_SHEET_MATCHES_LIST + assumptions: + - datasets: ["d_1,...,d_n"] + - tool: + in: {i: "collection"} + out: {o: dataset} + - collections: + C: [sample_sheet, {i1: d_1, ..., in: d_n}] + then: + type: reduction + invocation: + inputs: + i: {type: collection, ref: C} + produces: + o: {type: dataset} + tests: + workflow_runtime: + framework_test: "collection_semantics_cat_collection_1" + workflow_editor: "accepts sample_sheet -> list connection (canMatch)" + +- doc: | + Sub-collection mapping works the same as for ``list`` composites. A + ``sample_sheet:paired`` can be mapped over a ``paired`` collection input, + extracting each inner pair and producing a ``sample_sheet`` implicit output. + +- example: + label: SAMPLE_SHEET_PAIRED_MAPPING_OVER_PAIRED + assumptions: + - datasets: [d_f, d_r] + - tool: + in: {i: "collection"} + out: {o: dataset} + - collections: + C: ["sample_sheet:paired", {el1: {forward: d_f, reverse: d_r}}] + "C\\_PAIRED": [paired, {forward: d_f, reverse: d_r}] + then: + type: map_over + invocation: + inputs: + i: {type: map_over, collection: C, sub_collection_type: paired} + produces: + o: + type: collection + collection_type: sample_sheet + elements: + el1: + type: tool_output_ref + invocation: {inputs: {i: {type: collection, ref: "C\\_PAIRED"}}} + output: o + tests: + workflow_runtime: + framework_test: "collection_semantics_paired_or_unpaired_5" + workflow_editor: "accepts sample_sheet:paired -> paired connection (maps over like list:paired)" + +- example: + label: SAMPLE_SHEET_PAIRED_MATCHES_LIST_PAIRED + assumptions: + - datasets: [d_f, d_r] + - tool: + in: {i: "collection"} + out: {o: dataset} + - collections: + C: ["sample_sheet:paired", {el1: {forward: d_f, reverse: d_r}}] + then: + type: reduction + invocation: + inputs: + i: {type: collection, ref: C} + produces: + o: {type: dataset} + tests: + workflow_runtime: + framework_test: "collection_semantics_list_paired_0" + workflow_editor: "accepts sample_sheet:paired -> list:paired connection (canMatch)" + +- doc: | + The ``paired_or_unpaired`` integration rules carry over from ``list`` to + ``sample_sheet``. A flat ``sample_sheet`` can be mapped over a + ``paired_or_unpaired`` input via ``single_datasets`` sub-collection mapping, + and ``sample_sheet:paired`` can be mapped over ``paired_or_unpaired`` just as + ``list:paired`` can. + +- example: + label: SAMPLE_SHEET_MAPPING_OVER_PAIRED_OR_UNPAIRED + assumptions: + - datasets: ["d_1,...,d_n"] + - tool: + in: {i: "collection"} + out: {o: dataset} + - collections: + C: [sample_sheet, {i1: d_1, ..., in: d_n}] + - "C_AS_UNPAIRED_i = CollectionInstance for i from 1...n" + then: + type: map_over + invocation: + inputs: + i: {type: map_over, collection: C, sub_collection_type: single_datasets} + produces: + o: + type: collection + collection_type: sample_sheet + elements: + i1: + type: tool_output_ref + invocation: {inputs: {i: {type: collection, ref: C_AS_UNPAIRED_1}}} + output: o + "...": {type: ellipsis} + in: + type: tool_output_ref + invocation: {inputs: {i: {type: collection, ref: C_AS_UNPAIRED_n}}} + output: o + tests: + tool_runtime: + api_test: "test_tool_execute.py::test_map_over_paired_or_unpaired_with_sample_sheet" + workflow_runtime: + framework_test: "collection_semantics_paired_or_unpaired_8" + workflow_editor: "accepts sample_sheet -> paired_or_unpaired connection" + +- example: + label: SAMPLE_SHEET_PAIRED_MAPPING_OVER_PAIRED_OR_UNPAIRED + assumptions: + - datasets: [d_f, d_r] + - tool: + in: {i: "collection"} + out: {o: dataset} + - collections: + C: ["sample_sheet:paired", {el1: {forward: d_f, reverse: d_r}}] + C_AS_MIXED: ["sample_sheet:paired_or_unpaired", {el1: {forward: d_f, reverse: d_r}}] + then: + type: equivalence + left: + inputs: + i: {type: map_over, collection: C} + right: + inputs: + i: {type: map_over, collection: C_AS_MIXED} + tests: + workflow_runtime: + framework_test: "collection_semantics_paired_or_unpaired_6" + workflow_editor: "accepts sample_sheet:paired -> paired_or_unpaired connection" + +- example: + label: SAMPLE_SHEET_PAIRED_OR_UNPAIRED_MATCHES_LIST_PAIRED_OR_UNPAIRED + assumptions: + - datasets: [d_f, d_r] + - tool: + in: {i: "collection"} + out: {o: dataset} + - collections: + C: ["sample_sheet:paired_or_unpaired", {el1: {forward: d_f, reverse: d_r}}] + then: + type: reduction + invocation: + inputs: + i: {type: collection, ref: C} + produces: + o: {type: dataset} + tests: + workflow_runtime: + framework_test: "collection_semantics_list_paired_or_unpaired_1" + workflow_editor: "accepts sample_sheet:paired_or_unpaired -> list:paired_or_unpaired connection (canMatch)" + +- doc: | + The type matching is asymmetric: a ``sample_sheet`` output can satisfy a ``list`` + input because sample sheets carry all the structural information lists have (plus + metadata). However, a ``list`` output cannot satisfy a ``sample_sheet`` input because + lists lack the ``column_definitions`` and per-element ``columns`` metadata that + sample sheet consumers expect. + +- example: + label: LIST_NOT_MATCHES_SAMPLE_SHEET + assumptions: + - datasets: ["d_1,...,d_n"] + - tool: + in: {i: "collection"} + out: {o: dataset} + - collections: + C: [list, {i1: d_1, ..., in: d_n}] + then: + type: invalid + invocation: + inputs: + i: {type: collection, ref: C} + is_valid: false + tests: + workflow_editor: "rejects list -> sample_sheet connection (asymmetry)" + +- example: + label: LIST_PAIRED_NOT_MATCHES_SAMPLE_SHEET_PAIRED + assumptions: + - datasets: [d_f, d_r] + - tool: + in: {i: "collection"} + out: {o: dataset} + - collections: + C: ["list:paired", {el1: {forward: d_f, reverse: d_r}}] + then: + type: invalid + invocation: + inputs: + i: {type: collection, ref: C} + is_valid: false + tests: + workflow_editor: "rejects list:paired -> sample_sheet:paired connection (asymmetry)" + +- example: + label: SAMPLE_SHEET_MATCHES_SAMPLE_SHEET + assumptions: + - datasets: ["d_1,...,d_n"] + - tool: + in: {i: "collection"} + out: {o: dataset} + - collections: + C: [sample_sheet, {i1: d_1, ..., in: d_n}] + then: + type: reduction + invocation: + inputs: + i: {type: collection, ref: C} + produces: + o: {type: dataset} + tests: + workflow_runtime: + framework_test: "collection_semantics_cat_sample_sheet_0" + workflow_editor: "accepts sample_sheet -> sample_sheet connection" diff --git a/lib/galaxy/model/dataset_collections/types/semantics.py b/lib/galaxy/model/dataset_collections/types/semantics.py index 8005ccbc169..5e42b02894e 100644 --- a/lib/galaxy/model/dataset_collections/types/semantics.py +++ b/lib/galaxy/model/dataset_collections/types/semantics.py @@ -3,11 +3,15 @@ # how to use this function... # PYTHONPATH=lib python lib/galaxy/model/dataset_collections/types/semantics.py import argparse +import ast import os +import re import sys from io import StringIO from typing import ( + Annotated, Any, + Literal, NamedTuple, Optional, Union, @@ -16,6 +20,7 @@ from typing import ( import yaml from pydantic import ( BaseModel, + ConfigDict, Field, RootModel, ) @@ -42,8 +47,16 @@ class ToolRuntimeFramework(BaseModel): ToolRuntimeTest = Union[ToolRuntimeApi, ToolRuntimeFramework] +class WorkflowRuntimeTest(BaseModel): + api_test: Optional[str] = None + framework_test: Optional[str] = None + + class ExampleTests(BaseModel): + model_config = ConfigDict(extra="forbid") + tool_runtime: Optional[ToolRuntimeTest] = None + workflow_runtime: Optional[WorkflowRuntimeTest] = None workflow_editor: Optional[str] = None @@ -59,8 +72,8 @@ class ToolDefinition(BaseModel): outputs: dict[str, str] = Field(alias="out") def as_latex(self) -> str: - inputs = ", ".join([f"{k}: \\text{{ {v} }}" for (k, v) in self.inputs.items()]) - outputs = ", ".join([f"{k}: \\text{{ {v} }}" for (k, v) in self.outputs.items()]) + inputs = ", ".join([f"{k}: \\text{{ {_escape_underscores(v)} }}" for (k, v) in self.inputs.items()]) + outputs = ", ".join([f"{k}: \\text{{ {_escape_underscores(v)} }}" for (k, v) in self.outputs.items()]) return f"tool \\text{{ is }} ({inputs}) \\Rightarrow \\{{ {outputs} \\}}" @@ -71,14 +84,14 @@ class ToolDeclaration(BaseModel): return self.tool.as_latex() -def elements_to_latex(elements: dict[str, Any]): +def _assumption_elements_to_latex(elements: dict[str, Any]): elements_as_strings = [] for identifier, value in elements.items(): if value is None: elements_as_strings.append(identifier) else: if isinstance(value, dict): - value_str = elements_to_latex(value) + value_str = _assumption_elements_to_latex(value) else: value_str = str(value) elements_as_strings.append(f"\\text{{ {identifier} }}={value_str}") @@ -91,8 +104,8 @@ class CollectionDefinition(NamedTuple): elements: dict[str, Any] def as_latex(self) -> str: - collection_type = self.collection_type.replace("_", "\\_") - return f"\\text{{CollectionInstance<}}{collection_type},{elements_to_latex(self.elements)}\\text{{>}}" + ct = _latex_collection_type(self.collection_type) + return f"\\text{{CollectionInstance<}}{ct},{_assumption_elements_to_latex(self.elements)}\\text{{>}}" class CollectionDeclarations(BaseModel): @@ -102,10 +115,192 @@ class CollectionDeclarations(BaseModel): Expression = Union[str, DatasetsDeclaration, ToolDeclaration, CollectionDeclarations] +# --- Structured Then Expression Models --- + + +def _escape_underscores(text: str) -> str: + return text.replace("_", "\\_") + + +def _latex_type_word(word: str) -> str: + if "_" in word: + escaped = word.replace("_", "\\_") + return "\\text{" + escaped + "}" + return "\\text{" + word + "}" + + +def _latex_collection_type(ct: str) -> str: + return ":".join(_latex_type_word(p) for p in ct.split(":")) + + +def _output_elements_to_latex(elements: dict) -> str: + parts = [] + for key, value in elements.items(): + if key == "...": + parts.append("...") + else: + parts.append(f"\\text{{{key}}}={value.as_latex()}") + # Join without spaces around ellipsis entries + result = "" + for i, part in enumerate(parts): + if i > 0: + if part == "..." or parts[i - 1] == "...": + result += "," + else: + result += ", " + result += part + return "\\left\\{" + result + "\\right\\}" + + +def _produces_to_latex(produces: dict) -> str: + parts = [] + for key, value in produces.items(): + parts.append(f"{key}: {value.as_latex()}") + return "\\left\\{" + ", ".join(parts) + "\\right\\}" + + +class DatasetInput(BaseModel): + type: Literal["dataset"] + ref: str + + def as_latex(self) -> str: + return self.ref + + +class MapOverInput(BaseModel): + type: Literal["map_over"] + collection: str + sub_collection_type: Optional[str] = None + + def as_latex(self) -> str: + if self.sub_collection_type: + sct = _latex_collection_type(self.sub_collection_type) + return f"\\text{{mapOver}}({self.collection}, '{sct}')" + return f"\\text{{mapOver}}({self.collection})" + + +class CollectionInput(BaseModel): + type: Literal["collection"] + ref: str + + def as_latex(self) -> str: + return self.ref + + +class DatasetListInput(BaseModel): + type: Literal["dataset_list"] + refs: list[str] + + def as_latex(self) -> str: + return "[" + ",".join(self.refs) + "]" + + +InputBinding = Annotated[ + Union[DatasetInput, MapOverInput, CollectionInput, DatasetListInput], Field(discriminator="type") +] + + +class ToolInvocation(BaseModel): + inputs: dict[str, InputBinding] + + def as_latex(self) -> str: + args = ", ".join(f"{k}={v.as_latex()}" for k, v in self.inputs.items()) + return f"tool({args})" + + +class DatasetOutput(BaseModel): + type: Literal["dataset"] + + def as_latex(self) -> str: + return "\\text{dataset}" + + +class EllipsisMarker(BaseModel): + type: Literal["ellipsis"] + + def as_latex(self) -> str: + return "..." + + +class ToolOutputRef(BaseModel): + type: Literal["tool_output_ref"] + invocation: ToolInvocation + output: str + + def as_latex(self) -> str: + return f"{self.invocation.as_latex()}[{self.output}]" + + +class NestedElements(BaseModel): + type: Literal["nested_elements"] + elements: dict[str, "OutputBinding"] + + def as_latex(self) -> str: + return _output_elements_to_latex(self.elements) + + +OutputBinding = Annotated[ + Union[DatasetOutput, EllipsisMarker, ToolOutputRef, NestedElements], Field(discriminator="type") +] +NestedElements.model_rebuild() + + +class CollectionOutput(BaseModel): + type: Literal["collection"] + collection_type: str + elements: dict[str, OutputBinding] + + def as_latex(self) -> str: + ct = _latex_collection_type(self.collection_type) + el = _output_elements_to_latex(self.elements) + return f"\\text{{collection}}<{ct},{el}>" + + +OutputSpec = Annotated[Union[DatasetOutput, CollectionOutput], Field(discriminator="type")] + + +class MapOverThen(BaseModel): + type: Literal["map_over"] + invocation: ToolInvocation + produces: dict[str, OutputSpec] + + def as_latex(self) -> str: + return f"{self.invocation.as_latex()} \\mapsto {_produces_to_latex(self.produces)}" + + +class ReductionThen(BaseModel): + type: Literal["reduction"] + invocation: ToolInvocation + produces: dict[str, OutputSpec] + + def as_latex(self) -> str: + return f"{self.invocation.as_latex()} \\rightarrow {_produces_to_latex(self.produces)}" + + +class EquivalenceThen(BaseModel): + type: Literal["equivalence"] + left: ToolInvocation + right: ToolInvocation + + def as_latex(self) -> str: + return f"{self.left.as_latex()} == {self.right.as_latex()}" + + +class InvalidThen(BaseModel): + type: Literal["invalid"] + invocation: ToolInvocation + + def as_latex(self) -> str: + return self.invocation.as_latex() + + +ThenExpression = Annotated[Union[MapOverThen, ReductionThen, EquivalenceThen, InvalidThen], Field(discriminator="type")] + + class Example(BaseModel): label: str assumptions: Optional[list[Expression]] = None - then: Optional[str] = None + then: Optional[ThenExpression] = None is_valid: bool = True tests: Optional[ExampleTests] = None @@ -117,7 +312,11 @@ class ExampleEntry(BaseModel): YAMLRootModel = RootModel[list[Union[DocEntry, ExampleEntry]]] -WORDS_TO_TEXTIFY = ["list", "forward", "reverse", "mapOver"] +WORDS_TO_TEXTIFY = ["list", "forward", "reverse", "mapOver", "collection", "dataset", "inner"] +_POU_PLACEHOLDER = "\x00POU\x00" +_UNPAIRED_PLACEHOLDER = "\x00UNPAIRED\x00" +_SINGLE_DATASETS_PLACEHOLDER = "\x00SINGLEDATASETS\x00" +_SAMPLE_SHEET_PLACEHOLDER = "\x00SAMPLESHEET\x00" def expression_to_latex(expression: str, wrap: bool = True): @@ -125,9 +324,19 @@ def expression_to_latex(expression: str, wrap: bool = True): expression = expression.replace("~>", "\\mapsto") expression = expression.replace("{", "\\left\\{") expression = expression.replace("}", "\\right\\}") - expression = expression.replace("paired_or_unpaired", "paired\\_or\\_unpaired") + # Replace multi-word identifiers with placeholders before WORDS_TO_TEXTIFY + # to prevent double-replacement (e.g. "dataset" inside "single_datasets") + expression = expression.replace("single_datasets", _SINGLE_DATASETS_PLACEHOLDER) + expression = expression.replace("sample_sheet", _SAMPLE_SHEET_PLACEHOLDER) + expression = expression.replace("paired_or_unpaired", _POU_PLACEHOLDER) + expression = expression.replace("unpaired", _UNPAIRED_PLACEHOLDER) + expression = expression.replace("paired", "\\text{paired}") + expression = expression.replace(_UNPAIRED_PLACEHOLDER, "\\text{unpaired}") + expression = expression.replace(_POU_PLACEHOLDER, "\\text{paired\\_or\\_unpaired}") for word in WORDS_TO_TEXTIFY: expression = expression.replace(word, "\\text{" + word + "}") + expression = expression.replace(_SINGLE_DATASETS_PLACEHOLDER, "\\text{single\\_datasets}") + expression = expression.replace(_SAMPLE_SHEET_PLACEHOLDER, "\\text{sample\\_sheet}") if wrap: return f"$ {expression} $" else: @@ -165,9 +374,129 @@ def main(argv=None) -> None: generate_docs() -def check(): - # todo - pass +def _load_examples() -> list["Example"]: + semantics_yaml = yaml.safe_load( + resource_string("galaxy.model.dataset_collections.types", "collection_semantics.yml") + ) + root = YAMLRootModel.model_validate(semantics_yaml) + return [e.example for e in root.root if isinstance(e, ExampleEntry)] + + +def check() -> list[str]: + examples = _load_examples() + errors: list[str] = [] + errors.extend(validate_api_test_refs(examples)) + errors.extend(validate_tool_refs(examples)) + errors.extend(validate_workflow_editor_refs(examples)) + return errors + + +def _validate_api_test_ref(label: str, ref: str, api_test_dir: str) -> Optional[str]: + parts = ref.split("::") + filename = parts[0] + filepath = os.path.join(api_test_dir, filename) + if not os.path.exists(filepath): + return f"[{label}] api_test file not found: {filename}" + with open(filepath) as f: + tree = ast.parse(f.read(), filename=filepath) + if len(parts) == 2: + func_name = parts[1] + found = any( + isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) and node.name == func_name + for node in ast.walk(tree) + ) + if not found: + return f"[{label}] api_test function not found: {ref}" + elif len(parts) == 3: + class_name, method_name = parts[1], parts[2] + found = False + for node in ast.walk(tree): + if isinstance(node, ast.ClassDef) and node.name == class_name: + found = any( + isinstance(child, (ast.FunctionDef, ast.AsyncFunctionDef)) and child.name == method_name + for child in node.body + ) + break + if not found: + return f"[{label}] api_test method not found: {ref}" + return None + + +def _validate_framework_test_ref(label: str, ref: str, workflow_dir: str) -> Optional[str]: + """Validate a framework test ref like 'collection_semantics_cat_0'.""" + parts = ref.rsplit("_", 1) + if len(parts) != 2 or not parts[1].isdigit(): + return f"[{label}] framework_test must be 'workflow_name_index': {ref}" + workflow_name = parts[0] + workflow_file = os.path.join(workflow_dir, f"{workflow_name}.gxwf.yml") + if not os.path.exists(workflow_file): + return f"[{label}] framework_test workflow not found: {workflow_name}.gxwf.yml" + test_file = os.path.join(workflow_dir, f"{workflow_name}.gxwf-tests.yml") + if not os.path.exists(test_file): + return f"[{label}] framework_test test file not found: {workflow_name}.gxwf-tests.yml" + return None + + +def validate_api_test_refs(examples: list["Example"]) -> list[str]: + errors: list[str] = [] + api_test_dir = os.path.join(galaxy_directory(), "lib", "galaxy_test", "api") + workflow_dir = os.path.join(galaxy_directory(), "lib", "galaxy_test", "workflow") + for ex in examples: + if ex.tests and ex.tests.tool_runtime and isinstance(ex.tests.tool_runtime, ToolRuntimeApi): + error = _validate_api_test_ref(ex.label, ex.tests.tool_runtime.api_test, api_test_dir) + if error: + errors.append(error) + if ex.tests and ex.tests.workflow_runtime: + if ex.tests.workflow_runtime.api_test: + error = _validate_api_test_ref(ex.label, ex.tests.workflow_runtime.api_test, api_test_dir) + if error: + errors.append(error) + if ex.tests.workflow_runtime.framework_test: + error = _validate_framework_test_ref(ex.label, ex.tests.workflow_runtime.framework_test, workflow_dir) + if error: + errors.append(error) + return errors + + +def validate_tool_refs(examples: list["Example"]) -> list[str]: + errors: list[str] = [] + tools_dir = os.path.join(galaxy_directory(), "test", "functional", "tools") + for ex in examples: + if not ex.tests or not ex.tests.tool_runtime: + continue + if not isinstance(ex.tests.tool_runtime, ToolRuntimeFramework): + continue + tool_id = ex.tests.tool_runtime.tool + tool_path = os.path.join(tools_dir, f"{tool_id}.xml") + if not os.path.exists(tool_path): + errors.append(f"[{ex.label}] framework tool XML not found: {tool_id}.xml") + return errors + + +def validate_workflow_editor_refs(examples: list["Example"]) -> list[str]: + errors: list[str] = [] + test_file = os.path.join( + galaxy_directory(), + "client", + "src", + "components", + "Workflow", + "Editor", + "modules", + "terminals.test.ts", + ) + if not os.path.exists(test_file): + return [f"workflow editor test file not found: {test_file}"] + with open(test_file) as f: + content = f.read() + it_descriptions = set(re.findall(r'it\(["\'](.+?)["\']\s*,', content)) + for ex in examples: + if not ex.tests or not ex.tests.workflow_editor: + continue + desc = ex.tests.workflow_editor + if desc not in it_descriptions: + errors.append(f'[{ex.label}] workflow_editor test not found: "{desc}"') + return errors def generate_docs(): @@ -215,9 +544,8 @@ def generate_docs(): validity_str = "" if not example.is_valid: validity_str = "\\text{ is invalid}" - markdown_content.write( - f"\n\nthen\n\n$${expression_to_latex(example.then, wrap=False)}{validity_str}$$\n\n" - ) + then_latex = example.then.as_latex() + markdown_content.write(f"\n\nthen\n\n$${then_latex}{validity_str}$$\n\n") markdown_content.write(":::\n\n") markdown_content.write("\n\n

\n\n") markdown_content.write("\n\n") diff --git a/lib/galaxy/tool_util/cwl/util.py b/lib/galaxy/tool_util/cwl/util.py index 2a455085aa0..fde6669af57 100644 --- a/lib/galaxy/tool_util/cwl/util.py +++ b/lib/galaxy/tool_util/cwl/util.py @@ -365,7 +365,7 @@ def galactic_job_json( elements = to_elements(value, collection_type) kwds = {} if collection_type.startswith("sample_sheet"): - kwds["rows"] = value["rows"] + kwds["rows"] = value.get("rows") if "name" in value: kwds["name"] = value["name"] collection = collection_create_func(elements, collection_type, **kwds) diff --git a/lib/galaxy_test/api/test_tool_execute.py b/lib/galaxy_test/api/test_tool_execute.py index bb3c7bb5c11..0a877ce9dce 100644 --- a/lib/galaxy_test/api/test_tool_execute.py +++ b/lib/galaxy_test/api/test_tool_execute.py @@ -571,6 +571,31 @@ def test_map_over_paired_or_unpaired_with_list_of_lists(target_history: TargetHi assert len(as_dict_0["object"]["elements"]) == 3 +@requires_tool_id("collection_list_paired_or_unpaired") +def test_map_over_list_paired_or_unpaired_with_list_of_lists( + target_history: TargetHistory, required_tool: RequiredTool +): + hdca = target_history.with_example_list_of_lists() + execute = required_tool.execute().with_inputs( + {"f1": {"batch": True, "values": [{"map_over_type": "list", **hdca.src_dict}]}} + ) + execute.assert_has_n_jobs(1).assert_creates_n_implicit_collections(1) + output_collection = execute.assert_creates_implicit_collection(0) + assert output_collection.details["collection_type"] == "list" + + +@requires_tool_id("collection_paired_test") +def test_paired_or_unpaired_not_consumed_by_paired_when_mapping( + target_history: TargetHistory, required_tool: RequiredTool +): + """PAIRED_OR_UNPAIRED_NOT_CONSUMED_BY_PAIRED_WHEN_MAPPING: + list:paired_or_unpaired cannot be mapped over a paired tool input.""" + hdca = target_history.with_list_of_paired_and_unpaired() + required_tool.execute().with_inputs( + {"f1": {"batch": True, "values": [{"map_over_type": "paired", **hdca.src_dict}]}} + ).assert_fails.with_error_containing("Cannot split collection") + + @requires_tool_id("collection_paired_test") def test_simple_subcollection_mapping( target_history: TargetHistory, required_tool: RequiredTool, tool_input_format: DescribeToolInputs diff --git a/lib/galaxy_test/base/populators.py b/lib/galaxy_test/base/populators.py index 6e5776f11ae..ebd5b0d1a87 100644 --- a/lib/galaxy_test/base/populators.py +++ b/lib/galaxy_test/base/populators.py @@ -4062,6 +4062,10 @@ def load_data_dict( fetch_response = dataset_collection_populator.create_list_of_pairs_in_history( history_id, contents=elements, wait=True, **new_collection_kwds ).json() + elif collection_type == "list:paired_or_unpaired": + fetch_response = dataset_collection_populator.create_list_of_paired_and_unpaired_in_history( + history_id, wait=True, **new_collection_kwds + ).json() elif collection_type and ":" in collection_type: fetch_response = { "outputs": [ @@ -4074,6 +4078,15 @@ def load_data_dict( fetch_response = dataset_collection_populator.create_list_in_history( history_id, contents=elements, direct_upload=True, wait=True, **new_collection_kwds ).json() + elif collection_type == "paired_or_unpaired": + if elements: + fetch_response = dataset_collection_populator.upload_collection( + history_id, "paired_or_unpaired", elements=elements, wait=True, **new_collection_kwds + ).json() + else: + fetch_response = dataset_collection_populator.create_paired_or_unpaired_pair_in_history( + history_id, wait=True, **new_collection_kwds + ).json() else: fetch_response = dataset_collection_populator.create_pair_in_history( history_id, contents=elements or None, direct_upload=True, wait=True, **new_collection_kwds diff --git a/lib/galaxy_test/workflow/collection_semantics_cat.gxwf-tests.yml b/lib/galaxy_test/workflow/collection_semantics_cat.gxwf-tests.yml new file mode 100644 index 00000000000..8b682a6ab98 --- /dev/null +++ b/lib/galaxy_test/workflow/collection_semantics_cat.gxwf-tests.yml @@ -0,0 +1,141 @@ +- doc: | + BASIC_MAPPING_PAIRED: A paired collection mapped over a data input produces + a paired output collection. + job: + input1: + class: Collection + collection_type: paired + elements: + - identifier: forward + class: File + contents: "forward content" + - identifier: reverse + class: File + contents: "reverse content" + outputs: + wf_output: + class: Collection + collection_type: paired + elements: + forward: + asserts: + - that: has_text + text: "forward content" + reverse: + asserts: + - that: has_text + text: "reverse content" + +- doc: | + BASIC_MAPPING_PAIRED_OR_UNPAIRED_PAIRED: A paired_or_unpaired collection (paired variant) + mapped over a data input produces a paired_or_unpaired output collection with 2 elements. + job: + input1: + class: Collection + collection_type: paired_or_unpaired + elements: + - identifier: forward + class: File + contents: "forward content" + - identifier: reverse + class: File + contents: "reverse content" + outputs: + wf_output: + class: Collection + collection_type: paired_or_unpaired + elements: + forward: + asserts: + - that: has_text + text: "forward content" + reverse: + asserts: + - that: has_text + text: "reverse content" + +- doc: | + BASIC_MAPPING_PAIRED_OR_UNPAIRED_UNPAIRED: A paired_or_unpaired collection (unpaired variant) + mapped over a data input produces a paired_or_unpaired output collection with 1 element. + job: + input1: + class: Collection + collection_type: paired_or_unpaired + elements: + - identifier: unpaired + class: File + contents: "unpaired content" + outputs: + wf_output: + class: Collection + collection_type: paired_or_unpaired + elements: + unpaired: + asserts: + - that: has_text + text: "unpaired content" + +- doc: | + BASIC_MAPPING_LIST_PAIRED_OR_UNPAIRED: A list:paired_or_unpaired collection mapped over + a data input produces a list:paired_or_unpaired output collection. + job: + input1: + class: Collection + collection_type: list:paired_or_unpaired + elements: + - identifier: el1 + class: Collection + type: paired_or_unpaired + elements: + - identifier: forward + class: File + contents: "forward content" + - identifier: reverse + class: File + contents: "reverse content" + outputs: + wf_output: + class: Collection + collection_type: list:paired_or_unpaired + elements: + el1: + elements: + forward: + asserts: + - that: has_text + text: "forward content" + reverse: + asserts: + - that: has_text + text: "reverse content" + +- doc: | + SAMPLE_SHEET_MAPPING: A sample_sheet mapped over a data input produces + a sample_sheet output collection. + job: + input1: + class: Collection + collection_type: sample_sheet + rows: + el1: [] + el2: [] + elements: + - identifier: el1 + class: File + contents: "element 1" + - identifier: el2 + class: File + contents: "element 2" + outputs: + wf_output: + class: Collection + collection_type: sample_sheet + elements: + el1: + asserts: + - that: has_text + text: "element 1" + el2: + asserts: + - that: has_text + text: "element 2" diff --git a/lib/galaxy_test/workflow/collection_semantics_cat.gxwf.yml b/lib/galaxy_test/workflow/collection_semantics_cat.gxwf.yml new file mode 100644 index 00000000000..b76cf93b744 --- /dev/null +++ b/lib/galaxy_test/workflow/collection_semantics_cat.gxwf.yml @@ -0,0 +1,11 @@ +class: GalaxyWorkflow +inputs: + input1: data +outputs: + wf_output: + outputSource: cat_step/out_file1 +steps: + cat_step: + tool_id: cat + in: + input1: input1 diff --git a/lib/galaxy_test/workflow/collection_semantics_cat_collection.gxwf-tests.yml b/lib/galaxy_test/workflow/collection_semantics_cat_collection.gxwf-tests.yml new file mode 100644 index 00000000000..36b84a84ceb --- /dev/null +++ b/lib/galaxy_test/workflow/collection_semantics_cat_collection.gxwf-tests.yml @@ -0,0 +1,46 @@ +- doc: | + COLLECTION_INPUT_LIST: A list collection consumed by a tool expecting a list + collection input produces a single dataset. + job: + input1: + class: Collection + collection_type: list + elements: + - identifier: el1 + class: File + contents: "element 1" + - identifier: el2 + class: File + contents: "element 2" + outputs: + wf_output: + asserts: + - that: has_text + text: "element 1" + - that: has_text + text: "element 2" + +- doc: | + SAMPLE_SHEET_MATCHES_LIST: A sample_sheet collection consumed by a list + collection input via canMatch (reduction). + job: + input1: + class: Collection + collection_type: sample_sheet + rows: + el1: [] + el2: [] + elements: + - identifier: el1 + class: File + contents: "element 1" + - identifier: el2 + class: File + contents: "element 2" + outputs: + wf_output: + asserts: + - that: has_text + text: "element 1" + - that: has_text + text: "element 2" diff --git a/lib/galaxy_test/workflow/collection_semantics_cat_collection.gxwf.yml b/lib/galaxy_test/workflow/collection_semantics_cat_collection.gxwf.yml new file mode 100644 index 00000000000..25ddcacdc7e --- /dev/null +++ b/lib/galaxy_test/workflow/collection_semantics_cat_collection.gxwf.yml @@ -0,0 +1,13 @@ +class: GalaxyWorkflow +inputs: + input1: + type: collection + collection_type: list +outputs: + wf_output: + outputSource: tool_step/out_file1 +steps: + tool_step: + tool_id: cat_collection + in: + input1: input1 diff --git a/lib/galaxy_test/workflow/collection_semantics_cat_sample_sheet.gxwf-tests.yml b/lib/galaxy_test/workflow/collection_semantics_cat_sample_sheet.gxwf-tests.yml new file mode 100644 index 00000000000..dc70eb23d0d --- /dev/null +++ b/lib/galaxy_test/workflow/collection_semantics_cat_sample_sheet.gxwf-tests.yml @@ -0,0 +1,24 @@ +- doc: | + SAMPLE_SHEET_MATCHES_SAMPLE_SHEET: A sample_sheet consumed by a sample_sheet + collection input (identity match, reduction). + job: + input1: + class: Collection + collection_type: sample_sheet + rows: + el1: [] + el2: [] + elements: + - identifier: el1 + class: File + contents: "element 1" + - identifier: el2 + class: File + contents: "element 2" + outputs: + wf_output: + asserts: + - that: has_text + text: "el1" + - that: has_text + text: "el2" diff --git a/lib/galaxy_test/workflow/collection_semantics_cat_sample_sheet.gxwf.yml b/lib/galaxy_test/workflow/collection_semantics_cat_sample_sheet.gxwf.yml new file mode 100644 index 00000000000..033361052f2 --- /dev/null +++ b/lib/galaxy_test/workflow/collection_semantics_cat_sample_sheet.gxwf.yml @@ -0,0 +1,13 @@ +class: GalaxyWorkflow +inputs: + input1: + type: collection + collection_type: sample_sheet +outputs: + wf_output: + outputSource: tool_step/output +steps: + tool_step: + tool_id: __SAMPLE_SHEET_TO_TABULAR__ + in: + input: input1 diff --git a/lib/galaxy_test/workflow/collection_semantics_cat_two_inputs.gxwf-tests.yml b/lib/galaxy_test/workflow/collection_semantics_cat_two_inputs.gxwf-tests.yml new file mode 100644 index 00000000000..b3d1a0f3cdb --- /dev/null +++ b/lib/galaxy_test/workflow/collection_semantics_cat_two_inputs.gxwf-tests.yml @@ -0,0 +1,76 @@ +- doc: | + BASIC_MAPPING_INCLUDING_SINGLE_DATASET: Mapping a list collection over a data input + while another input is a single dataset produces a list output collection. + job: + input1: + class: Collection + collection_type: list + elements: + - identifier: el1 + class: File + contents: "list element 1" + - identifier: el2 + class: File + contents: "list element 2" + input2: + class: File + contents: "single dataset content" + outputs: + wf_output: + class: Collection + collection_type: list + elements: + el1: + asserts: + - that: has_text + text: "list element 1" + - that: has_text + text: "single dataset content" + el2: + asserts: + - that: has_text + text: "list element 2" + - that: has_text + text: "single dataset content" + +- doc: | + BASIC_MAPPING_TWO_INPUTS_WITH_IDENTICAL_STRUCTURE: Mapping two list collections with + identical structure over two data inputs produces a list output collection. + job: + input1: + class: Collection + collection_type: list + elements: + - identifier: el1 + class: File + contents: "collection 1 element 1" + - identifier: el2 + class: File + contents: "collection 1 element 2" + input2: + class: Collection + collection_type: list + elements: + - identifier: el1 + class: File + contents: "collection 2 element 1" + - identifier: el2 + class: File + contents: "collection 2 element 2" + outputs: + wf_output: + class: Collection + collection_type: list + elements: + el1: + asserts: + - that: has_text + text: "collection 1 element 1" + - that: has_text + text: "collection 2 element 1" + el2: + asserts: + - that: has_text + text: "collection 1 element 2" + - that: has_text + text: "collection 2 element 2" diff --git a/lib/galaxy_test/workflow/collection_semantics_cat_two_inputs.gxwf.yml b/lib/galaxy_test/workflow/collection_semantics_cat_two_inputs.gxwf.yml new file mode 100644 index 00000000000..86a90d7aef7 --- /dev/null +++ b/lib/galaxy_test/workflow/collection_semantics_cat_two_inputs.gxwf.yml @@ -0,0 +1,13 @@ +class: GalaxyWorkflow +inputs: + input1: data + input2: data +outputs: + wf_output: + outputSource: cat_step/out_file1 +steps: + cat_step: + tool_id: cat + in: + input1: input1 + queries_0|input2: input2 diff --git a/lib/galaxy_test/workflow/collection_semantics_list_paired.gxwf-tests.yml b/lib/galaxy_test/workflow/collection_semantics_list_paired.gxwf-tests.yml new file mode 100644 index 00000000000..13aca3bc2f7 --- /dev/null +++ b/lib/galaxy_test/workflow/collection_semantics_list_paired.gxwf-tests.yml @@ -0,0 +1,27 @@ +- doc: | + SAMPLE_SHEET_PAIRED_MATCHES_LIST_PAIRED: A sample_sheet:paired consumed by a + list:paired collection input via canMatch (reduction). + job: + input1: + class: Collection + collection_type: sample_sheet:paired + rows: + el1: [] + elements: + - identifier: el1 + class: Collection + type: paired + elements: + - identifier: forward + class: File + contents: "forward content" + - identifier: reverse + class: File + contents: "reverse content" + outputs: + wf_output: + asserts: + - that: has_text + text: "identifier is el1" + - that: has_text + text: "collection_type" diff --git a/lib/galaxy_test/workflow/collection_semantics_list_paired.gxwf.yml b/lib/galaxy_test/workflow/collection_semantics_list_paired.gxwf.yml new file mode 100644 index 00000000000..d1671ce66e2 --- /dev/null +++ b/lib/galaxy_test/workflow/collection_semantics_list_paired.gxwf.yml @@ -0,0 +1,13 @@ +class: GalaxyWorkflow +inputs: + input1: + type: collection + collection_type: list:paired +outputs: + wf_output: + outputSource: tool_step/out1 +steps: + tool_step: + tool_id: collection_paired_or_list_paired_input + in: + f1: input1 diff --git a/lib/galaxy_test/workflow/collection_semantics_list_paired_or_unpaired.gxwf-tests.yml b/lib/galaxy_test/workflow/collection_semantics_list_paired_or_unpaired.gxwf-tests.yml new file mode 100644 index 00000000000..0b8c3d26751 --- /dev/null +++ b/lib/galaxy_test/workflow/collection_semantics_list_paired_or_unpaired.gxwf-tests.yml @@ -0,0 +1,84 @@ +- doc: | + COLLECTION_INPUT_LIST_PAIRED_OR_UNPAIRED: A list:paired_or_unpaired collection + consumed by a tool expecting list:paired_or_unpaired produces a single dataset. + job: + input1: + class: Collection + collection_type: list:paired_or_unpaired + elements: + - identifier: el1 + class: Collection + type: paired_or_unpaired + elements: + - identifier: forward + class: File + contents: "forward content" + - identifier: reverse + class: File + contents: "reverse content" + outputs: + wf_output: + asserts: + - that: has_text + text: "forward content" + - that: has_text + text: "reverse content" + +- doc: | + SAMPLE_SHEET_PAIRED_OR_UNPAIRED_MATCHES_LIST_PAIRED_OR_UNPAIRED: A + sample_sheet:paired_or_unpaired consumed by list:paired_or_unpaired via canMatch. + job: + input1: + class: Collection + collection_type: sample_sheet:paired_or_unpaired + rows: + el1: [] + elements: + - identifier: el1 + class: Collection + type: paired_or_unpaired + elements: + - identifier: forward + class: File + contents: "forward content" + - identifier: reverse + class: File + contents: "reverse content" + outputs: + wf_output: + asserts: + - that: has_text + text: "forward content" + - that: has_text + text: "reverse content" + +- doc: | + MAPPING_LIST_LIST_OVER_LIST_PAIRED_OR_UNPAIRED: A list:list collection mapped + over a list:paired_or_unpaired tool input produces a list output collection. + Each inner list is consumed as list:paired_or_unpaired via single_datasets. + job: + input1: + class: Collection + collection_type: list:list + elements: + - identifier: outer1 + class: Collection + type: list + elements: + - identifier: el1 + class: File + contents: "data from outer1 el1" + - identifier: el2 + class: File + contents: "data from outer1 el2" + - identifier: outer2 + class: Collection + type: list + elements: + - identifier: el1 + class: File + contents: "data from outer2 el1" + outputs: + wf_output: + class: Collection + collection_type: list diff --git a/lib/galaxy_test/workflow/collection_semantics_list_paired_or_unpaired.gxwf.yml b/lib/galaxy_test/workflow/collection_semantics_list_paired_or_unpaired.gxwf.yml new file mode 100644 index 00000000000..cd5937397a8 --- /dev/null +++ b/lib/galaxy_test/workflow/collection_semantics_list_paired_or_unpaired.gxwf.yml @@ -0,0 +1,13 @@ +class: GalaxyWorkflow +inputs: + input1: + type: collection + collection_type: list:paired_or_unpaired +outputs: + wf_output: + outputSource: tool_step/out1 +steps: + tool_step: + tool_id: collection_list_paired_or_unpaired + in: + f1: input1 diff --git a/lib/galaxy_test/workflow/collection_semantics_multi_data_optional.gxwf-tests.yml b/lib/galaxy_test/workflow/collection_semantics_multi_data_optional.gxwf-tests.yml new file mode 100644 index 00000000000..30685576196 --- /dev/null +++ b/lib/galaxy_test/workflow/collection_semantics_multi_data_optional.gxwf-tests.yml @@ -0,0 +1,21 @@ +- doc: | + LIST_REDUCTION: A list collection consumed by a multi-data input (reduction) + produces a single dataset containing all elements. + job: + input1: + class: Collection + collection_type: list + elements: + - identifier: el1 + class: File + contents: "element 1" + - identifier: el2 + class: File + contents: "element 2" + outputs: + wf_output: + asserts: + - that: has_text + text: "element 1" + - that: has_text + text: "element 2" diff --git a/lib/galaxy_test/workflow/collection_semantics_multi_data_optional.gxwf.yml b/lib/galaxy_test/workflow/collection_semantics_multi_data_optional.gxwf.yml new file mode 100644 index 00000000000..3f60c35b20f --- /dev/null +++ b/lib/galaxy_test/workflow/collection_semantics_multi_data_optional.gxwf.yml @@ -0,0 +1,13 @@ +class: GalaxyWorkflow +inputs: + input1: + type: collection + collection_type: list +outputs: + wf_output: + outputSource: tool_step/out1 +steps: + tool_step: + tool_id: multi_data_optional + in: + input1: input1 diff --git a/lib/galaxy_test/workflow/collection_semantics_paired_or_unpaired.gxwf-tests.yml b/lib/galaxy_test/workflow/collection_semantics_paired_or_unpaired.gxwf-tests.yml new file mode 100644 index 00000000000..c7658405026 --- /dev/null +++ b/lib/galaxy_test/workflow/collection_semantics_paired_or_unpaired.gxwf-tests.yml @@ -0,0 +1,207 @@ +- doc: | + COLLECTION_INPUT_PAIRED_OR_UNPAIRED: A paired_or_unpaired collection consumed + directly by a tool expecting paired_or_unpaired produces a single dataset. + job: + input1: + class: Collection + collection_type: paired_or_unpaired + elements: + - identifier: forward + class: File + contents: "forward content" + - identifier: reverse + class: File + contents: "reverse content" + outputs: + wf_output: + asserts: + - that: has_text + text: "forward content" + - that: has_text + text: "reverse content" + +- doc: | + PAIRED_OR_UNPAIRED_CONSUMES_PAIRED: A paired collection can be consumed by a + paired_or_unpaired tool input (subtype compatibility). + job: + input1: + class: Collection + collection_type: paired + elements: + - identifier: forward + class: File + contents: "forward content" + - identifier: reverse + class: File + contents: "reverse content" + outputs: + wf_output: + asserts: + - that: has_text + text: "forward content" + - that: has_text + text: "reverse content" + +- doc: | + MAPPING_LIST_PAIRED_OVER_PAIRED_OR_UNPAIRED: A list:paired collection mapped over + a paired_or_unpaired tool input produces a list output collection (subtype + mapping). + job: + input1: + class: Collection + collection_type: list:paired + elements: + - identifier: test_level_1 + class: Collection + type: paired + elements: + - identifier: forward + class: File + contents: "TestData123" + - identifier: reverse + class: File + contents: "TestData123" + outputs: + wf_output: + class: Collection + collection_type: list + +- doc: | + MAPPING_LIST_LIST_PAIRED_OVER_PAIRED_OR_UNPAIRED: A list:list:paired collection mapped + over a paired_or_unpaired tool input produces a list:list output collection. + job: + input1: + class: Collection + collection_type: list:list:paired + elements: + - identifier: test_level_2 + class: Collection + type: list:paired + elements: + - identifier: test_level_1 + class: Collection + type: paired + elements: + - identifier: forward + class: File + contents: "TestData123" + - identifier: reverse + class: File + contents: "TestData123" + outputs: + wf_output: + class: Collection + collection_type: list:list + +- doc: | + MAPPING_LIST_PAIRED_OR_UNPAIRED_OVER_PAIRED_OR_UNPAIRED: A list:paired_or_unpaired + collection mapped over a paired_or_unpaired tool input produces a list output collection. + job: + input1: + class: Collection + collection_type: list:paired_or_unpaired + elements: + - identifier: test_level_1 + class: Collection + type: paired_or_unpaired + elements: + - identifier: forward + class: File + contents: "TestData123" + - identifier: reverse + class: File + contents: "TestData123" + outputs: + wf_output: + class: Collection + collection_type: list + +- doc: | + SAMPLE_SHEET_PAIRED_MAPPING_OVER_PAIRED: A sample_sheet:paired mapped over + a paired_or_unpaired tool input produces a sample_sheet output. + job: + input1: + class: Collection + collection_type: sample_sheet:paired + rows: + el1: [] + elements: + - identifier: el1 + class: Collection + type: paired + elements: + - identifier: forward + class: File + contents: "forward content" + - identifier: reverse + class: File + contents: "reverse content" + outputs: + wf_output: + class: Collection + collection_type: sample_sheet + +- doc: | + SAMPLE_SHEET_PAIRED_MAPPING_OVER_PAIRED_OR_UNPAIRED: A sample_sheet:paired + mapped over paired_or_unpaired (equiv to sample_sheet:paired_or_unpaired mapping). + job: + input1: + class: Collection + collection_type: sample_sheet:paired + rows: + el1: [] + elements: + - identifier: el1 + class: Collection + type: paired + elements: + - identifier: forward + class: File + contents: "forward content" + - identifier: reverse + class: File + contents: "reverse content" + outputs: + wf_output: + class: Collection + collection_type: sample_sheet + +- doc: | + MAPPING_LIST_OVER_PAIRED_OR_UNPAIRED: A flat list mapped over a + paired_or_unpaired tool input via single_datasets produces a list output. + job: + input1: + class: Collection + collection_type: list + elements: + - identifier: el1 + class: File + contents: "element 1" + - identifier: el2 + class: File + contents: "element 2" + outputs: + wf_output: + class: Collection + collection_type: list + +- doc: | + SAMPLE_SHEET_MAPPING_OVER_PAIRED_OR_UNPAIRED: A flat sample_sheet mapped over + a paired_or_unpaired tool input via single_datasets produces a sample_sheet output. + job: + input1: + class: Collection + collection_type: sample_sheet + rows: + el1: [] + el2: [] + elements: + - identifier: el1 + class: File + contents: "element 1" + - identifier: el2 + class: File + contents: "element 2" + outputs: + wf_output: + class: Collection + collection_type: sample_sheet diff --git a/lib/galaxy_test/workflow/collection_semantics_paired_or_unpaired.gxwf.yml b/lib/galaxy_test/workflow/collection_semantics_paired_or_unpaired.gxwf.yml new file mode 100644 index 00000000000..98252c3723c --- /dev/null +++ b/lib/galaxy_test/workflow/collection_semantics_paired_or_unpaired.gxwf.yml @@ -0,0 +1,13 @@ +class: GalaxyWorkflow +inputs: + input1: + type: collection + collection_type: paired_or_unpaired +outputs: + wf_output: + outputSource: tool_step/out1 +steps: + tool_step: + tool_id: collection_paired_or_unpaired + in: + f1: input1 diff --git a/test/unit/data/dataset_collections/test_type_descriptions.py b/test/unit/data/dataset_collections/test_type_descriptions.py index 8416ed2edb2..0f8d1d5ab15 100644 --- a/test/unit/data/dataset_collections/test_type_descriptions.py +++ b/test/unit/data/dataset_collections/test_type_descriptions.py @@ -86,6 +86,91 @@ def test_sample_sheet_acts_like_list(): assert sample_sheet_paired.effective_collection_type(paired_type) == "sample_sheet" +def test_effective_collection_type_paired_or_unpaired_over_paired(): + # list:paired mapped over paired_or_unpaired should yield "list" + assert c_t("list:paired").effective_collection_type("paired_or_unpaired") == "list" + # list:list:paired mapped over paired_or_unpaired should yield "list:list" + assert c_t("list:list:paired").effective_collection_type("paired_or_unpaired") == "list:list" + # list:paired_or_unpaired mapped over paired_or_unpaired should still yield "list" + assert c_t("list:paired_or_unpaired").effective_collection_type("paired_or_unpaired") == "list" + + +def test_endswith_colon_boundary(): + """Regression: endswith must require colon boundary to avoid partial matches. + + 'list:paired_or_unpaired'.endswith('paired') was True because 'paired' + is a substring at the end of 'paired_or_unpaired'. The fix requires + ':paired' (with colon prefix) so only proper rank boundaries match. + """ + # list:paired_or_unpaired does NOT have subcollections of type paired + # PAIRED_OR_UNPAIRED_NOT_CONSUMED_BY_PAIRED_WHEN_MAPPING + assert not c_t("list:paired_or_unpaired").has_subcollections_of_type("paired") + + # But list:paired DOES have subcollections of type paired (proper boundary) + assert c_t("list:paired").has_subcollections_of_type("paired") + + # And list:list:paired_or_unpaired does NOT have subcollections of type paired + assert not c_t("list:list:paired_or_unpaired").has_subcollections_of_type("paired") + + # Existing cases still work + assert c_t("list:list:paired").has_subcollections_of_type("paired") + assert c_t("list:list:paired").has_subcollections_of_type("list:paired") + assert not c_t("list:list:paired").has_subcollections_of_type("list") + + +def test_compound_paired_or_unpaired_has_subcollections(): + """Test compound :paired_or_unpaired suffix in has_subcollections_of_type. + + Covers collection_semantics.yml examples: + - MAPPING_LIST_LIST_OVER_LIST_PAIRED_OR_UNPAIRED + - MAPPING_LIST_LIST_PAIRED_OVER_PAIRED_OR_UNPAIRED (via compound path) + """ + # list:list can map over list:paired_or_unpaired + assert c_t("list:list").has_subcollections_of_type("list:paired_or_unpaired") + + # list:list:paired can map over list:paired_or_unpaired + # (paired consumed by paired_or_unpaired, higher ranks list == list) + assert c_t("list:list:paired").has_subcollections_of_type("list:paired_or_unpaired") + + # list:list:list can map over list:paired_or_unpaired + assert c_t("list:list:list").has_subcollections_of_type("list:paired_or_unpaired") + + # list:paired cannot map over list:paired_or_unpaired — same rank, + # after stripping :paired the higher ranks are equal (no remainder) + assert not c_t("list:paired").has_subcollections_of_type("list:paired_or_unpaired") + + # list cannot map over list:paired_or_unpaired — lower rank + assert not c_t("list").has_subcollections_of_type("list:paired_or_unpaired") + + # paired cannot map over list:paired_or_unpaired — higher ranks don't align + assert not c_t("paired").has_subcollections_of_type("list:paired_or_unpaired") + + # paired:paired cannot map over list:paired_or_unpaired — higher ranks + # don't align (paired != list) + assert not c_t("paired:paired").has_subcollections_of_type("list:paired_or_unpaired") + + +def test_compound_paired_or_unpaired_effective_collection_type(): + """Test effective_collection_type with compound :paired_or_unpaired.""" + # list:list over list:paired_or_unpaired → list + assert c_t("list:list").effective_collection_type("list:paired_or_unpaired") == "list" + + # list:list:paired over list:paired_or_unpaired → list + # (peel off list and paired_or_unpaired ranks, then strip :paired) + assert c_t("list:list:paired").effective_collection_type("list:paired_or_unpaired") == "list" + + # list:list:list over list:paired_or_unpaired → list:list + assert c_t("list:list:list").effective_collection_type("list:paired_or_unpaired") == "list:list" + + +def test_effective_collection_type_paired_or_unpaired_non_paired(): + """Test that paired_or_unpaired acts like single_datasets for non-paired collections.""" + # list over paired_or_unpaired → list (full type preserved, like single_datasets) + assert c_t("list").effective_collection_type("paired_or_unpaired") == "list" + # list:list over paired_or_unpaired → list:list + assert c_t("list:list").effective_collection_type("paired_or_unpaired") == "list:list" + + def test_validate(): c_t("list").validate() c_t("list:paired").validate() diff --git a/test/unit/data/model/test_collection_semantics.py b/test/unit/data/model/test_collection_semantics.py new file mode 100644 index 00000000000..a12f24f96b6 --- /dev/null +++ b/test/unit/data/model/test_collection_semantics.py @@ -0,0 +1,609 @@ +import pytest +import yaml +from pydantic import ValidationError + +from galaxy.model.dataset_collections.types.semantics import ( + _assumption_elements_to_latex, + arg_parser, + check, + collect_docs_with_examples, + CollectionDefinition, + CollectionInput, + CollectionOutput, + DatasetInput, + DatasetListInput, + DatasetOutput, + DatasetsDeclaration, + DocEntry, + EllipsisMarker, + EquivalenceThen, + Example, + ExampleEntry, + ExampleTests, + expression_to_latex, + generate_docs, + InvalidThen, + MapOverInput, + MapOverThen, + NestedElements, + ReductionThen, + ToolDefinition, + ToolInvocation, + ToolOutputRef, + ToolRuntimeApi, + ToolRuntimeFramework, + validate_api_test_refs, + validate_tool_refs, + validate_workflow_editor_refs, + YAMLRootModel, +) +from galaxy.util.resources import resource_string + + +def _load_root_model() -> YAMLRootModel: + raw = yaml.safe_load(resource_string("galaxy.model.dataset_collections.types", "collection_semantics.yml")) + return YAMLRootModel.model_validate(raw) + + +def test_yaml_loads_and_parses(): + root = _load_root_model() + assert len(root.root) > 0 + + +def test_all_entries_are_doc_or_example(): + root = _load_root_model() + for entry in root.root: + assert isinstance(entry, (DocEntry, ExampleEntry)) + + +def test_has_both_doc_and_example_entries(): + root = _load_root_model() + has_doc = any(isinstance(e, DocEntry) for e in root.root) + has_example = any(isinstance(e, ExampleEntry) for e in root.root) + assert has_doc + assert has_example + + +@pytest.mark.parametrize( + "input_expr, expected_substr", + [ + ("->", "\\rightarrow"), + ("~>", "\\mapsto"), + ("list", "\\text{list}"), + ("forward", "\\text{forward}"), + ("reverse", "\\text{reverse}"), + ("mapOver", "\\text{mapOver}"), + ("collection", "\\text{collection}"), + ("dataset", "\\text{dataset}"), + ], +) +def test_expression_to_latex_replacements(input_expr, expected_substr): + result = expression_to_latex(input_expr) + assert expected_substr in result + + +def test_expression_to_latex_paired_or_unpaired_atomic(): + result = expression_to_latex("paired_or_unpaired") + assert "\\text{paired\\_or\\_unpaired}" in result + # should NOT contain bare \text{paired} or \text{unpaired} separately + without_pou = result.replace("\\text{paired\\_or\\_unpaired}", "") + assert "\\text{paired}" not in without_pou + assert "\\text{unpaired}" not in without_pou + + +def test_expression_to_latex_wrap_true(): + result = expression_to_latex("x", wrap=True) + assert result.startswith("$ ") + assert result.endswith(" $") + + +def test_expression_to_latex_wrap_false(): + result = expression_to_latex("x", wrap=False) + assert not result.startswith("$ ") + + +def test_assumption_elements_to_latex_single_none(): + result = _assumption_elements_to_latex({"a": None}) + assert "a" in result + assert "\\left\\{" in result + assert "\\right\\}" in result + + +def test_assumption_elements_to_latex_nested_dict(): + result = _assumption_elements_to_latex({"outer": {"inner": None}}) + assert "\\text{ outer }" in result + assert "inner" in result + + +def test_assumption_elements_to_latex_multiple_keys(): + result = _assumption_elements_to_latex({"a": None, "b": None}) + assert "a" in result + assert "b" in result + assert "," in result + + +def test_datasets_declaration_as_latex_single(): + d = DatasetsDeclaration(datasets=["x"]) + assert d.as_latex() == "$ x $" + + +def test_datasets_declaration_as_latex_multiple(): + d = DatasetsDeclaration(datasets=["x", "y"]) + result = d.as_latex() + assert "$ x $" in result + assert "$ y $" in result + + +def test_tool_definition_as_latex(): + t = ToolDefinition(**{"in": {"i": "dataset"}, "out": {"o": "dataset"}}) + result = t.as_latex() + assert "\\Rightarrow" in result + assert "i:" in result + assert "o:" in result + + +def test_collection_definition_as_latex(): + c = CollectionDefinition(collection_type="paired", elements={"forward": None, "reverse": None}) + result = c.as_latex() + assert "paired" in result + assert "forward" in result + assert "reverse" in result + + +def _make_root(entries: list) -> YAMLRootModel: + return YAMLRootModel.model_validate(entries) + + +def test_collect_doc_with_no_examples(): + root = _make_root([{"doc": "Some doc text"}]) + result = collect_docs_with_examples(root) + assert len(result) == 1 + doc, examples = result[0] + assert doc.doc == "Some doc text" + assert examples == [] + + +_SIMPLE_THEN = {"type": "invalid", "invocation": {"inputs": {"i": {"type": "collection", "ref": "C"}}}} + + +def test_collect_doc_with_examples(): + root = _make_root( + [ + {"doc": "Doc"}, + {"example": {"label": "EX1", "then": _SIMPLE_THEN}}, + ] + ) + result = collect_docs_with_examples(root) + assert len(result) == 1 + _, examples = result[0] + assert len(examples) == 1 + assert examples[0].example.label == "EX1" + + +def test_collect_example_without_then_excluded(): + root = _make_root( + [ + {"doc": "Doc"}, + {"example": {"label": "NO_THEN"}}, + {"example": {"label": "HAS_THEN", "then": _SIMPLE_THEN}}, + ] + ) + result = collect_docs_with_examples(root) + _, examples = result[0] + assert len(examples) == 1 + assert examples[0].example.label == "HAS_THEN" + + +def test_collect_multiple_doc_sections(): + root = _make_root( + [ + {"doc": "Doc1"}, + {"example": {"label": "EX1", "then": _SIMPLE_THEN}}, + {"doc": "Doc2"}, + {"example": {"label": "EX2", "then": _SIMPLE_THEN}}, + ] + ) + result = collect_docs_with_examples(root) + assert len(result) == 2 + assert result[0][0].doc == "Doc1" + assert result[1][0].doc == "Doc2" + assert len(result[0][1]) == 1 + assert len(result[1][1]) == 1 + + +def test_arg_parser_default(): + args = arg_parser().parse_args([]) + assert args.check is False + + +def test_arg_parser_check_flag(): + args = arg_parser().parse_args(["--check"]) + assert args.check is True + + +def test_arg_parser_check_short_flag(): + args = arg_parser().parse_args(["-c"]) + assert args.check is True + + +def test_example_tests_rejects_unknown_keys(): + with pytest.raises(ValidationError): + ExampleTests(**{"wf_editor": "some value"}) + + +def test_example_tests_accepts_valid_keys(): + et = ExampleTests(workflow_editor="some test description") + assert et.workflow_editor == "some test description" + + +def _extract_examples(root: YAMLRootModel) -> list[Example]: + return [e.example for e in root.root if isinstance(e, ExampleEntry)] + + +def test_validate_api_test_refs_clean(): + """Current YAML should have no api_test reference errors.""" + root = _load_root_model() + examples = _extract_examples(root) + errors = validate_api_test_refs(examples) + assert errors == [] + + +def test_validate_api_test_refs_bad_file(): + """Mutated api_test ref with nonexistent file should be caught.""" + ex = Example( + label="BAD_FILE", + tests=ExampleTests(tool_runtime=ToolRuntimeApi(api_test="nonexistent_file.py::some_func")), + ) + errors = validate_api_test_refs([ex]) + assert len(errors) == 1 + assert "nonexistent_file.py" in errors[0] + + +def test_validate_api_test_refs_bad_function(): + """Mutated api_test ref with real file but nonexistent function should be caught.""" + ex = Example( + label="BAD_FUNC", + tests=ExampleTests(tool_runtime=ToolRuntimeApi(api_test="test_tool_execute.py::nonexistent_function")), + ) + errors = validate_api_test_refs([ex]) + assert len(errors) == 1 + assert "nonexistent_function" in errors[0] + + +def test_validate_api_test_refs_bad_method(): + """Mutated api_test ref with real class but nonexistent method should be caught.""" + ex = Example( + label="BAD_METHOD", + tests=ExampleTests(tool_runtime=ToolRuntimeApi(api_test="test_tools.py::TestToolsApi::nonexistent_method")), + ) + errors = validate_api_test_refs([ex]) + assert len(errors) == 1 + assert "nonexistent_method" in errors[0] + + +def test_validate_tool_refs_clean(): + """Current YAML should have no framework tool reference errors.""" + root = _load_root_model() + examples = _extract_examples(root) + errors = validate_tool_refs(examples) + assert errors == [] + + +def test_validate_tool_refs_bad_tool(): + """Nonexistent tool XML should be caught.""" + ex = Example( + label="BAD_TOOL", + tests=ExampleTests(tool_runtime=ToolRuntimeFramework(tool="nonexistent_tool_xyz")), + ) + errors = validate_tool_refs([ex]) + assert len(errors) == 1 + assert "nonexistent_tool_xyz" in errors[0] + + +def test_validate_workflow_editor_refs_clean(): + """Current YAML should have no workflow_editor reference errors.""" + root = _load_root_model() + examples = _extract_examples(root) + errors = validate_workflow_editor_refs(examples) + assert errors == [] + + +def test_validate_workflow_editor_refs_bad_ref(): + """Nonexistent workflow_editor description should be caught.""" + ex = Example( + label="BAD_WF", + tests=ExampleTests(workflow_editor="totally nonexistent test description xyz"), + ) + errors = validate_workflow_editor_refs([ex]) + assert len(errors) == 1 + assert "totally nonexistent test description xyz" in errors[0] + + +def test_all_tested_examples_have_then(): + """Every example with test references should also have a then expression.""" + root = _load_root_model() + missing = [] + for entry in root.root: + if isinstance(entry, ExampleEntry): + ex = entry.example + if ex.tests and not ex.then: + missing.append(ex.label) + assert missing == [], f"Examples with tests but no then: {missing}" + + +def test_check_returns_no_errors(): + """check() against current YAML should return empty error list.""" + errors = check() + assert errors == [] + + +# --- Structured Then Expression Model Tests --- + + +def test_invalid_then_collection_input(): + """InvalidThen with collection input: tool(i=C)""" + then = InvalidThen( + type="invalid", + invocation=ToolInvocation(inputs={"i": CollectionInput(type="collection", ref="C")}), + ) + assert then.as_latex() == expression_to_latex("tool(i=C)", wrap=False) + + +def test_invalid_then_map_over_with_sub_collection(): + """InvalidThen with mapOver sub-collection: tool(i=mapOver(C, 'paired'))""" + then = InvalidThen( + type="invalid", + invocation=ToolInvocation( + inputs={"i": MapOverInput(type="map_over", collection="C", sub_collection_type="paired")} + ), + ) + assert then.as_latex() == expression_to_latex("tool(i=mapOver(C, 'paired'))", wrap=False) + + +def test_invalid_then_map_over_paired_or_unpaired(): + """InvalidThen with mapOver paired_or_unpaired sub-collection.""" + then = InvalidThen( + type="invalid", + invocation=ToolInvocation( + inputs={"i": MapOverInput(type="map_over", collection="C", sub_collection_type="paired_or_unpaired")} + ), + ) + assert then.as_latex() == expression_to_latex("tool(i=mapOver(C, 'paired_or_unpaired'))", wrap=False) + + +def test_invalid_then_map_over_no_sub_collection(): + """InvalidThen with bare mapOver: tool(i=mapOver(C))""" + then = InvalidThen( + type="invalid", + invocation=ToolInvocation(inputs={"i": MapOverInput(type="map_over", collection="C")}), + ) + assert then.as_latex() == expression_to_latex("tool(i=mapOver(C))", wrap=False) + + +def test_reduction_then(): + """ReductionThen: tool(i=C) -> {o: dataset}""" + then = ReductionThen( + type="reduction", + invocation=ToolInvocation(inputs={"i": CollectionInput(type="collection", ref="C")}), + produces={"o": DatasetOutput(type="dataset")}, + ) + assert then.as_latex() == expression_to_latex("tool(i=C) -> {o: dataset}", wrap=False) + + +def test_equivalence_then_list_reduction(): + """EquivalenceThen: tool(i=C) == tool(i=[d_1,...,d_n])""" + then = EquivalenceThen( + type="equivalence", + left=ToolInvocation(inputs={"i": CollectionInput(type="collection", ref="C")}), + right=ToolInvocation(inputs={"i": DatasetListInput(type="dataset_list", refs=["d_1", "...", "d_n"])}), + ) + assert then.as_latex() == expression_to_latex("tool(i=C) == tool(i=[d_1,...,d_n])", wrap=False) + + +def test_equivalence_then_collection_refs(): + """EquivalenceThen: tool(i=C) == tool(i=C_AS_MIXED)""" + then = EquivalenceThen( + type="equivalence", + left=ToolInvocation(inputs={"i": CollectionInput(type="collection", ref="C")}), + right=ToolInvocation(inputs={"i": CollectionInput(type="collection", ref="C_AS_MIXED")}), + ) + assert then.as_latex() == expression_to_latex("tool(i=C) == tool(i=C_AS_MIXED)", wrap=False) + + +def test_map_over_then_paired(): + """MapOverThen for BASIC_MAPPING_PAIRED pattern.""" + then = MapOverThen( + type="map_over", + invocation=ToolInvocation(inputs={"i": MapOverInput(type="map_over", collection="C")}), + produces={ + "o": CollectionOutput( + type="collection", + collection_type="paired", + elements={ + "forward": ToolOutputRef( + type="tool_output_ref", + invocation=ToolInvocation(inputs={"i": DatasetInput(type="dataset", ref="d_f")}), + output="o", + ), + "reverse": ToolOutputRef( + type="tool_output_ref", + invocation=ToolInvocation(inputs={"i": DatasetInput(type="dataset", ref="d_r")}), + output="o", + ), + }, + ) + }, + ) + result = then.as_latex() + assert "\\text{mapOver}(C)" in result + assert "\\text{paired}" in result + assert "\\mapsto" in result + assert "tool(i=d_f)[o]" in result + assert "tool(i=d_r)[o]" in result + + +def test_map_over_then_sub_collection(): + """MapOverThen for sub-collection mapping (MAPPING_LIST_PAIRED_OVER_PAIRED).""" + then = MapOverThen( + type="map_over", + invocation=ToolInvocation( + inputs={"i": MapOverInput(type="map_over", collection="C", sub_collection_type="paired")} + ), + produces={ + "o": CollectionOutput( + type="collection", + collection_type="list", + elements={ + "el1": ToolOutputRef( + type="tool_output_ref", + invocation=ToolInvocation(inputs={"i": CollectionInput(type="collection", ref="C\\_PAIRED")}), + output="o", + ), + }, + ) + }, + ) + result = then.as_latex() + assert "\\text{mapOver}(C, '\\text{paired}')" in result + assert "\\text{list}" in result + assert "C\\_PAIRED" in result + + +def test_pydantic_parse_invalid_then_yaml(): + """Structured InvalidThen parses from dict (simulating YAML load).""" + data = { + "type": "invalid", + "invocation": {"inputs": {"i": {"type": "collection", "ref": "C"}}}, + } + then = InvalidThen.model_validate(data) + assert isinstance(then.invocation.inputs["i"], CollectionInput) + assert then.invocation.inputs["i"].ref == "C" + + +def test_pydantic_parse_reduction_then_yaml(): + """Structured ReductionThen parses from dict.""" + data = { + "type": "reduction", + "invocation": {"inputs": {"i": {"type": "collection", "ref": "C"}}}, + "produces": {"o": {"type": "dataset"}}, + } + then = ReductionThen.model_validate(data) + assert isinstance(then.produces["o"], DatasetOutput) + + +def test_example_accepts_structured_then(): + """Example model accepts structured then expression.""" + data = { + "label": "TEST", + "then": { + "type": "invalid", + "invocation": {"inputs": {"i": {"type": "collection", "ref": "C"}}}, + }, + "is_valid": False, + } + ex = Example.model_validate(data) + assert isinstance(ex.then, InvalidThen) + assert ex.is_valid is False + + +def test_example_rejects_string_then(): + """Example model no longer accepts string then after migration.""" + with pytest.raises(ValidationError): + Example(label="TEST", then="tool(i=C) -> {o: dataset}") + + +# --- Smoke Tests (Steps 3-4) --- + + +def test_all_then_as_latex_succeeds(): + """Every then expression produces valid LaTeX without error.""" + root = _load_root_model() + for entry in root.root: + if isinstance(entry, ExampleEntry) and entry.example.then: + latex = entry.example.then.as_latex() + assert isinstance(latex, str) + assert len(latex) > 0, f"{entry.example.label} produced empty LaTeX" + + +def test_generate_docs_succeeds(): + """generate_docs() runs to completion without errors.""" + generate_docs() + + +# --- Unit Tests for Uncovered Models (Step 5) --- + + +def test_ellipsis_marker_as_latex(): + m = EllipsisMarker(type="ellipsis") + assert m.as_latex() == "..." + + +def test_dataset_list_input_as_latex(): + d = DatasetListInput(type="dataset_list", refs=["d_1", "...", "d_n"]) + assert d.as_latex() == "[d_1,...,d_n]" + + +def test_nested_elements_as_latex(): + ne = NestedElements.model_validate( + { + "type": "nested_elements", + "elements": { + "forward": { + "type": "tool_output_ref", + "invocation": {"inputs": {"i": {"type": "dataset", "ref": "d_f"}}}, + "output": "o", + }, + "reverse": { + "type": "tool_output_ref", + "invocation": {"inputs": {"i": {"type": "dataset", "ref": "d_r"}}}, + "output": "o", + }, + }, + } + ) + result = ne.as_latex() + assert "\\text{forward}=" in result + assert "\\text{reverse}=" in result + assert "tool(i=d_f)[o]" in result + + +def test_collection_output_with_ellipsis_as_latex(): + co = CollectionOutput.model_validate( + { + "type": "collection", + "collection_type": "list", + "elements": { + "el1": { + "type": "tool_output_ref", + "invocation": {"inputs": {"i": {"type": "dataset", "ref": "d_1"}}}, + "output": "o", + }, + "...": {"type": "ellipsis"}, + "eln": { + "type": "tool_output_ref", + "invocation": {"inputs": {"i": {"type": "dataset", "ref": "d_n"}}}, + "output": "o", + }, + }, + } + ) + result = co.as_latex() + assert "\\text{list}" in result + assert ",...," in result + + +def test_map_over_input_compound_sub_collection_type(): + mi = MapOverInput(type="map_over", collection="C", sub_collection_type="list:paired_or_unpaired") + result = mi.as_latex() + assert "\\text{list}:\\text{paired\\_or\\_unpaired}" in result + + +def test_map_over_then_produces_dataset(): + then = MapOverThen( + type="map_over", + invocation=ToolInvocation(inputs={"i": MapOverInput(type="map_over", collection="C")}), + produces={"o": DatasetOutput(type="dataset")}, + ) + result = then.as_latex() + assert "\\mapsto" in result + assert "\\text{dataset}" in result diff --git a/test/unit/tool_util/test_cwl_util.py b/test/unit/tool_util/test_cwl_util.py index 57881e15981..84c513c76bb 100644 --- a/test/unit/tool_util/test_cwl_util.py +++ b/test/unit/tool_util/test_cwl_util.py @@ -151,3 +151,107 @@ def test_galactic_job_json_collection_element_filetype(): for target in captured_targets: assert isinstance(target, FileLiteralTarget) assert target.properties.get("filetype") == "fastqsanger" + + +def test_galactic_job_json_sample_sheet_collection_without_rows(): + """sample_sheet collection works without rows metadata.""" + created_collections = [] + + def collection_create_func(element_identifiers, collection_type, rows=None, name=None): + created_collections.append( + {"element_identifiers": element_identifiers, "collection_type": collection_type, "rows": rows, "name": name} + ) + return {"id": f"collection_{collection_type}"} + + job = { + "input1": { + "class": "Collection", + "collection_type": "sample_sheet", + "elements": [ + {"identifier": "el1", "class": "File", "contents": "element 1"}, + {"identifier": "el2", "class": "File", "contents": "element 2"}, + ], + } + } + result_job, datasets = galactic_job_json( + job, ".", _mock_upload_func, collection_create_func, tool_or_workflow="workflow" + ) + assert len(created_collections) == 1 + coll = created_collections[0] + assert coll["collection_type"] == "sample_sheet" + assert coll["rows"] is None + assert len(coll["element_identifiers"]) == 2 + assert coll["element_identifiers"][0]["name"] == "el1" + assert coll["element_identifiers"][1]["name"] == "el2" + + +def test_galactic_job_json_sample_sheet_collection_with_rows(): + """sample_sheet collection passes rows metadata through.""" + created_collections = [] + + def collection_create_func(element_identifiers, collection_type, rows=None, name=None): + created_collections.append( + {"element_identifiers": element_identifiers, "collection_type": collection_type, "rows": rows, "name": name} + ) + return {"id": f"collection_{collection_type}"} + + job = { + "input1": { + "class": "Collection", + "collection_type": "sample_sheet", + "elements": [ + {"identifier": "el1", "class": "File", "contents": "element 1"}, + {"identifier": "el2", "class": "File", "contents": "element 2"}, + ], + "rows": {"el1": {"condition": "treatment"}, "el2": {"condition": "control"}}, + } + } + result_job, datasets = galactic_job_json( + job, ".", _mock_upload_func, collection_create_func, tool_or_workflow="workflow" + ) + assert len(created_collections) == 1 + coll = created_collections[0] + assert coll["collection_type"] == "sample_sheet" + assert coll["rows"] == {"el1": {"condition": "treatment"}, "el2": {"condition": "control"}} + + +def test_galactic_job_json_sample_sheet_paired_collection(): + """sample_sheet:paired creates nested collection with paired sub-elements.""" + created_collections = [] + + def collection_create_func(element_identifiers, collection_type, rows=None, name=None): + created_collections.append( + {"element_identifiers": element_identifiers, "collection_type": collection_type, "rows": rows, "name": name} + ) + return {"id": f"collection_{collection_type}"} + + job = { + "input1": { + "class": "Collection", + "collection_type": "sample_sheet:paired", + "elements": [ + { + "identifier": "sample1", + "class": "Collection", + "type": "paired", + "elements": [ + {"identifier": "forward", "class": "File", "contents": "fwd reads"}, + {"identifier": "reverse", "class": "File", "contents": "rev reads"}, + ], + } + ], + } + } + result_job, datasets = galactic_job_json( + job, ".", _mock_upload_func, collection_create_func, tool_or_workflow="workflow" + ) + assert len(created_collections) == 1 + coll = created_collections[0] + assert coll["collection_type"] == "sample_sheet:paired" + assert coll["rows"] is None + assert len(coll["element_identifiers"]) == 1 + el = coll["element_identifiers"][0] + assert el["name"] == "sample1" + assert el["src"] == "new_collection" + assert el["collection_type"] == "paired" + assert len(el["element_identifiers"]) == 2