opensource/galaxy
3
0
Fork 0
mirror of https://github.com/galaxyproject/galaxy.git synced 2026-10-31 02:28:09 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #22170 from jmchilton/fix_paired_unpaired_map_over

Browse Source 
More paired_or_unpaired fixes.
This commit is contained in:
Marius van den Beek
2026-03-19 10:13:52 +01:00
committed by GitHub
parent e994848f48 17d1e2286b
commit f7258eabfa
26 changed files with 3109 additions and 133 deletions
Download Patch File Download Diff File Expand all files Collapse all files
doc/source/dev/collection_semantics.md
+421 -63
View File
@@ -8,7 +8,7 @@ In particular it describes how they operate within Galaxy tools and workflows.
Any significantly sophisticated workflow language will have ways to collect data
into arrays or vectors or dictionaries and apply operations across this data (mapping)
or reduce the dimensionality of this data (reductions). Typically, this explicitly
or reduce the dimensionality of this data (reductions). Typically, this is explicitly
annotated with map functions or for loops. Galaxy however is designed to be a point
and click interface for connecting steps and running tools. It is important that steps
just connect and just do the most natural thing - and this is what Galaxy does.
@@ -42,12 +42,12 @@ Assuming,
* $ d_f $, $ d_r $ are datasets
* $ tool \text{ is } (i: \text{ dataset }) \Rightarrow \{ o: \text{ dataset } \} $
* $ C $ is $ \text{CollectionInstance<}paired,\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\}\text{>} $
* $ C $ is $ \text{CollectionInstance<}\text{paired},\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\}\text{>} $
then
$$tool(i=\text{mapOver}(C)) \mapsto \left\{o: collection<paired, \left\{\text{forward}=tool(i=d_f)[o], \text{reverse}=tool(i=d_r)[o]\right\}> \right\}$$
$$tool(i=\text{mapOver}(C)) \mapsto \left\{o: \text{collection}<\text{paired},\left\{\text{forward}=tool(i=d_f)[o], \text{reverse}=tool(i=d_r)[o]\right\}>\right\}$$
:::
@@ -60,12 +60,12 @@ Assuming,
* $ d_f $, $ d_r $ are datasets
* $ tool \text{ is } (i: \text{ dataset }) \Rightarrow \{ o: \text{ dataset } \} $
* $ C $ is $ \text{CollectionInstance<}paired\_or\_unpaired,\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\}\text{>} $
* $ C $ is $ \text{CollectionInstance<}\text{paired\_or\_unpaired},\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\}\text{>} $
then
$$tool(i=\text{mapOver}(C)) \mapsto \left\{o: collection<paired\_or\_unpaired,\left\{\text{forward}=tool(i=d_f)[o], \text{reverse}=tool(i=d_r)[o]\right\}>\right\}$$
$$tool(i=\text{mapOver}(C)) \mapsto \left\{o: \text{collection}<\text{paired\_or\_unpaired},\left\{\text{forward}=tool(i=d_f)[o], \text{reverse}=tool(i=d_r)[o]\right\}>\right\}$$
:::
@@ -78,12 +78,12 @@ Assuming,
* $ d_u $ is a dataset
* $ tool \text{ is } (i: \text{ dataset }) \Rightarrow \{ o: \text{ dataset } \} $
* $ C $ is $ \text{CollectionInstance<}paired\_or\_unpaired,\left\{ \text{ unpaired }=d_u \right\}\text{>} $
* $ C $ is $ \text{CollectionInstance<}\text{paired\_or\_unpaired},\left\{ \text{ unpaired }=d_u \right\}\text{>} $
then
$$tool(i=\text{mapOver}(C)) \mapsto \left\{o: collection<paired\_or\_unpaired,\left\{unpaired=tool(i=d_u)[o]\right\}>\right\}$$
$$tool(i=\text{mapOver}(C)) \mapsto \left\{o: \text{collection}<\text{paired\_or\_unpaired},\left\{\text{unpaired}=tool(i=d_u)[o]\right\}>\right\}$$
:::
@@ -96,12 +96,12 @@ Assuming,
* $ d_1,...,d_n $ are datasets
* $ tool \text{ is } (i: \text{ dataset }) \Rightarrow \{ o: \text{ dataset } \} $
* $ C $ is $ \text{CollectionInstance<}list,\left\{ \text{ i1 }=d_1, ..., \text{ in }=d_n \right\}\text{>} $
* $ C $ is $ \text{CollectionInstance<}\text{list},\left\{ \text{ i1 }=d_1, ..., \text{ in }=d_n \right\}\text{>} $
then
$$tool(i=\text{mapOver}(C)) \mapsto \left\{o: collection<\text{list},\left\{i1=tool(i=d_1)[o],...,in=tool(i=d_n)[o]]\right\}>\right\}$$
$$tool(i=\text{mapOver}(C)) \mapsto \left\{o: \text{collection}<\text{list},\left\{\text{i1}=tool(i=d_1)[o],...,\text{in}=tool(i=d_n)[o]\right\}>\right\}$$
:::
@@ -126,12 +126,12 @@ Assuming,
* $ d_1,...,d_n $ are datasets
* $ tool \text{ is } (i: \text{ dataset }) \Rightarrow \{ o: \text{ dataset } \} $
* $ C $ is $ \text{CollectionInstance<}list:list,\left\{ \text{ o1 }=\left\{ \text{ inner }=d_1 \right\}, ..., \text{ on }=\left\{ \text{ inner }=d_n \right\} \right\}\text{>} $
* $ C $ is $ \text{CollectionInstance<}\text{list}:\text{list},\left\{ \text{ o1 }=\left\{ \text{ inner }=d_1 \right\}, ..., \text{ on }=\left\{ \text{ inner }=d_n \right\} \right\}\text{>} $
then
$$tool(i=\text{mapOver}(C)) \mapsto \left\{o: collection<\text{list}:\text{list},\left\{o1=\left\{inner=tool(i=d_1)[o]\right\}\right\},...,\left\{on=\left\{inner=tool(i=d_n)[o]\right\}\right\}>\right\}$$
$$tool(i=\text{mapOver}(C)) \mapsto \left\{o: \text{collection}<\text{list}:\text{list},\left\{\text{o1}=\left\{\text{inner}=tool(i=d_1)[o]\right\},...,\text{on}=\left\{\text{inner}=tool(i=d_n)[o]\right\}\right\}>\right\}$$
:::
@@ -144,12 +144,12 @@ Assuming,
* $ d_f $, $ d_r $ are datasets
* $ tool \text{ is } (i: \text{ dataset }) \Rightarrow \{ o: \text{ dataset } \} $
* $ C $ is $ \text{CollectionInstance<}list:paired\_or\_unpaired,\left\{ \text{ el1 }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\} \right\}\text{>} $
* $ C $ is $ \text{CollectionInstance<}\text{list}:\text{paired\_or\_unpaired},\left\{ \text{ el1 }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\} \right\}\text{>} $
then
$$tool(i=\text{mapOver}(C)) \mapsto \left\{o: collection<\text{list}:paired\_or\_unpaired,\left\{el1=\left\{\text{forward}=tool(i=d_f)[o],\text{reverse}=tool(i=d_r)[o]\right\}\right\}>\right\}$$
$$tool(i=\text{mapOver}(C)) \mapsto \left\{o: \text{collection}<\text{list}:\text{paired\_or\_unpaired},\left\{\text{el1}=\left\{\text{forward}=tool(i=d_f)[o], \text{reverse}=tool(i=d_r)[o]\right\}\right\}>\right\}$$
:::
@@ -175,12 +175,12 @@ Assuming,
* $ d_1,...,d_n $, $ d_o $ are datasets
* $ tool \text{ is } (i: \text{ dataset }, i2: \text{ dataset }) \Rightarrow \{ o: \text{ dataset } \} $
* $ C $ is $ \text{CollectionInstance<}list,\left\{ \text{ i1 }=d_1, ..., \text{ in }=d_n \right\}\text{>} $
* $ C $ is $ \text{CollectionInstance<}\text{list},\left\{ \text{ i1 }=d_1, ..., \text{ in }=d_n \right\}\text{>} $
then
$$tool(i=\text{mapOver}(C),i2=d_o) \mapsto \left\{o: collection<\text{list},\left\{i1=tool(i=d_1, i2=d_o)[o],...,in=tool(i=d_n, i2=d_o)[o]\right\}>\right\}$$
$$tool(i=\text{mapOver}(C), i2=d_o) \mapsto \left\{o: \text{collection}<\text{list},\left\{\text{i1}=tool(i=d_1, i2=d_o)[o],...,\text{in}=tool(i=d_n, i2=d_o)[o]\right\}>\right\}$$
:::
@@ -217,13 +217,13 @@ Assuming,
* $ d1_1,...,d1_n $, $ d2_1,...,d2_n $ are datasets
* $ tool \text{ is } (i: \text{ dataset }, i2: \text{ dataset }) \Rightarrow \{ o: \text{ dataset } \} $
* $ C1 $ is $ \text{CollectionInstance<}list,\left\{ \text{ i1 }=d1_1, ..., \text{ in }=d1_n \right\}\text{>} $
* $ C2 $ is $ \text{CollectionInstance<}list,\left\{ \text{ i1 }=d2_1, ..., \text{ in }=d2_n \right\}\text{>} $
* $ C1 $ is $ \text{CollectionInstance<}\text{list},\left\{ \text{ i1 }=d1_1, ..., \text{ in }=d1_n \right\}\text{>} $
* $ C2 $ is $ \text{CollectionInstance<}\text{list},\left\{ \text{ i1 }=d2_1, ..., \text{ in }=d2_n \right\}\text{>} $
then
$$tool(i=\text{mapOver}(C1), i2=\text{mapOver}(C2)) \mapsto \left\{o: collection<\text{list},\left\{i1=tool(i=d1_1, i2=d2_1)[o],...,in=tool(i=d1_n, i2=d2_n)[o]]\right\}>\right\}$$
$$tool(i=\text{mapOver}(C1), i2=\text{mapOver}(C2)) \mapsto \left\{o: \text{collection}<\text{list},\left\{\text{i1}=tool(i=d1_1, i2=d2_1)[o],...,\text{in}=tool(i=d1_n, i2=d2_n)[o]\right\}>\right\}$$
:::
@@ -241,7 +241,7 @@ collections directly and do not necessarily result in mapping over.
Tools that consume collections and output datasets effectively
reduce the dimension of the Galaxy data structure. When used at runtime
this is often referred to a "reduction" in the code.
this is often referred to as a "reduction" in the code.
(COLLECTION_INPUT_PAIRED)=
@@ -257,12 +257,12 @@ Assuming,
* $ d_f $, $ d_r $ are datasets
* $ tool \text{ is } (i: \text{ collection<paired> }) \Rightarrow \{ o: \text{ dataset } \} $
* $ C $ is $ \text{CollectionInstance<}paired,\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\}\text{>} $
* $ C $ is $ \text{CollectionInstance<}\text{paired},\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\}\text{>} $
then
$$tool(i=C) \rightarrow \left\{o: dataset\right\}$$
$$tool(i=C) \rightarrow \left\{o: \text{dataset}\right\}$$
:::
@@ -273,14 +273,14 @@ $$tool(i=C) \rightarrow \left\{o: dataset\right\}$$
Assuming,
* $ d1,...,dn $ are datasets
* $ d_1,...,d_n $ are datasets
* $ tool \text{ is } (i: \text{ collection<list> }) \Rightarrow \{ o: \text{ dataset } \} $
* $ C $ is $ \text{CollectionInstance<}list,\left\{ \text{ el1 }=d_1, ..., \text{ eln }=d_n \right\}\text{>} $
* $ C $ is $ \text{CollectionInstance<}\text{list},\left\{ \text{ el1 }=d_1, ..., \text{ eln }=d_n \right\}\text{>} $
then
$$tool(i=C) \rightarrow \left\{o: dataset\right\}$$
$$tool(i=C) \rightarrow \left\{o: \text{dataset}\right\}$$
:::
@@ -292,13 +292,13 @@ $$tool(i=C) \rightarrow \left\{o: dataset\right\}$$
Assuming,
* $ d_f $, $ d_r $ are datasets
* $ tool \text{ is } (i: \text{ collection<paired_or_unpaired> }) \Rightarrow \{ o: \text{ dataset } \} $
* $ C $ is $ \text{CollectionInstance<}paired\_or\_unpaired,\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\}\text{>} $
* $ tool \text{ is } (i: \text{ collection<paired\_or\_unpaired> }) \Rightarrow \{ o: \text{ dataset } \} $
* $ C $ is $ \text{CollectionInstance<}\text{paired\_or\_unpaired},\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\}\text{>} $
then
$$tool(i=C) \rightarrow \left\{o: dataset\right\}$$
$$tool(i=C) \rightarrow \left\{o: \text{dataset}\right\}$$
:::
@@ -310,13 +310,13 @@ $$tool(i=C) \rightarrow \left\{o: dataset\right\}$$
Assuming,
* $ d_f $, $ d_r $ are datasets
* $ tool \text{ is } (i: \text{ collection<list:paired_or_unpaired> }) \Rightarrow \{ o: \text{ dataset } \} $
* $ C $ is $ \text{CollectionInstance<}list:paired\_or\_unpaired,\left\{ \text{ el1 }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\} \right\}\text{>} $
* $ tool \text{ is } (i: \text{ collection<list:paired\_or\_unpaired> }) \Rightarrow \{ o: \text{ dataset } \} $
* $ C $ is $ \text{CollectionInstance<}\text{list}:\text{paired\_or\_unpaired},\left\{ \text{ el1 }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\} \right\}\text{>} $
then
$$tool(i=C) \rightarrow \left\{o: dataset\right\}$$
$$tool(i=C) \rightarrow \left\{o: \text{dataset}\right\}$$
:::
@@ -332,6 +332,8 @@ then collection inputs must match every part of the collection type input defini
(COLLECTION_INPUT_LIST_NOT_CONSUMES_PAIRS)=
(COLLECTION_INPUT_PAIRED_NOT_CONSUMES_LIST)=
(COLLECTION_INPUT_LIST_PAIRED_NOT_CONSUMES_PAIRED_PAIRED)=
(COLLECTION_INPUT_LIST_PAIRED_OR_NOT_PAIRED_NOT_CONSUMES_PAIRED_PAIRED)=
<details><summary>Examples</summary>
:::{admonition} Example: `COLLECTION_INPUT_LIST_NOT_CONSUMES_PAIRS`
@@ -341,7 +343,7 @@ Assuming,
* $ d_f $, $ d_r $ are datasets
* $ tool \text{ is } (i: \text{ collection<list> }) \Rightarrow \{ o: \text{ dataset } \} $
* $ C $ is $ \text{CollectionInstance<}paired,\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\}\text{>} $
* $ C $ is $ \text{CollectionInstance<}\text{paired},\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\}\text{>} $
then
@@ -359,7 +361,43 @@ Assuming,
* $ d_1,...,d_n $ are datasets
* $ tool \text{ is } (i: \text{ collection<paired> }) \Rightarrow \{ o: \text{ dataset } \} $
* $ C $ is $ \text{CollectionInstance<}list,\left\{ \text{ i1 }=d_1, ..., \text{ in }=d_n \right\}\text{>} $
* $ C $ is $ \text{CollectionInstance<}\text{list},\left\{ \text{ i1 }=d_1, ..., \text{ in }=d_n \right\}\text{>} $
then
$$tool(i=C)\text{ is invalid}$$
:::
:::{admonition} Example: `COLLECTION_INPUT_LIST_PAIRED_NOT_CONSUMES_PAIRED_PAIRED`
:class: note
Assuming,
* $ d_f $, $ d_r $ are datasets
* $ tool \text{ is } (i: \text{ collection<list:paired> }) \Rightarrow \{ o: \text{ dataset } \} $
* $ C $ is $ \text{CollectionInstance<}\text{paired}:\text{paired},\left\{ \text{ forward }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\}, \text{ reverse }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\} \right\}\text{>} $
then
$$tool(i=C)\text{ is invalid}$$
:::
:::{admonition} Example: `COLLECTION_INPUT_LIST_PAIRED_OR_NOT_PAIRED_NOT_CONSUMES_PAIRED_PAIRED`
:class: note
Assuming,
* $ d_f $, $ d_r $ are datasets
* $ tool \text{ is } (i: \text{ collection<list:paired\_or\_unpaired> }) \Rightarrow \{ o: \text{ dataset } \} $
* $ C $ is $ \text{CollectionInstance<}\text{paired}:\text{paired},\left\{ \text{ forward }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\}, \text{ reverse }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\} \right\}\text{>} $
then
@@ -389,7 +427,7 @@ Assuming,
* $ d_1,...,d_n $ are datasets
* $ tool \text{ is } (i: \text{ dataset<multiple=true> }) \Rightarrow \{ o: \text{ dataset } \} $
* $ C $ is $ \text{CollectionInstance<}list,\left\{ \text{ i1 }=d_1, ..., \text{ in }=d_n \right\}\text{>} $
* $ C $ is $ \text{CollectionInstance<}\text{list},\left\{ \text{ i1 }=d_1, ..., \text{ in }=d_n \right\}\text{>} $
then
@@ -419,7 +457,7 @@ Assuming,
* $ d_f $, $ d_r $ are datasets
* $ tool \text{ is } (i: \text{ dataset<multiple=true> }) \Rightarrow \{ o: \text{ dataset } \} $
* $ C $ is $ \text{CollectionInstance<}paired,\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\}\text{>} $
* $ C $ is $ \text{CollectionInstance<}\text{paired},\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\}\text{>} $
then
@@ -437,7 +475,7 @@ Assuming,
* $ d_f $, $ d_r $ are datasets
* $ tool \text{ is } (i: \text{ dataset<multiple=true> }) \Rightarrow \{ o: \text{ dataset } \} $
* $ C $ is $ \text{CollectionInstance<}paired\_or\_unpaired,\left\{ forward=d_f, reverse=d_r \right\}\text{>} $
* $ C $ is $ \text{CollectionInstance<}\text{paired\_or\_unpaired},\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\}\text{>} $
then
@@ -467,13 +505,13 @@ Assuming,
* $ d_f $, $ d_r $ are datasets
* $ tool \text{ is } (i: \text{ collection<paired> }) \Rightarrow \{ o: \text{ dataset } \} $
* $ C $ is $ \text{CollectionInstance<}list:paired,\left\{ \text{ el1 }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\} \right\}\text{>} $
* $ C\_PAIRED $ is $ \text{CollectionInstance<}paired,\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\}\text{>} $
* $ C $ is $ \text{CollectionInstance<}\text{list}:\text{paired},\left\{ \text{ el1 }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\} \right\}\text{>} $
* $ C\_PAIRED $ is $ \text{CollectionInstance<}\text{paired},\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\}\text{>} $
then
$$tool(i=\text{mapOver}(C, 'paired')) \mapsto \left\{o: collection<\text{list}, \left\{el1: tool(i=C\_PAIRED)[o]\right\}>\right\}$$
$$tool(i=\text{mapOver}(C, '\text{paired}')) \mapsto \left\{o: \text{collection}<\text{list},\left\{\text{el1}=tool(i=C\_PAIRED)[o]\right\}>\right\}$$
:::
@@ -483,7 +521,7 @@ $$tool(i=\text{mapOver}(C, 'paired')) \mapsto \left\{o: collection<\text{list},
The natural extension of multiple data input parameters consuming list collections as describe
The natural extension of multiple data input parameters consuming list collections as described
above when discussing reductions is that nested lists of lists (``list:list``) can be mapped
over a multiple data input parameter. Each nested list will be reduced by this operation but the
results will be mapped over. The result will be a list with the same structure as the outer list
@@ -500,12 +538,12 @@ Assuming,
* $ d_1,...,d_n $ are datasets
* $ tool \text{ is } (i: \text{ dataset<multiple=true> }) \Rightarrow \{ o: \text{ dataset } \} $
* $ C $ is $ \text{CollectionInstance<}list:list,\left\{ \text{ o1 }=\left\{ \text{ inner }=d_1 \right\}, ..., \text{ on }=\left\{ \text{ inner }=d_n \right\} \right\}\text{>} $
* $ C $ is $ \text{CollectionInstance<}\text{list}:\text{list},\left\{ \text{ o1 }=\left\{ \text{ inner }=d_1 \right\}, ..., \text{ on }=\left\{ \text{ inner }=d_n \right\} \right\}\text{>} $
then
$$tool(i=\text{mapOver}(C, '\text{list}')) \mapsto \left\{o: collection<\text{list},\left\{o1: tool(i=[d_1])[o]\right\},...,on: tool(i=[d_n])[o]\right\}>\right\}$$
$$tool(i=\text{mapOver}(C, '\text{list}')) \mapsto \left\{o: \text{collection}<\text{list},\left\{\text{o1}=tool(i=[d_1])[o],...,\text{on}=tool(i=[d_n])[o]\right\}>\right\}$$
:::
@@ -531,12 +569,12 @@ Assuming,
* $ d_f $, $ d_r $ are datasets
* $ tool \text{ is } (i: \text{ dataset<multiple=true> }) \Rightarrow \{ o: \text{ dataset } \} $
* $ C $ is $ \text{CollectionInstance<}list:paired,\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\}\text{>} $
* $ C $ is $ \text{CollectionInstance<}\text{list}:\text{paired},\left\{ \text{ el1 }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\} \right\}\text{>} $
then
$$tool(i=\text{mapOver}(C, 'paired'))\text{ is invalid}$$
$$tool(i=\text{mapOver}(C, '\text{paired}'))\text{ is invalid}$$
:::
@@ -549,12 +587,12 @@ Assuming,
* $ d_f $, $ d_r $ are datasets
* $ tool \text{ is } (i: \text{ dataset<multiple=true> }) \Rightarrow \{ o: \text{ dataset } \} $
* $ C $ is $ \text{CollectionInstance<}list:paired\_or\_unpaired,\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\}\text{>} $
* $ C $ is $ \text{CollectionInstance<}\text{list}:\text{paired\_or\_unpaired},\left\{ \text{ el1 }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\} \right\}\text{>} $
then
$$tool(i=\text{mapOver}(C, 'paired\_or\_unpaired'))\text{ is invalid}$$
$$tool(i=\text{mapOver}(C, '\text{paired\_or\_unpaired}'))\text{ is invalid}$$
:::
@@ -585,9 +623,9 @@ normally or can consume a ``paired`` input.
Assuming,
* $ d_f $, $ d_r $ are datasets
* $ tool \text{ is } (i: \text{ collection<paired_or_unpaired> }) \Rightarrow \{ o: \text{ dataset } \} $
* $ C $ is $ \text{CollectionInstance<}paired,\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\}\text{>} $
* $ C_AS_MIXED = CollectionInstance<paired\_or\_unpaired, \left\{\text{forward}: d_f, \text{reverse}: d_r\right\}> $
* $ tool \text{ is } (i: \text{ collection<paired\_or\_unpaired> }) \Rightarrow \{ o: \text{ dataset } \} $
* $ C $ is $ \text{CollectionInstance<}\text{paired},\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\}\text{>} $
* $ C_AS_MIXED = CollectionInstance<\text{paired\_or\_unpaired}, \left\{\text{forward}: d_f, \text{reverse}: d_r\right\}> $
then
@@ -603,7 +641,7 @@ $$tool(i=C) == tool(i=C_AS_MIXED)$$
This inverse of this doesn't work intentionally. In some ways a ``paired`` collection
The inverse of this doesn't work intentionally. In some ways a ``paired`` collection
acts as a ``paired_or_unpaired`` collection but a ``paired_or_unpaired`` is not a paired
collection. This makes a lot of sense in terms of tools - a tool consuming a ``paired``
dataset expects to find both a ``forward`` and ``reverse`` element but these may not exist
@@ -620,12 +658,12 @@ Assuming,
* $ d_f $, $ d_r $ are datasets
* $ tool \text{ is } (i: \text{ collection<paired> }) \Rightarrow \{ o: \text{ dataset } \} $
* $ C $ is $ \text{CollectionInstance<}paired\_or\_unpaired,\left\{ forward=d_f, \text{ reverse }=d_r \right\}\text{>} $
* $ C $ is $ \text{CollectionInstance<}\text{paired\_or\_unpaired},\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\}\text{>} $
then
$$tool(i=C) is invalid$$
$$tool(i=C)\text{ is invalid}$$
:::
@@ -653,9 +691,9 @@ be mapped over a ``list`` input or multiple data input.
Assuming,
* $ d_f $, $ d_r $ are datasets
* $ tool \text{ is } (i: \text{ collection<paired_or_unpaired> }) \Rightarrow \{ o: \text{ dataset } \} $
* $ C $ is $ \text{CollectionInstance<}list:paired,\left\{ \text{ el }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\} \right\}\text{>} $
* $ C_AS_MIXED $ is $ \text{CollectionInstance<}list:paired\_or\_unpaired,\left\{ \text{ el }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\} \right\}\text{>} $
* $ tool \text{ is } (i: \text{ collection<paired\_or\_unpaired> }) \Rightarrow \{ o: \text{ dataset } \} $
* $ C $ is $ \text{CollectionInstance<}\text{list}:\text{paired},\left\{ \text{ el }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\} \right\}\text{>} $
* $ C_AS_MIXED $ is $ \text{CollectionInstance<}\text{list}:\text{paired\_or\_unpaired},\left\{ \text{ el }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\} \right\}\text{>} $
then
@@ -673,12 +711,12 @@ Assuming,
* $ d_f $, $ d_r $ are datasets
* $ tool \text{ is } (i: \text{ collection<paired> }) \Rightarrow \{ o: \text{ dataset } \} $
* $ C $ is $ \text{CollectionInstance<}list:paired\_or\_unpaired,\left\{ \text{ el }=\left\{ \text{ forward }=f, \text{ reverse }=r \right\} \right\}\text{>} $
* $ C $ is $ \text{CollectionInstance<}\text{list}:\text{paired\_or\_unpaired},\left\{ \text{ el }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\} \right\}\text{>} $
then
$$tool(i=\text{mapOver}(C)) is invalid$$
$$tool(i=\text{mapOver}(C))\text{ is invalid}$$
:::
@@ -691,12 +729,12 @@ Assuming,
* $ d_f $, $ d_r $ are datasets
* $ tool \text{ is } (i: \text{ collection<list> }) \Rightarrow \{ o: \text{ dataset } \} $
* $ C $ is $ \text{CollectionInstance<}list:paired\_or\_unpaired,\left\{ \text{ el }=\left\{ \text{ forward }=f, \text{ reverse }=r \right\} \right\}\text{>} $
* $ C $ is $ \text{CollectionInstance<}\text{list}:\text{paired\_or\_unpaired},\left\{ \text{ el }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\} \right\}\text{>} $
then
$$tool(i=\text{mapOver}(C)) is invalid$$
$$tool(i=\text{mapOver}(C))\text{ is invalid}$$
:::
@@ -711,6 +749,32 @@ can be mapped over either a ``paired_or_unpaired`` input to produce a nested lis
or a ``list:paired_or_unpaired`` input to produce a flat list (``list``).
(MAPPING_LIST_LIST_PAIRED_OVER_PAIRED_OR_UNPAIRED)=
<details><summary>Examples</summary>
:::{admonition} Example: `MAPPING_LIST_LIST_PAIRED_OVER_PAIRED_OR_UNPAIRED`
:class: note
Assuming,
* $ d_f $, $ d_r $ are datasets
* $ tool \text{ is } (i: \text{ collection<paired\_or\_unpaired> }) \Rightarrow \{ o: \text{ dataset } \} $
* $ C $ is $ \text{CollectionInstance<}\text{list}:\text{list}:\text{paired},\left\{ \text{ o1 }=\left\{ \text{ el1 }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\} \right\} \right\}\text{>} $
* $ C_AS_MIXED $ is $ \text{CollectionInstance<}\text{list}:\text{list}:\text{paired\_or\_unpaired},\left\{ \text{ o1 }=\left\{ \text{ el1 }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\} \right\} \right\}\text{>} $
then
$$tool(i=\text{mapOver}(C)) == tool(i=\text{mapOver}(C_AS_MIXED))$$
:::
</details><br>
In order for ``paired_or_unpaired`` collections to also act as a single dataset,
a flat list can be mapped over a such an input with a special sub collection mapping
@@ -726,14 +790,14 @@ type of 'single_datasets'.
Assuming,
* $ d_1,...,d_n $ are datasets
* $ tool \text{ is } (i: \text{ collection<paired_or_unpaired> }) \Rightarrow \{ o: \text{ dataset } \} $
* $ C $ is $ \text{CollectionInstance<}list,\left\{ \text{ i1 }=d_1, ..., \text{ in }=d_n \right\}\text{>} $
* $ C_AS_UNPAIRED_i = CollectionInstance<paired\_or\_unpaired,\left\{unpaired=di\right\}> for i from 1...n $
* $ tool \text{ is } (i: \text{ collection<paired\_or\_unpaired> }) \Rightarrow \{ o: \text{ dataset } \} $
* $ C $ is $ \text{CollectionInstance<}\text{list},\left\{ \text{ i1 }=d_1, ..., \text{ in }=d_n \right\}\text{>} $
* $ C_AS_UNPAIRED_i = CollectionInstance<\text{paired\_or\_unpaired},\left\{\text{unpaired}=di\right\}> for i from 1...n $
then
$$tool(i=\text{mapOver}(C, 'single_datasets')) \mapsto \left\{o: collection<\text{list},\left\{i1=tool(i=C_AS_UNPAIRED_1)[o],...,in=tool(i=C_AS_UNPAIRED_n)[o]]\right\}>\right\}$$
$$tool(i=\text{mapOver}(C, '\text{single\_datasets}')) \mapsto \left\{o: \text{collection}<\text{list},\left\{\text{i1}=tool(i=C_AS_UNPAIRED_1)[o],...,\text{in}=tool(i=C_AS_UNPAIRED_n)[o]\right\}>\right\}$$
:::
@@ -751,7 +815,53 @@ a ``list:paired_or_unpaired`` input to produce a flat list with the same structu
as the outer list of the input.
Due only implementation time, the special casing of allowing paired_or_unpaired
(MAPPING_LIST_LIST_OVER_PAIRED_OR_UNPAIRED)=
(MAPPING_LIST_LIST_OVER_LIST_PAIRED_OR_UNPAIRED)=
<details><summary>Examples</summary>
:::{admonition} Example: `MAPPING_LIST_LIST_OVER_PAIRED_OR_UNPAIRED`
:class: note
Assuming,
* $ d_1,...,d_n $ are datasets
* $ tool \text{ is } (i: \text{ collection<paired\_or\_unpaired> }) \Rightarrow \{ o: \text{ dataset } \} $
* $ C $ is $ \text{CollectionInstance<}\text{list}:\text{list},\left\{ \text{ o1 }=\left\{ \text{ inner }=d_1 \right\}, ..., \text{ on }=\left\{ \text{ inner }=d_n \right\} \right\}\text{>} $
* $ C_AS_UNPAIRED_j = CollectionInstance<\text{paired\_or\_unpaired},\left\{\text{unpaired}=dj\right\}> for each \text{dataset} dj in C $
then
$$tool(i=\text{mapOver}(C, '\text{single\_datasets}')) \mapsto \left\{o: \text{collection}<\text{list}:\text{list},\left\{\text{o1}=\left\{\text{inner}=tool(i=C_AS_UNPAIRED_1)[o]\right\},...,\text{on}=\left\{\text{inner}=tool(i=C_AS_UNPAIRED_n)[o]\right\}\right\}>\right\}$$
:::
:::{admonition} Example: `MAPPING_LIST_LIST_OVER_LIST_PAIRED_OR_UNPAIRED`
:class: note
Assuming,
* $ d_1,...,d_n $ are datasets
* $ tool \text{ is } (i: \text{ collection<list:paired\_or\_unpaired> }) \Rightarrow \{ o: \text{ dataset } \} $
* $ C $ is $ \text{CollectionInstance<}\text{list}:\text{list},\left\{ \text{ o1 }=\left\{ \text{ inner }=d_1 \right\}, ..., \text{ on }=\left\{ \text{ inner }=d_n \right\} \right\}\text{>} $
* $ C_AS_LPU_i = \text{inner} \text{list} of C at position i, treated as \text{list}:\text{paired\_or\_unpaired} via \text{single\_datasets} $
then
$$tool(i=\text{mapOver}(C, '\text{list}:\text{paired\_or\_unpaired}')) \mapsto \left\{o: \text{collection}<\text{list},\left\{\text{o1}=tool(i=C_AS_LPU_1)[o],...,\text{on}=tool(i=C_AS_LPU_n)[o]\right\}>\right\}$$
:::
</details><br>
Due only to implementation time, the special casing of allowing paired_or_unpaired
act as both datasets and paired collections only works when it is the deepest
collection type. So while list:paired can be consumed by a list:paired_or_unpaired
input, a paired:list cannot be consumed by a paired_or_unpaired:list input though
@@ -760,3 +870,251 @@ more likely to be used in actual Galaxy analyses given current and guessed futur
usage.
## sample_sheet Collections
The collection type ``sample_sheet`` attaches typed, columnar metadata to each
element of a dataset collection. For mapping and type matching purposes,
``sample_sheet`` behaves identically to ``list`` - it can be mapped over tool
inputs, matched against ``list`` collection inputs, and composed with inner types
(``sample_sheet:paired``, ``sample_sheet:paired_or_unpaired``). The key asymmetry
is that while a ``sample_sheet`` output can satisfy a ``list`` input, a ``list``
output cannot satisfy a ``sample_sheet`` input - sample sheets carry metadata
that plain lists do not.
(SAMPLE_SHEET_MAPPING)=
(SAMPLE_SHEET_MATCHES_LIST)=
<details><summary>Examples</summary>
:::{admonition} Example: `SAMPLE_SHEET_MAPPING`
:class: note
Assuming,
* $ d_1,...,d_n $ are datasets
* $ tool \text{ is } (i: \text{ dataset }) \Rightarrow \{ o: \text{ dataset } \} $
* $ C $ is $ \text{CollectionInstance<}\text{sample\_sheet},\left\{ \text{ i1 }=d_1, ..., \text{ in }=d_n \right\}\text{>} $
then
$$tool(i=\text{mapOver}(C)) \mapsto \left\{o: \text{collection}<\text{sample\_sheet},\left\{\text{i1}=tool(i=d_1)[o],...,\text{in}=tool(i=d_n)[o]\right\}>\right\}$$
:::
:::{admonition} Example: `SAMPLE_SHEET_MATCHES_LIST`
:class: note
Assuming,
* $ d_1,...,d_n $ are datasets
* $ tool \text{ is } (i: \text{ collection<list> }) \Rightarrow \{ o: \text{ dataset } \} $
* $ C $ is $ \text{CollectionInstance<}\text{sample\_sheet},\left\{ \text{ i1 }=d_1, ..., \text{ in }=d_n \right\}\text{>} $
then
$$tool(i=C) \rightarrow \left\{o: \text{dataset}\right\}$$
:::
</details><br>
Sub-collection mapping works the same as for ``list`` composites. A
``sample_sheet:paired`` can be mapped over a ``paired`` collection input,
extracting each inner pair and producing a ``sample_sheet`` implicit output.
(SAMPLE_SHEET_PAIRED_MAPPING_OVER_PAIRED)=
(SAMPLE_SHEET_PAIRED_MATCHES_LIST_PAIRED)=
<details><summary>Examples</summary>
:::{admonition} Example: `SAMPLE_SHEET_PAIRED_MAPPING_OVER_PAIRED`
:class: note
Assuming,
* $ d_f $, $ d_r $ are datasets
* $ tool \text{ is } (i: \text{ collection<paired> }) \Rightarrow \{ o: \text{ dataset } \} $
* $ C $ is $ \text{CollectionInstance<}\text{sample\_sheet}:\text{paired},\left\{ \text{ el1 }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\} \right\}\text{>} $
* $ C\_PAIRED $ is $ \text{CollectionInstance<}\text{paired},\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\}\text{>} $
then
$$tool(i=\text{mapOver}(C, '\text{paired}')) \mapsto \left\{o: \text{collection}<\text{sample\_sheet},\left\{\text{el1}=tool(i=C\_PAIRED)[o]\right\}>\right\}$$
:::
:::{admonition} Example: `SAMPLE_SHEET_PAIRED_MATCHES_LIST_PAIRED`
:class: note
Assuming,
* $ d_f $, $ d_r $ are datasets
* $ tool \text{ is } (i: \text{ collection<list:paired> }) \Rightarrow \{ o: \text{ dataset } \} $
* $ C $ is $ \text{CollectionInstance<}\text{sample\_sheet}:\text{paired},\left\{ \text{ el1 }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\} \right\}\text{>} $
then
$$tool(i=C) \rightarrow \left\{o: \text{dataset}\right\}$$
:::
</details><br>
The ``paired_or_unpaired`` integration rules carry over from ``list`` to
``sample_sheet``. A flat ``sample_sheet`` can be mapped over a
``paired_or_unpaired`` input via ``single_datasets`` sub-collection mapping,
and ``sample_sheet:paired`` can be mapped over ``paired_or_unpaired`` just as
``list:paired`` can.
(SAMPLE_SHEET_MAPPING_OVER_PAIRED_OR_UNPAIRED)=
(SAMPLE_SHEET_PAIRED_MAPPING_OVER_PAIRED_OR_UNPAIRED)=
(SAMPLE_SHEET_PAIRED_OR_UNPAIRED_MATCHES_LIST_PAIRED_OR_UNPAIRED)=
<details><summary>Examples</summary>
:::{admonition} Example: `SAMPLE_SHEET_MAPPING_OVER_PAIRED_OR_UNPAIRED`
:class: note
Assuming,
* $ d_1,...,d_n $ are datasets
* $ tool \text{ is } (i: \text{ collection<paired\_or\_unpaired> }) \Rightarrow \{ o: \text{ dataset } \} $
* $ C $ is $ \text{CollectionInstance<}\text{sample\_sheet},\left\{ \text{ i1 }=d_1, ..., \text{ in }=d_n \right\}\text{>} $
* $ C_AS_UNPAIRED_i = CollectionInstance<\text{paired\_or\_unpaired},\left\{\text{unpaired}=di\right\}> for i from 1...n $
then
$$tool(i=\text{mapOver}(C, '\text{single\_datasets}')) \mapsto \left\{o: \text{collection}<\text{sample\_sheet},\left\{\text{i1}=tool(i=C_AS_UNPAIRED_1)[o],...,\text{in}=tool(i=C_AS_UNPAIRED_n)[o]\right\}>\right\}$$
:::
:::{admonition} Example: `SAMPLE_SHEET_PAIRED_MAPPING_OVER_PAIRED_OR_UNPAIRED`
:class: note
Assuming,
* $ d_f $, $ d_r $ are datasets
* $ tool \text{ is } (i: \text{ collection<paired\_or\_unpaired> }) \Rightarrow \{ o: \text{ dataset } \} $
* $ C $ is $ \text{CollectionInstance<}\text{sample\_sheet}:\text{paired},\left\{ \text{ el1 }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\} \right\}\text{>} $
* $ C_AS_MIXED $ is $ \text{CollectionInstance<}\text{sample\_sheet}:\text{paired\_or\_unpaired},\left\{ \text{ el1 }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\} \right\}\text{>} $
then
$$tool(i=\text{mapOver}(C)) == tool(i=\text{mapOver}(C_AS_MIXED))$$
:::
:::{admonition} Example: `SAMPLE_SHEET_PAIRED_OR_UNPAIRED_MATCHES_LIST_PAIRED_OR_UNPAIRED`
:class: note
Assuming,
* $ d_f $, $ d_r $ are datasets
* $ tool \text{ is } (i: \text{ collection<list:paired\_or\_unpaired> }) \Rightarrow \{ o: \text{ dataset } \} $
* $ C $ is $ \text{CollectionInstance<}\text{sample\_sheet}:\text{paired\_or\_unpaired},\left\{ \text{ el1 }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\} \right\}\text{>} $
then
$$tool(i=C) \rightarrow \left\{o: \text{dataset}\right\}$$
:::
</details><br>
The type matching is asymmetric: a ``sample_sheet`` output can satisfy a ``list``
input because sample sheets carry all the structural information lists have (plus
metadata). However, a ``list`` output cannot satisfy a ``sample_sheet`` input because
lists lack the ``column_definitions`` and per-element ``columns`` metadata that
sample sheet consumers expect.
(LIST_NOT_MATCHES_SAMPLE_SHEET)=
(LIST_PAIRED_NOT_MATCHES_SAMPLE_SHEET_PAIRED)=
(SAMPLE_SHEET_MATCHES_SAMPLE_SHEET)=
<details><summary>Examples</summary>
:::{admonition} Example: `LIST_NOT_MATCHES_SAMPLE_SHEET`
:class: note
Assuming,
* $ d_1,...,d_n $ are datasets
* $ tool \text{ is } (i: \text{ collection<sample\_sheet> }) \Rightarrow \{ o: \text{ dataset } \} $
* $ C $ is $ \text{CollectionInstance<}\text{list},\left\{ \text{ i1 }=d_1, ..., \text{ in }=d_n \right\}\text{>} $
then
$$tool(i=C)\text{ is invalid}$$
:::
:::{admonition} Example: `LIST_PAIRED_NOT_MATCHES_SAMPLE_SHEET_PAIRED`
:class: note
Assuming,
* $ d_f $, $ d_r $ are datasets
* $ tool \text{ is } (i: \text{ collection<sample\_sheet:paired> }) \Rightarrow \{ o: \text{ dataset } \} $
* $ C $ is $ \text{CollectionInstance<}\text{list}:\text{paired},\left\{ \text{ el1 }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\} \right\}\text{>} $
then
$$tool(i=C)\text{ is invalid}$$
:::
:::{admonition} Example: `SAMPLE_SHEET_MATCHES_SAMPLE_SHEET`
:class: note
Assuming,
* $ d_1,...,d_n $ are datasets
* $ tool \text{ is } (i: \text{ collection<sample\_sheet> }) \Rightarrow \{ o: \text{ dataset } \} $
* $ C $ is $ \text{CollectionInstance<}\text{sample\_sheet},\left\{ \text{ i1 }=d_1, ..., \text{ in }=d_n \right\}\text{>} $
then
$$tool(i=C) \rightarrow \left\{o: \text{dataset}\right\}$$
:::
</details><br>
lib/galaxy/model/dataset_collections/type_description.py
+40 -1
View File
@@ -71,6 +71,34 @@ class CollectionTypeDescription:
if subcollection_type == "single_datasets":
return self.collection_type
normalized = _normalize_collection_type(self.collection_type)
normalized_sub = _normalize_collection_type(subcollection_type)
if subcollection_type == "paired_or_unpaired":
if self.collection_type.endswith(":paired"):
# paired_or_unpaired consumes the :paired suffix
return self.collection_type[: -len(":paired")]
elif normalized.endswith("list"):
# paired_or_unpaired acts like single_datasets for collections
# whose innermost type is list (each element wrapped as unpaired)
return self.collection_type
else:
# strip last rank (paired_or_unpaired consumes it)
return self.collection_type[: self.collection_type.rfind(":")]
if normalized_sub.endswith(":paired_or_unpaired"):
# Compound :paired_or_unpaired suffix — iterative peel-off matching
# TS effectiveMapOver logic. Strip ranks from both sides, then
# optionally strip one more if :paired was consumed.
current = self.collection_type
current_other = subcollection_type
while ":" in current_other:
current_other = current_other[: current_other.rfind(":")]
current = current[: current.rfind(":")]
if normalized.endswith(":paired"):
current = current[: current.rfind(":")]
return current
return self.collection_type[: -(len(subcollection_type) + 1)]
def has_subcollections_of_type(self, other_collection_type) -> bool:
@@ -88,12 +116,23 @@ class CollectionTypeDescription:
other_collection_type = _normalize_collection_type(other_collection_type)
if collection_type == other_collection_type:
return False
if collection_type.endswith(other_collection_type):
if collection_type.endswith(f":{other_collection_type}"):
return True
if other_collection_type == "paired_or_unpaired":
# this can be thought of as a subcollection of anything except a pair
# since it would match a pair exactly
return collection_type != "paired"
if other_collection_type.endswith(":paired_or_unpaired"):
# Compound :paired_or_unpaired suffix — e.g. list:list can map over
# list:paired_or_unpaired. Strip the :paired_or_unpaired to get the
# required higher ranks, optionally strip :paired from self (since
# paired_or_unpaired consumes paired), then check alignment.
higher_ranks_required = other_collection_type[: other_collection_type.rfind(":")]
if collection_type.endswith(":paired"):
higher_ranks = collection_type[: collection_type.rfind(":")]
else:
higher_ranks = collection_type
return higher_ranks.endswith(higher_ranks_required) and higher_ranks != higher_ranks_required
if other_collection_type == "single_datasets":
# effectively any collection has unpaired subcollections
return True
lib/galaxy/model/dataset_collections/types/collection_semantics.yml
+740 -53
View File
File diff suppressed because it is too large Load Diff
lib/galaxy/model/dataset_collections/types/semantics.py
+343 -15
View File
@@ -3,11 +3,15 @@
# how to use this function...
# PYTHONPATH=lib python lib/galaxy/model/dataset_collections/types/semantics.py
import argparse
import ast
import os
import re
import sys
from io import StringIO
from typing import (
Annotated,
Any,
Literal,
NamedTuple,
Optional,
Union,
@@ -16,6 +20,7 @@ from typing import (
import yaml
from pydantic import (
BaseModel,
ConfigDict,
Field,
RootModel,
)
@@ -42,8 +47,16 @@ class ToolRuntimeFramework(BaseModel):
ToolRuntimeTest = Union[ToolRuntimeApi, ToolRuntimeFramework]
class WorkflowRuntimeTest(BaseModel):
api_test: Optional[str] = None
framework_test: Optional[str] = None
class ExampleTests(BaseModel):
model_config = ConfigDict(extra="forbid")
tool_runtime: Optional[ToolRuntimeTest] = None
workflow_runtime: Optional[WorkflowRuntimeTest] = None
workflow_editor: Optional[str] = None
@@ -59,8 +72,8 @@ class ToolDefinition(BaseModel):
outputs: dict[str, str] = Field(alias="out")
def as_latex(self) -> str:
inputs = ", ".join([f"{k}: \\text{{ {v} }}" for (k, v) in self.inputs.items()])
outputs = ", ".join([f"{k}: \\text{{ {v} }}" for (k, v) in self.outputs.items()])
inputs = ", ".join([f"{k}: \\text{{ {_escape_underscores(v)} }}" for (k, v) in self.inputs.items()])
outputs = ", ".join([f"{k}: \\text{{ {_escape_underscores(v)} }}" for (k, v) in self.outputs.items()])
return f"tool \\text{{ is }} ({inputs}) \\Rightarrow \\{{ {outputs} \\}}"
@@ -71,14 +84,14 @@ class ToolDeclaration(BaseModel):
return self.tool.as_latex()
def elements_to_latex(elements: dict[str, Any]):
def _assumption_elements_to_latex(elements: dict[str, Any]):
elements_as_strings = []
for identifier, value in elements.items():
if value is None:
elements_as_strings.append(identifier)
else:
if isinstance(value, dict):
value_str = elements_to_latex(value)
value_str = _assumption_elements_to_latex(value)
else:
value_str = str(value)
elements_as_strings.append(f"\\text{{ {identifier} }}={value_str}")
@@ -91,8 +104,8 @@ class CollectionDefinition(NamedTuple):
elements: dict[str, Any]
def as_latex(self) -> str:
collection_type = self.collection_type.replace("_", "\\_")
return f"\\text{{CollectionInstance<}}{collection_type},{elements_to_latex(self.elements)}\\text{{>}}"
ct = _latex_collection_type(self.collection_type)
return f"\\text{{CollectionInstance<}}{ct},{_assumption_elements_to_latex(self.elements)}\\text{{>}}"
class CollectionDeclarations(BaseModel):
@@ -102,10 +115,192 @@ class CollectionDeclarations(BaseModel):
Expression = Union[str, DatasetsDeclaration, ToolDeclaration, CollectionDeclarations]
# --- Structured Then Expression Models ---
def _escape_underscores(text: str) -> str:
return text.replace("_", "\\_")
def _latex_type_word(word: str) -> str:
if "_" in word:
escaped = word.replace("_", "\\_")
return "\\text{" + escaped + "}"
return "\\text{" + word + "}"
def _latex_collection_type(ct: str) -> str:
return ":".join(_latex_type_word(p) for p in ct.split(":"))
def _output_elements_to_latex(elements: dict) -> str:
parts = []
for key, value in elements.items():
if key == "...":
parts.append("...")
else:
parts.append(f"\\text{{{key}}}={value.as_latex()}")
# Join without spaces around ellipsis entries
result = ""
for i, part in enumerate(parts):
if i > 0:
if part == "..." or parts[i - 1] == "...":
result += ","
else:
result += ", "
result += part
return "\\left\\{" + result + "\\right\\}"
def _produces_to_latex(produces: dict) -> str:
parts = []
for key, value in produces.items():
parts.append(f"{key}: {value.as_latex()}")
return "\\left\\{" + ", ".join(parts) + "\\right\\}"
class DatasetInput(BaseModel):
type: Literal["dataset"]
ref: str
def as_latex(self) -> str:
return self.ref
class MapOverInput(BaseModel):
type: Literal["map_over"]
collection: str
sub_collection_type: Optional[str] = None
def as_latex(self) -> str:
if self.sub_collection_type:
sct = _latex_collection_type(self.sub_collection_type)
return f"\\text{{mapOver}}({self.collection}, '{sct}')"
return f"\\text{{mapOver}}({self.collection})"
class CollectionInput(BaseModel):
type: Literal["collection"]
ref: str
def as_latex(self) -> str:
return self.ref
class DatasetListInput(BaseModel):
type: Literal["dataset_list"]
refs: list[str]
def as_latex(self) -> str:
return "[" + ",".join(self.refs) + "]"
InputBinding = Annotated[
Union[DatasetInput, MapOverInput, CollectionInput, DatasetListInput], Field(discriminator="type")
]
class ToolInvocation(BaseModel):
inputs: dict[str, InputBinding]
def as_latex(self) -> str:
args = ", ".join(f"{k}={v.as_latex()}" for k, v in self.inputs.items())
return f"tool({args})"
class DatasetOutput(BaseModel):
type: Literal["dataset"]
def as_latex(self) -> str:
return "\\text{dataset}"
class EllipsisMarker(BaseModel):
type: Literal["ellipsis"]
def as_latex(self) -> str:
return "..."
class ToolOutputRef(BaseModel):
type: Literal["tool_output_ref"]
invocation: ToolInvocation
output: str
def as_latex(self) -> str:
return f"{self.invocation.as_latex()}[{self.output}]"
class NestedElements(BaseModel):
type: Literal["nested_elements"]
elements: dict[str, "OutputBinding"]
def as_latex(self) -> str:
return _output_elements_to_latex(self.elements)
OutputBinding = Annotated[
Union[DatasetOutput, EllipsisMarker, ToolOutputRef, NestedElements], Field(discriminator="type")
]
NestedElements.model_rebuild()
class CollectionOutput(BaseModel):
type: Literal["collection"]
collection_type: str
elements: dict[str, OutputBinding]
def as_latex(self) -> str:
ct = _latex_collection_type(self.collection_type)
el = _output_elements_to_latex(self.elements)
return f"\\text{{collection}}<{ct},{el}>"
OutputSpec = Annotated[Union[DatasetOutput, CollectionOutput], Field(discriminator="type")]
class MapOverThen(BaseModel):
type: Literal["map_over"]
invocation: ToolInvocation
produces: dict[str, OutputSpec]
def as_latex(self) -> str:
return f"{self.invocation.as_latex()} \\mapsto {_produces_to_latex(self.produces)}"
class ReductionThen(BaseModel):
type: Literal["reduction"]
invocation: ToolInvocation
produces: dict[str, OutputSpec]
def as_latex(self) -> str:
return f"{self.invocation.as_latex()} \\rightarrow {_produces_to_latex(self.produces)}"
class EquivalenceThen(BaseModel):
type: Literal["equivalence"]
left: ToolInvocation
right: ToolInvocation
def as_latex(self) -> str:
return f"{self.left.as_latex()} == {self.right.as_latex()}"
class InvalidThen(BaseModel):
type: Literal["invalid"]
invocation: ToolInvocation
def as_latex(self) -> str:
return self.invocation.as_latex()
ThenExpression = Annotated[Union[MapOverThen, ReductionThen, EquivalenceThen, InvalidThen], Field(discriminator="type")]
class Example(BaseModel):
label: str
assumptions: Optional[list[Expression]] = None
then: Optional[str] = None
then: Optional[ThenExpression] = None
is_valid: bool = True
tests: Optional[ExampleTests] = None
@@ -117,7 +312,11 @@ class ExampleEntry(BaseModel):
YAMLRootModel = RootModel[list[Union[DocEntry, ExampleEntry]]]
WORDS_TO_TEXTIFY = ["list", "forward", "reverse", "mapOver"]
WORDS_TO_TEXTIFY = ["list", "forward", "reverse", "mapOver", "collection", "dataset", "inner"]
_POU_PLACEHOLDER = "\x00POU\x00"
_UNPAIRED_PLACEHOLDER = "\x00UNPAIRED\x00"
_SINGLE_DATASETS_PLACEHOLDER = "\x00SINGLEDATASETS\x00"
_SAMPLE_SHEET_PLACEHOLDER = "\x00SAMPLESHEET\x00"
def expression_to_latex(expression: str, wrap: bool = True):
@@ -125,9 +324,19 @@ def expression_to_latex(expression: str, wrap: bool = True):
expression = expression.replace("~>", "\\mapsto")
expression = expression.replace("{", "\\left\\{")
expression = expression.replace("}", "\\right\\}")
expression = expression.replace("paired_or_unpaired", "paired\\_or\\_unpaired")
# Replace multi-word identifiers with placeholders before WORDS_TO_TEXTIFY
# to prevent double-replacement (e.g. "dataset" inside "single_datasets")
expression = expression.replace("single_datasets", _SINGLE_DATASETS_PLACEHOLDER)
expression = expression.replace("sample_sheet", _SAMPLE_SHEET_PLACEHOLDER)
expression = expression.replace("paired_or_unpaired", _POU_PLACEHOLDER)
expression = expression.replace("unpaired", _UNPAIRED_PLACEHOLDER)
expression = expression.replace("paired", "\\text{paired}")
expression = expression.replace(_UNPAIRED_PLACEHOLDER, "\\text{unpaired}")
expression = expression.replace(_POU_PLACEHOLDER, "\\text{paired\\_or\\_unpaired}")
for word in WORDS_TO_TEXTIFY:
expression = expression.replace(word, "\\text{" + word + "}")
expression = expression.replace(_SINGLE_DATASETS_PLACEHOLDER, "\\text{single\\_datasets}")
expression = expression.replace(_SAMPLE_SHEET_PLACEHOLDER, "\\text{sample\\_sheet}")
if wrap:
return f"$ {expression} $"
else:
@@ -165,9 +374,129 @@ def main(argv=None) -> None:
generate_docs()
def check():
# todo
pass
def _load_examples() -> list["Example"]:
semantics_yaml = yaml.safe_load(
resource_string("galaxy.model.dataset_collections.types", "collection_semantics.yml")
)
root = YAMLRootModel.model_validate(semantics_yaml)
return [e.example for e in root.root if isinstance(e, ExampleEntry)]
def check() -> list[str]:
examples = _load_examples()
errors: list[str] = []
errors.extend(validate_api_test_refs(examples))
errors.extend(validate_tool_refs(examples))
errors.extend(validate_workflow_editor_refs(examples))
return errors
def _validate_api_test_ref(label: str, ref: str, api_test_dir: str) -> Optional[str]:
parts = ref.split("::")
filename = parts[0]
filepath = os.path.join(api_test_dir, filename)
if not os.path.exists(filepath):
return f"[{label}] api_test file not found: {filename}"
with open(filepath) as f:
tree = ast.parse(f.read(), filename=filepath)
if len(parts) == 2:
func_name = parts[1]
found = any(
isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) and node.name == func_name
for node in ast.walk(tree)
)
if not found:
return f"[{label}] api_test function not found: {ref}"
elif len(parts) == 3:
class_name, method_name = parts[1], parts[2]
found = False
for node in ast.walk(tree):
if isinstance(node, ast.ClassDef) and node.name == class_name:
found = any(
isinstance(child, (ast.FunctionDef, ast.AsyncFunctionDef)) and child.name == method_name
for child in node.body
)
break
if not found:
return f"[{label}] api_test method not found: {ref}"
return None
def _validate_framework_test_ref(label: str, ref: str, workflow_dir: str) -> Optional[str]:
"""Validate a framework test ref like 'collection_semantics_cat_0'."""
parts = ref.rsplit("_", 1)
if len(parts) != 2 or not parts[1].isdigit():
return f"[{label}] framework_test must be 'workflow_name_index': {ref}"
workflow_name = parts[0]
workflow_file = os.path.join(workflow_dir, f"{workflow_name}.gxwf.yml")
if not os.path.exists(workflow_file):
return f"[{label}] framework_test workflow not found: {workflow_name}.gxwf.yml"
test_file = os.path.join(workflow_dir, f"{workflow_name}.gxwf-tests.yml")
if not os.path.exists(test_file):
return f"[{label}] framework_test test file not found: {workflow_name}.gxwf-tests.yml"
return None
def validate_api_test_refs(examples: list["Example"]) -> list[str]:
errors: list[str] = []
api_test_dir = os.path.join(galaxy_directory(), "lib", "galaxy_test", "api")
workflow_dir = os.path.join(galaxy_directory(), "lib", "galaxy_test", "workflow")
for ex in examples:
if ex.tests and ex.tests.tool_runtime and isinstance(ex.tests.tool_runtime, ToolRuntimeApi):
error = _validate_api_test_ref(ex.label, ex.tests.tool_runtime.api_test, api_test_dir)
if error:
errors.append(error)
if ex.tests and ex.tests.workflow_runtime:
if ex.tests.workflow_runtime.api_test:
error = _validate_api_test_ref(ex.label, ex.tests.workflow_runtime.api_test, api_test_dir)
if error:
errors.append(error)
if ex.tests.workflow_runtime.framework_test:
error = _validate_framework_test_ref(ex.label, ex.tests.workflow_runtime.framework_test, workflow_dir)
if error:
errors.append(error)
return errors
def validate_tool_refs(examples: list["Example"]) -> list[str]:
errors: list[str] = []
tools_dir = os.path.join(galaxy_directory(), "test", "functional", "tools")
for ex in examples:
if not ex.tests or not ex.tests.tool_runtime:
continue
if not isinstance(ex.tests.tool_runtime, ToolRuntimeFramework):
continue
tool_id = ex.tests.tool_runtime.tool
tool_path = os.path.join(tools_dir, f"{tool_id}.xml")
if not os.path.exists(tool_path):
errors.append(f"[{ex.label}] framework tool XML not found: {tool_id}.xml")
return errors
def validate_workflow_editor_refs(examples: list["Example"]) -> list[str]:
errors: list[str] = []
test_file = os.path.join(
galaxy_directory(),
"client",
"src",
"components",
"Workflow",
"Editor",
"modules",
"terminals.test.ts",
)
if not os.path.exists(test_file):
return [f"workflow editor test file not found: {test_file}"]
with open(test_file) as f:
content = f.read()
it_descriptions = set(re.findall(r'it\(["\'](.+?)["\']\s*,', content))
for ex in examples:
if not ex.tests or not ex.tests.workflow_editor:
continue
desc = ex.tests.workflow_editor
if desc not in it_descriptions:
errors.append(f'[{ex.label}] workflow_editor test not found: "{desc}"')
return errors
def generate_docs():
@@ -215,9 +544,8 @@ def generate_docs():
validity_str = ""
if not example.is_valid:
validity_str = "\\text{ is invalid}"
markdown_content.write(
f"\n\nthen\n\n$${expression_to_latex(example.then, wrap=False)}{validity_str}$$\n\n"
)
then_latex = example.then.as_latex()
markdown_content.write(f"\n\nthen\n\n$${then_latex}{validity_str}$$\n\n")
markdown_content.write(":::\n\n")
markdown_content.write("\n\n</details><br>\n\n")
markdown_content.write("\n\n")
lib/galaxy/tool_util/cwl/util.py
+1 -1
View File
@@ -365,7 +365,7 @@ def galactic_job_json(
elements = to_elements(value, collection_type)
kwds = {}
if collection_type.startswith("sample_sheet"):
kwds["rows"] = value["rows"]
kwds["rows"] = value.get("rows")
if "name" in value:
kwds["name"] = value["name"]
collection = collection_create_func(elements, collection_type, **kwds)
lib/galaxy_test/api/test_tool_execute.py
+25
View File
@@ -571,6 +571,31 @@ def test_map_over_paired_or_unpaired_with_list_of_lists(target_history: TargetHi
assert len(as_dict_0["object"]["elements"]) == 3
@requires_tool_id("collection_list_paired_or_unpaired")
def test_map_over_list_paired_or_unpaired_with_list_of_lists(
target_history: TargetHistory, required_tool: RequiredTool
):
hdca = target_history.with_example_list_of_lists()
execute = required_tool.execute().with_inputs(
{"f1": {"batch": True, "values": [{"map_over_type": "list", **hdca.src_dict}]}}
)
execute.assert_has_n_jobs(1).assert_creates_n_implicit_collections(1)
output_collection = execute.assert_creates_implicit_collection(0)
assert output_collection.details["collection_type"] == "list"
@requires_tool_id("collection_paired_test")
def test_paired_or_unpaired_not_consumed_by_paired_when_mapping(
target_history: TargetHistory, required_tool: RequiredTool
):
"""PAIRED_OR_UNPAIRED_NOT_CONSUMED_BY_PAIRED_WHEN_MAPPING:
list:paired_or_unpaired cannot be mapped over a paired tool input."""
hdca = target_history.with_list_of_paired_and_unpaired()
required_tool.execute().with_inputs(
{"f1": {"batch": True, "values": [{"map_over_type": "paired", **hdca.src_dict}]}}
).assert_fails.with_error_containing("Cannot split collection")
@requires_tool_id("collection_paired_test")
def test_simple_subcollection_mapping(
target_history: TargetHistory, required_tool: RequiredTool, tool_input_format: DescribeToolInputs
lib/galaxy_test/base/populators.py
+13
View File
@@ -4062,6 +4062,10 @@ def load_data_dict(
fetch_response = dataset_collection_populator.create_list_of_pairs_in_history(
history_id, contents=elements, wait=True, **new_collection_kwds
).json()
elif collection_type == "list:paired_or_unpaired":
fetch_response = dataset_collection_populator.create_list_of_paired_and_unpaired_in_history(
history_id, wait=True, **new_collection_kwds
).json()
elif collection_type and ":" in collection_type:
fetch_response = {
"outputs": [
@@ -4074,6 +4078,15 @@ def load_data_dict(
fetch_response = dataset_collection_populator.create_list_in_history(
history_id, contents=elements, direct_upload=True, wait=True, **new_collection_kwds
).json()
elif collection_type == "paired_or_unpaired":
if elements:
fetch_response = dataset_collection_populator.upload_collection(
history_id, "paired_or_unpaired", elements=elements, wait=True, **new_collection_kwds
).json()
else:
fetch_response = dataset_collection_populator.create_paired_or_unpaired_pair_in_history(
history_id, wait=True, **new_collection_kwds
).json()
else:
fetch_response = dataset_collection_populator.create_pair_in_history(
history_id, contents=elements or None, direct_upload=True, wait=True, **new_collection_kwds
lib/galaxy_test/workflow/collection_semantics_cat.gxwf-tests.yml
+141
View File
@@ -0,0 +1,141 @@
- doc: |
BASIC_MAPPING_PAIRED: A paired collection mapped over a data input produces
a paired output collection.
job:
input1:
class: Collection
collection_type: paired
elements:
- identifier: forward
class: File
contents: "forward content"
- identifier: reverse
class: File
contents: "reverse content"
outputs:
wf_output:
class: Collection
collection_type: paired
elements:
forward:
asserts:
- that: has_text
text: "forward content"
reverse:
asserts:
- that: has_text
text: "reverse content"
- doc: |
BASIC_MAPPING_PAIRED_OR_UNPAIRED_PAIRED: A paired_or_unpaired collection (paired variant)
mapped over a data input produces a paired_or_unpaired output collection with 2 elements.
job:
input1:
class: Collection
collection_type: paired_or_unpaired
elements:
- identifier: forward
class: File
contents: "forward content"
- identifier: reverse
class: File
contents: "reverse content"
outputs:
wf_output:
class: Collection
collection_type: paired_or_unpaired
elements:
forward:
asserts:
- that: has_text
text: "forward content"
reverse:
asserts:
- that: has_text
text: "reverse content"
- doc: |
BASIC_MAPPING_PAIRED_OR_UNPAIRED_UNPAIRED: A paired_or_unpaired collection (unpaired variant)
mapped over a data input produces a paired_or_unpaired output collection with 1 element.
job:
input1:
class: Collection
collection_type: paired_or_unpaired
elements:
- identifier: unpaired
class: File
contents: "unpaired content"
outputs:
wf_output:
class: Collection
collection_type: paired_or_unpaired
elements:
unpaired:
asserts:
- that: has_text
text: "unpaired content"
- doc: |
BASIC_MAPPING_LIST_PAIRED_OR_UNPAIRED: A list:paired_or_unpaired collection mapped over
a data input produces a list:paired_or_unpaired output collection.
job:
input1:
class: Collection
collection_type: list:paired_or_unpaired
elements:
- identifier: el1
class: Collection
type: paired_or_unpaired
elements:
- identifier: forward
class: File
contents: "forward content"
- identifier: reverse
class: File
contents: "reverse content"
outputs:
wf_output:
class: Collection
collection_type: list:paired_or_unpaired
elements:
el1:
elements:
forward:
asserts:
- that: has_text
text: "forward content"
reverse:
asserts:
- that: has_text
text: "reverse content"
- doc: |
SAMPLE_SHEET_MAPPING: A sample_sheet mapped over a data input produces
a sample_sheet output collection.
job:
input1:
class: Collection
collection_type: sample_sheet
rows:
el1: []
el2: []
elements:
- identifier: el1
class: File
contents: "element 1"
- identifier: el2
class: File
contents: "element 2"
outputs:
wf_output:
class: Collection
collection_type: sample_sheet
elements:
el1:
asserts:
- that: has_text
text: "element 1"
el2:
asserts:
- that: has_text
text: "element 2"
lib/galaxy_test/workflow/collection_semantics_cat.gxwf.yml
+11
View File
@@ -0,0 +1,11 @@
class: GalaxyWorkflow
inputs:
input1: data
outputs:
wf_output:
outputSource: cat_step/out_file1
steps:
cat_step:
tool_id: cat
in:
input1: input1
lib/galaxy_test/workflow/collection_semantics_cat_collection.gxwf-tests.yml
+46
View File
@@ -0,0 +1,46 @@
- doc: |
COLLECTION_INPUT_LIST: A list collection consumed by a tool expecting a list
collection input produces a single dataset.
job:
input1:
class: Collection
collection_type: list
elements:
- identifier: el1
class: File
contents: "element 1"
- identifier: el2
class: File
contents: "element 2"
outputs:
wf_output:
asserts:
- that: has_text
text: "element 1"
- that: has_text
text: "element 2"
- doc: |
SAMPLE_SHEET_MATCHES_LIST: A sample_sheet collection consumed by a list
collection input via canMatch (reduction).
job:
input1:
class: Collection
collection_type: sample_sheet
rows:
el1: []
el2: []
elements:
- identifier: el1
class: File
contents: "element 1"
- identifier: el2
class: File
contents: "element 2"
outputs:
wf_output:
asserts:
- that: has_text
text: "element 1"
- that: has_text
text: "element 2"
lib/galaxy_test/workflow/collection_semantics_cat_collection.gxwf.yml
+13
View File
@@ -0,0 +1,13 @@
class: GalaxyWorkflow
inputs:
input1:
type: collection
collection_type: list
outputs:
wf_output:
outputSource: tool_step/out_file1
steps:
tool_step:
tool_id: cat_collection
in:
input1: input1
lib/galaxy_test/workflow/collection_semantics_cat_sample_sheet.gxwf-tests.yml
+24
View File
@@ -0,0 +1,24 @@
- doc: |
SAMPLE_SHEET_MATCHES_SAMPLE_SHEET: A sample_sheet consumed by a sample_sheet
collection input (identity match, reduction).
job:
input1:
class: Collection
collection_type: sample_sheet
rows:
el1: []
el2: []
elements:
- identifier: el1
class: File
contents: "element 1"
- identifier: el2
class: File
contents: "element 2"
outputs:
wf_output:
asserts:
- that: has_text
text: "el1"
- that: has_text
text: "el2"
lib/galaxy_test/workflow/collection_semantics_cat_sample_sheet.gxwf.yml
+13
View File
@@ -0,0 +1,13 @@
class: GalaxyWorkflow
inputs:
input1:
type: collection
collection_type: sample_sheet
outputs:
wf_output:
outputSource: tool_step/output
steps:
tool_step:
tool_id: __SAMPLE_SHEET_TO_TABULAR__
in:
input: input1
lib/galaxy_test/workflow/collection_semantics_cat_two_inputs.gxwf-tests.yml
+76
View File
@@ -0,0 +1,76 @@
- doc: |
BASIC_MAPPING_INCLUDING_SINGLE_DATASET: Mapping a list collection over a data input
while another input is a single dataset produces a list output collection.
job:
input1:
class: Collection
collection_type: list
elements:
- identifier: el1
class: File
contents: "list element 1"
- identifier: el2
class: File
contents: "list element 2"
input2:
class: File
contents: "single dataset content"
outputs:
wf_output:
class: Collection
collection_type: list
elements:
el1:
asserts:
- that: has_text
text: "list element 1"
- that: has_text
text: "single dataset content"
el2:
asserts:
- that: has_text
text: "list element 2"
- that: has_text
text: "single dataset content"
- doc: |
BASIC_MAPPING_TWO_INPUTS_WITH_IDENTICAL_STRUCTURE: Mapping two list collections with
identical structure over two data inputs produces a list output collection.
job:
input1:
class: Collection
collection_type: list
elements:
- identifier: el1
class: File
contents: "collection 1 element 1"
- identifier: el2
class: File
contents: "collection 1 element 2"
input2:
class: Collection
collection_type: list
elements:
- identifier: el1
class: File
contents: "collection 2 element 1"
- identifier: el2
class: File
contents: "collection 2 element 2"
outputs:
wf_output:
class: Collection
collection_type: list
elements:
el1:
asserts:
- that: has_text
text: "collection 1 element 1"
- that: has_text
text: "collection 2 element 1"
el2:
asserts:
- that: has_text
text: "collection 1 element 2"
- that: has_text
text: "collection 2 element 2"
lib/galaxy_test/workflow/collection_semantics_cat_two_inputs.gxwf.yml
+13
View File
@@ -0,0 +1,13 @@
class: GalaxyWorkflow
inputs:
input1: data
input2: data
outputs:
wf_output:
outputSource: cat_step/out_file1
steps:
cat_step:
tool_id: cat
in:
input1: input1
queries_0|input2: input2
lib/galaxy_test/workflow/collection_semantics_list_paired.gxwf-tests.yml
+27
View File
@@ -0,0 +1,27 @@
- doc: |
SAMPLE_SHEET_PAIRED_MATCHES_LIST_PAIRED: A sample_sheet:paired consumed by a
list:paired collection input via canMatch (reduction).
job:
input1:
class: Collection
collection_type: sample_sheet:paired
rows:
el1: []
elements:
- identifier: el1
class: Collection
type: paired
elements:
- identifier: forward
class: File
contents: "forward content"
- identifier: reverse
class: File
contents: "reverse content"
outputs:
wf_output:
asserts:
- that: has_text
text: "identifier is el1"
- that: has_text
text: "collection_type<paired>"
lib/galaxy_test/workflow/collection_semantics_list_paired.gxwf.yml
+13
View File
@@ -0,0 +1,13 @@
class: GalaxyWorkflow
inputs:
input1:
type: collection
collection_type: list:paired
outputs:
wf_output:
outputSource: tool_step/out1
steps:
tool_step:
tool_id: collection_paired_or_list_paired_input
in:
f1: input1
lib/galaxy_test/workflow/collection_semantics_list_paired_or_unpaired.gxwf-tests.yml
+84
View File
@@ -0,0 +1,84 @@
- doc: |
COLLECTION_INPUT_LIST_PAIRED_OR_UNPAIRED: A list:paired_or_unpaired collection
consumed by a tool expecting list:paired_or_unpaired produces a single dataset.
job:
input1:
class: Collection
collection_type: list:paired_or_unpaired
elements:
- identifier: el1
class: Collection
type: paired_or_unpaired
elements:
- identifier: forward
class: File
contents: "forward content"
- identifier: reverse
class: File
contents: "reverse content"
outputs:
wf_output:
asserts:
- that: has_text
text: "forward content"
- that: has_text
text: "reverse content"
- doc: |
SAMPLE_SHEET_PAIRED_OR_UNPAIRED_MATCHES_LIST_PAIRED_OR_UNPAIRED: A
sample_sheet:paired_or_unpaired consumed by list:paired_or_unpaired via canMatch.
job:
input1:
class: Collection
collection_type: sample_sheet:paired_or_unpaired
rows:
el1: []
elements:
- identifier: el1
class: Collection
type: paired_or_unpaired
elements:
- identifier: forward
class: File
contents: "forward content"
- identifier: reverse
class: File
contents: "reverse content"
outputs:
wf_output:
asserts:
- that: has_text
text: "forward content"
- that: has_text
text: "reverse content"
- doc: |
MAPPING_LIST_LIST_OVER_LIST_PAIRED_OR_UNPAIRED: A list:list collection mapped
over a list:paired_or_unpaired tool input produces a list output collection.
Each inner list is consumed as list:paired_or_unpaired via single_datasets.
job:
input1:
class: Collection
collection_type: list:list
elements:
- identifier: outer1
class: Collection
type: list
elements:
- identifier: el1
class: File
contents: "data from outer1 el1"
- identifier: el2
class: File
contents: "data from outer1 el2"
- identifier: outer2
class: Collection
type: list
elements:
- identifier: el1
class: File
contents: "data from outer2 el1"
outputs:
wf_output:
class: Collection
collection_type: list
lib/galaxy_test/workflow/collection_semantics_list_paired_or_unpaired.gxwf.yml
+13
View File
@@ -0,0 +1,13 @@
class: GalaxyWorkflow
inputs:
input1:
type: collection
collection_type: list:paired_or_unpaired
outputs:
wf_output:
outputSource: tool_step/out1
steps:
tool_step:
tool_id: collection_list_paired_or_unpaired
in:
f1: input1
lib/galaxy_test/workflow/collection_semantics_multi_data_optional.gxwf-tests.yml
+21
View File
@@ -0,0 +1,21 @@
- doc: |
LIST_REDUCTION: A list collection consumed by a multi-data input (reduction)
produces a single dataset containing all elements.
job:
input1:
class: Collection
collection_type: list
elements:
- identifier: el1
class: File
contents: "element 1"
- identifier: el2
class: File
contents: "element 2"
outputs:
wf_output:
asserts:
- that: has_text
text: "element 1"
- that: has_text
text: "element 2"
lib/galaxy_test/workflow/collection_semantics_multi_data_optional.gxwf.yml
+13
View File
@@ -0,0 +1,13 @@
class: GalaxyWorkflow
inputs:
input1:
type: collection
collection_type: list
outputs:
wf_output:
outputSource: tool_step/out1
steps:
tool_step:
tool_id: multi_data_optional
in:
input1: input1
lib/galaxy_test/workflow/collection_semantics_paired_or_unpaired.gxwf-tests.yml
+207
View File
@@ -0,0 +1,207 @@
- doc: |
COLLECTION_INPUT_PAIRED_OR_UNPAIRED: A paired_or_unpaired collection consumed
directly by a tool expecting paired_or_unpaired produces a single dataset.
job:
input1:
class: Collection
collection_type: paired_or_unpaired
elements:
- identifier: forward
class: File
contents: "forward content"
- identifier: reverse
class: File
contents: "reverse content"
outputs:
wf_output:
asserts:
- that: has_text
text: "forward content"
- that: has_text
text: "reverse content"
- doc: |
PAIRED_OR_UNPAIRED_CONSUMES_PAIRED: A paired collection can be consumed by a
paired_or_unpaired tool input (subtype compatibility).
job:
input1:
class: Collection
collection_type: paired
elements:
- identifier: forward
class: File
contents: "forward content"
- identifier: reverse
class: File
contents: "reverse content"
outputs:
wf_output:
asserts:
- that: has_text
text: "forward content"
- that: has_text
text: "reverse content"
- doc: |
MAPPING_LIST_PAIRED_OVER_PAIRED_OR_UNPAIRED: A list:paired collection mapped over
a paired_or_unpaired tool input produces a list output collection (subtype + mapping).
job:
input1:
class: Collection
collection_type: list:paired
elements:
- identifier: test_level_1
class: Collection
type: paired
elements:
- identifier: forward
class: File
contents: "TestData123"
- identifier: reverse
class: File
contents: "TestData123"
outputs:
wf_output:
class: Collection
collection_type: list
- doc: |
MAPPING_LIST_LIST_PAIRED_OVER_PAIRED_OR_UNPAIRED: A list:list:paired collection mapped
over a paired_or_unpaired tool input produces a list:list output collection.
job:
input1:
class: Collection
collection_type: list:list:paired
elements:
- identifier: test_level_2
class: Collection
type: list:paired
elements:
- identifier: test_level_1
class: Collection
type: paired
elements:
- identifier: forward
class: File
contents: "TestData123"
- identifier: reverse
class: File
contents: "TestData123"
outputs:
wf_output:
class: Collection
collection_type: list:list
- doc: |
MAPPING_LIST_PAIRED_OR_UNPAIRED_OVER_PAIRED_OR_UNPAIRED: A list:paired_or_unpaired
collection mapped over a paired_or_unpaired tool input produces a list output collection.
job:
input1:
class: Collection
collection_type: list:paired_or_unpaired
elements:
- identifier: test_level_1
class: Collection
type: paired_or_unpaired
elements:
- identifier: forward
class: File
contents: "TestData123"
- identifier: reverse
class: File
contents: "TestData123"
outputs:
wf_output:
class: Collection
collection_type: list
- doc: |
SAMPLE_SHEET_PAIRED_MAPPING_OVER_PAIRED: A sample_sheet:paired mapped over
a paired_or_unpaired tool input produces a sample_sheet output.
job:
input1:
class: Collection
collection_type: sample_sheet:paired
rows:
el1: []
elements:
- identifier: el1
class: Collection
type: paired
elements:
- identifier: forward
class: File
contents: "forward content"
- identifier: reverse
class: File
contents: "reverse content"
outputs:
wf_output:
class: Collection
collection_type: sample_sheet
- doc: |
SAMPLE_SHEET_PAIRED_MAPPING_OVER_PAIRED_OR_UNPAIRED: A sample_sheet:paired
mapped over paired_or_unpaired (equiv to sample_sheet:paired_or_unpaired mapping).
job:
input1:
class: Collection
collection_type: sample_sheet:paired
rows:
el1: []
elements:
- identifier: el1
class: Collection
type: paired
elements:
- identifier: forward
class: File
contents: "forward content"
- identifier: reverse
class: File
contents: "reverse content"
outputs:
wf_output:
class: Collection
collection_type: sample_sheet
- doc: |
MAPPING_LIST_OVER_PAIRED_OR_UNPAIRED: A flat list mapped over a
paired_or_unpaired tool input via single_datasets produces a list output.
job:
input1:
class: Collection
collection_type: list
elements:
- identifier: el1
class: File
contents: "element 1"
- identifier: el2
class: File
contents: "element 2"
outputs:
wf_output:
class: Collection
collection_type: list
- doc: |
SAMPLE_SHEET_MAPPING_OVER_PAIRED_OR_UNPAIRED: A flat sample_sheet mapped over
a paired_or_unpaired tool input via single_datasets produces a sample_sheet output.
job:
input1:
class: Collection
collection_type: sample_sheet
rows:
el1: []
el2: []
elements:
- identifier: el1
class: File
contents: "element 1"
- identifier: el2
class: File
contents: "element 2"
outputs:
wf_output:
class: Collection
collection_type: sample_sheet
lib/galaxy_test/workflow/collection_semantics_paired_or_unpaired.gxwf.yml
+13
View File
@@ -0,0 +1,13 @@
class: GalaxyWorkflow
inputs:
input1:
type: collection
collection_type: paired_or_unpaired
outputs:
wf_output:
outputSource: tool_step/out1
steps:
tool_step:
tool_id: collection_paired_or_unpaired
in:
f1: input1
test/unit/data/dataset_collections/test_type_descriptions.py
+85
View File
@@ -86,6 +86,91 @@ def test_sample_sheet_acts_like_list():
assert sample_sheet_paired.effective_collection_type(paired_type) == "sample_sheet"
def test_effective_collection_type_paired_or_unpaired_over_paired():
# list:paired mapped over paired_or_unpaired should yield "list"
assert c_t("list:paired").effective_collection_type("paired_or_unpaired") == "list"
# list:list:paired mapped over paired_or_unpaired should yield "list:list"
assert c_t("list:list:paired").effective_collection_type("paired_or_unpaired") == "list:list"
# list:paired_or_unpaired mapped over paired_or_unpaired should still yield "list"
assert c_t("list:paired_or_unpaired").effective_collection_type("paired_or_unpaired") == "list"
def test_endswith_colon_boundary():
"""Regression: endswith must require colon boundary to avoid partial matches.
'list:paired_or_unpaired'.endswith('paired') was True because 'paired'
is a substring at the end of 'paired_or_unpaired'. The fix requires
':paired' (with colon prefix) so only proper rank boundaries match.
"""
# list:paired_or_unpaired does NOT have subcollections of type paired
# PAIRED_OR_UNPAIRED_NOT_CONSUMED_BY_PAIRED_WHEN_MAPPING
assert not c_t("list:paired_or_unpaired").has_subcollections_of_type("paired")
# But list:paired DOES have subcollections of type paired (proper boundary)
assert c_t("list:paired").has_subcollections_of_type("paired")
# And list:list:paired_or_unpaired does NOT have subcollections of type paired
assert not c_t("list:list:paired_or_unpaired").has_subcollections_of_type("paired")
# Existing cases still work
assert c_t("list:list:paired").has_subcollections_of_type("paired")
assert c_t("list:list:paired").has_subcollections_of_type("list:paired")
assert not c_t("list:list:paired").has_subcollections_of_type("list")
def test_compound_paired_or_unpaired_has_subcollections():
"""Test compound :paired_or_unpaired suffix in has_subcollections_of_type.
Covers collection_semantics.yml examples:
- MAPPING_LIST_LIST_OVER_LIST_PAIRED_OR_UNPAIRED
- MAPPING_LIST_LIST_PAIRED_OVER_PAIRED_OR_UNPAIRED (via compound path)
"""
# list:list can map over list:paired_or_unpaired
assert c_t("list:list").has_subcollections_of_type("list:paired_or_unpaired")
# list:list:paired can map over list:paired_or_unpaired
# (paired consumed by paired_or_unpaired, higher ranks list == list)
assert c_t("list:list:paired").has_subcollections_of_type("list:paired_or_unpaired")
# list:list:list can map over list:paired_or_unpaired
assert c_t("list:list:list").has_subcollections_of_type("list:paired_or_unpaired")
# list:paired cannot map over list:paired_or_unpaired — same rank,
# after stripping :paired the higher ranks are equal (no remainder)
assert not c_t("list:paired").has_subcollections_of_type("list:paired_or_unpaired")
# list cannot map over list:paired_or_unpaired — lower rank
assert not c_t("list").has_subcollections_of_type("list:paired_or_unpaired")
# paired cannot map over list:paired_or_unpaired — higher ranks don't align
assert not c_t("paired").has_subcollections_of_type("list:paired_or_unpaired")
# paired:paired cannot map over list:paired_or_unpaired — higher ranks
# don't align (paired != list)
assert not c_t("paired:paired").has_subcollections_of_type("list:paired_or_unpaired")
def test_compound_paired_or_unpaired_effective_collection_type():
"""Test effective_collection_type with compound :paired_or_unpaired."""
# list:list over list:paired_or_unpaired → list
assert c_t("list:list").effective_collection_type("list:paired_or_unpaired") == "list"
# list:list:paired over list:paired_or_unpaired → list
# (peel off list and paired_or_unpaired ranks, then strip :paired)
assert c_t("list:list:paired").effective_collection_type("list:paired_or_unpaired") == "list"
# list:list:list over list:paired_or_unpaired → list:list
assert c_t("list:list:list").effective_collection_type("list:paired_or_unpaired") == "list:list"
def test_effective_collection_type_paired_or_unpaired_non_paired():
"""Test that paired_or_unpaired acts like single_datasets for non-paired collections."""
# list over paired_or_unpaired → list (full type preserved, like single_datasets)
assert c_t("list").effective_collection_type("paired_or_unpaired") == "list"
# list:list over paired_or_unpaired → list:list
assert c_t("list:list").effective_collection_type("paired_or_unpaired") == "list:list"
def test_validate():
c_t("list").validate()
c_t("list:paired").validate()
test/unit/data/model/test_collection_semantics.py
+609
View File
@@ -0,0 +1,609 @@
import pytest
import yaml
from pydantic import ValidationError
from galaxy.model.dataset_collections.types.semantics import (
_assumption_elements_to_latex,
arg_parser,
check,
collect_docs_with_examples,
CollectionDefinition,
CollectionInput,
CollectionOutput,
DatasetInput,
DatasetListInput,
DatasetOutput,
DatasetsDeclaration,
DocEntry,
EllipsisMarker,
EquivalenceThen,
Example,
ExampleEntry,
ExampleTests,
expression_to_latex,
generate_docs,
InvalidThen,
MapOverInput,
MapOverThen,
NestedElements,
ReductionThen,
ToolDefinition,
ToolInvocation,
ToolOutputRef,
ToolRuntimeApi,
ToolRuntimeFramework,
validate_api_test_refs,
validate_tool_refs,
validate_workflow_editor_refs,
YAMLRootModel,
)
from galaxy.util.resources import resource_string
def _load_root_model() -> YAMLRootModel:
raw = yaml.safe_load(resource_string("galaxy.model.dataset_collections.types", "collection_semantics.yml"))
return YAMLRootModel.model_validate(raw)
def test_yaml_loads_and_parses():
root = _load_root_model()
assert len(root.root) > 0
def test_all_entries_are_doc_or_example():
root = _load_root_model()
for entry in root.root:
assert isinstance(entry, (DocEntry, ExampleEntry))
def test_has_both_doc_and_example_entries():
root = _load_root_model()
has_doc = any(isinstance(e, DocEntry) for e in root.root)
has_example = any(isinstance(e, ExampleEntry) for e in root.root)
assert has_doc
assert has_example
@pytest.mark.parametrize(
"input_expr, expected_substr",
[
("->", "\\rightarrow"),
("~>", "\\mapsto"),
("list", "\\text{list}"),
("forward", "\\text{forward}"),
("reverse", "\\text{reverse}"),
("mapOver", "\\text{mapOver}"),
("collection", "\\text{collection}"),
("dataset", "\\text{dataset}"),
],
)
def test_expression_to_latex_replacements(input_expr, expected_substr):
result = expression_to_latex(input_expr)
assert expected_substr in result
def test_expression_to_latex_paired_or_unpaired_atomic():
result = expression_to_latex("paired_or_unpaired")
assert "\\text{paired\\_or\\_unpaired}" in result
# should NOT contain bare \text{paired} or \text{unpaired} separately
without_pou = result.replace("\\text{paired\\_or\\_unpaired}", "")
assert "\\text{paired}" not in without_pou
assert "\\text{unpaired}" not in without_pou
def test_expression_to_latex_wrap_true():
result = expression_to_latex("x", wrap=True)
assert result.startswith("$ ")
assert result.endswith(" $")
def test_expression_to_latex_wrap_false():
result = expression_to_latex("x", wrap=False)
assert not result.startswith("$ ")
def test_assumption_elements_to_latex_single_none():
result = _assumption_elements_to_latex({"a": None})
assert "a" in result
assert "\\left\\{" in result
assert "\\right\\}" in result
def test_assumption_elements_to_latex_nested_dict():
result = _assumption_elements_to_latex({"outer": {"inner": None}})
assert "\\text{ outer }" in result
assert "inner" in result
def test_assumption_elements_to_latex_multiple_keys():
result = _assumption_elements_to_latex({"a": None, "b": None})
assert "a" in result
assert "b" in result
assert "," in result
def test_datasets_declaration_as_latex_single():
d = DatasetsDeclaration(datasets=["x"])
assert d.as_latex() == "$ x $"
def test_datasets_declaration_as_latex_multiple():
d = DatasetsDeclaration(datasets=["x", "y"])
result = d.as_latex()
assert "$ x $" in result
assert "$ y $" in result
def test_tool_definition_as_latex():
t = ToolDefinition(**{"in": {"i": "dataset"}, "out": {"o": "dataset"}})
result = t.as_latex()
assert "\\Rightarrow" in result
assert "i:" in result
assert "o:" in result
def test_collection_definition_as_latex():
c = CollectionDefinition(collection_type="paired", elements={"forward": None, "reverse": None})
result = c.as_latex()
assert "paired" in result
assert "forward" in result
assert "reverse" in result
def _make_root(entries: list) -> YAMLRootModel:
return YAMLRootModel.model_validate(entries)
def test_collect_doc_with_no_examples():
root = _make_root([{"doc": "Some doc text"}])
result = collect_docs_with_examples(root)
assert len(result) == 1
doc, examples = result[0]
assert doc.doc == "Some doc text"
assert examples == []
_SIMPLE_THEN = {"type": "invalid", "invocation": {"inputs": {"i": {"type": "collection", "ref": "C"}}}}
def test_collect_doc_with_examples():
root = _make_root(
[
{"doc": "Doc"},
{"example": {"label": "EX1", "then": _SIMPLE_THEN}},
]
)
result = collect_docs_with_examples(root)
assert len(result) == 1
_, examples = result[0]
assert len(examples) == 1
assert examples[0].example.label == "EX1"
def test_collect_example_without_then_excluded():
root = _make_root(
[
{"doc": "Doc"},
{"example": {"label": "NO_THEN"}},
{"example": {"label": "HAS_THEN", "then": _SIMPLE_THEN}},
]
)
result = collect_docs_with_examples(root)
_, examples = result[0]
assert len(examples) == 1
assert examples[0].example.label == "HAS_THEN"
def test_collect_multiple_doc_sections():
root = _make_root(
[
{"doc": "Doc1"},
{"example": {"label": "EX1", "then": _SIMPLE_THEN}},
{"doc": "Doc2"},
{"example": {"label": "EX2", "then": _SIMPLE_THEN}},
]
)
result = collect_docs_with_examples(root)
assert len(result) == 2
assert result[0][0].doc == "Doc1"
assert result[1][0].doc == "Doc2"
assert len(result[0][1]) == 1
assert len(result[1][1]) == 1
def test_arg_parser_default():
args = arg_parser().parse_args([])
assert args.check is False
def test_arg_parser_check_flag():
args = arg_parser().parse_args(["--check"])
assert args.check is True
def test_arg_parser_check_short_flag():
args = arg_parser().parse_args(["-c"])
assert args.check is True
def test_example_tests_rejects_unknown_keys():
with pytest.raises(ValidationError):
ExampleTests(**{"wf_editor": "some value"})
def test_example_tests_accepts_valid_keys():
et = ExampleTests(workflow_editor="some test description")
assert et.workflow_editor == "some test description"
def _extract_examples(root: YAMLRootModel) -> list[Example]:
return [e.example for e in root.root if isinstance(e, ExampleEntry)]
def test_validate_api_test_refs_clean():
"""Current YAML should have no api_test reference errors."""
root = _load_root_model()
examples = _extract_examples(root)
errors = validate_api_test_refs(examples)
assert errors == []
def test_validate_api_test_refs_bad_file():
"""Mutated api_test ref with nonexistent file should be caught."""
ex = Example(
label="BAD_FILE",
tests=ExampleTests(tool_runtime=ToolRuntimeApi(api_test="nonexistent_file.py::some_func")),
)
errors = validate_api_test_refs([ex])
assert len(errors) == 1
assert "nonexistent_file.py" in errors[0]
def test_validate_api_test_refs_bad_function():
"""Mutated api_test ref with real file but nonexistent function should be caught."""
ex = Example(
label="BAD_FUNC",
tests=ExampleTests(tool_runtime=ToolRuntimeApi(api_test="test_tool_execute.py::nonexistent_function")),
)
errors = validate_api_test_refs([ex])
assert len(errors) == 1
assert "nonexistent_function" in errors[0]
def test_validate_api_test_refs_bad_method():
"""Mutated api_test ref with real class but nonexistent method should be caught."""
ex = Example(
label="BAD_METHOD",
tests=ExampleTests(tool_runtime=ToolRuntimeApi(api_test="test_tools.py::TestToolsApi::nonexistent_method")),
)
errors = validate_api_test_refs([ex])
assert len(errors) == 1
assert "nonexistent_method" in errors[0]
def test_validate_tool_refs_clean():
"""Current YAML should have no framework tool reference errors."""
root = _load_root_model()
examples = _extract_examples(root)
errors = validate_tool_refs(examples)
assert errors == []
def test_validate_tool_refs_bad_tool():
"""Nonexistent tool XML should be caught."""
ex = Example(
label="BAD_TOOL",
tests=ExampleTests(tool_runtime=ToolRuntimeFramework(tool="nonexistent_tool_xyz")),
)
errors = validate_tool_refs([ex])
assert len(errors) == 1
assert "nonexistent_tool_xyz" in errors[0]
def test_validate_workflow_editor_refs_clean():
"""Current YAML should have no workflow_editor reference errors."""
root = _load_root_model()
examples = _extract_examples(root)
errors = validate_workflow_editor_refs(examples)
assert errors == []
def test_validate_workflow_editor_refs_bad_ref():
"""Nonexistent workflow_editor description should be caught."""
ex = Example(
label="BAD_WF",
tests=ExampleTests(workflow_editor="totally nonexistent test description xyz"),
)
errors = validate_workflow_editor_refs([ex])
assert len(errors) == 1
assert "totally nonexistent test description xyz" in errors[0]
def test_all_tested_examples_have_then():
"""Every example with test references should also have a then expression."""
root = _load_root_model()
missing = []
for entry in root.root:
if isinstance(entry, ExampleEntry):
ex = entry.example
if ex.tests and not ex.then:
missing.append(ex.label)
assert missing == [], f"Examples with tests but no then: {missing}"
def test_check_returns_no_errors():
"""check() against current YAML should return empty error list."""
errors = check()
assert errors == []
# --- Structured Then Expression Model Tests ---
def test_invalid_then_collection_input():
"""InvalidThen with collection input: tool(i=C)"""
then = InvalidThen(
type="invalid",
invocation=ToolInvocation(inputs={"i": CollectionInput(type="collection", ref="C")}),
)
assert then.as_latex() == expression_to_latex("tool(i=C)", wrap=False)
def test_invalid_then_map_over_with_sub_collection():
"""InvalidThen with mapOver sub-collection: tool(i=mapOver(C, 'paired'))"""
then = InvalidThen(
type="invalid",
invocation=ToolInvocation(
inputs={"i": MapOverInput(type="map_over", collection="C", sub_collection_type="paired")}
),
)
assert then.as_latex() == expression_to_latex("tool(i=mapOver(C, 'paired'))", wrap=False)
def test_invalid_then_map_over_paired_or_unpaired():
"""InvalidThen with mapOver paired_or_unpaired sub-collection."""
then = InvalidThen(
type="invalid",
invocation=ToolInvocation(
inputs={"i": MapOverInput(type="map_over", collection="C", sub_collection_type="paired_or_unpaired")}
),
)
assert then.as_latex() == expression_to_latex("tool(i=mapOver(C, 'paired_or_unpaired'))", wrap=False)
def test_invalid_then_map_over_no_sub_collection():
"""InvalidThen with bare mapOver: tool(i=mapOver(C))"""
then = InvalidThen(
type="invalid",
invocation=ToolInvocation(inputs={"i": MapOverInput(type="map_over", collection="C")}),
)
assert then.as_latex() == expression_to_latex("tool(i=mapOver(C))", wrap=False)
def test_reduction_then():
"""ReductionThen: tool(i=C) -> {o: dataset}"""
then = ReductionThen(
type="reduction",
invocation=ToolInvocation(inputs={"i": CollectionInput(type="collection", ref="C")}),
produces={"o": DatasetOutput(type="dataset")},
)
assert then.as_latex() == expression_to_latex("tool(i=C) -> {o: dataset}", wrap=False)
def test_equivalence_then_list_reduction():
"""EquivalenceThen: tool(i=C) == tool(i=[d_1,...,d_n])"""
then = EquivalenceThen(
type="equivalence",
left=ToolInvocation(inputs={"i": CollectionInput(type="collection", ref="C")}),
right=ToolInvocation(inputs={"i": DatasetListInput(type="dataset_list", refs=["d_1", "...", "d_n"])}),
)
assert then.as_latex() == expression_to_latex("tool(i=C) == tool(i=[d_1,...,d_n])", wrap=False)
def test_equivalence_then_collection_refs():
"""EquivalenceThen: tool(i=C) == tool(i=C_AS_MIXED)"""
then = EquivalenceThen(
type="equivalence",
left=ToolInvocation(inputs={"i": CollectionInput(type="collection", ref="C")}),
right=ToolInvocation(inputs={"i": CollectionInput(type="collection", ref="C_AS_MIXED")}),
)
assert then.as_latex() == expression_to_latex("tool(i=C) == tool(i=C_AS_MIXED)", wrap=False)
def test_map_over_then_paired():
"""MapOverThen for BASIC_MAPPING_PAIRED pattern."""
then = MapOverThen(
type="map_over",
invocation=ToolInvocation(inputs={"i": MapOverInput(type="map_over", collection="C")}),
produces={
"o": CollectionOutput(
type="collection",
collection_type="paired",
elements={
"forward": ToolOutputRef(
type="tool_output_ref",
invocation=ToolInvocation(inputs={"i": DatasetInput(type="dataset", ref="d_f")}),
output="o",
),
"reverse": ToolOutputRef(
type="tool_output_ref",
invocation=ToolInvocation(inputs={"i": DatasetInput(type="dataset", ref="d_r")}),
output="o",
),
},
)
},
)
result = then.as_latex()
assert "\\text{mapOver}(C)" in result
assert "\\text{paired}" in result
assert "\\mapsto" in result
assert "tool(i=d_f)[o]" in result
assert "tool(i=d_r)[o]" in result
def test_map_over_then_sub_collection():
"""MapOverThen for sub-collection mapping (MAPPING_LIST_PAIRED_OVER_PAIRED)."""
then = MapOverThen(
type="map_over",
invocation=ToolInvocation(
inputs={"i": MapOverInput(type="map_over", collection="C", sub_collection_type="paired")}
),
produces={
"o": CollectionOutput(
type="collection",
collection_type="list",
elements={
"el1": ToolOutputRef(
type="tool_output_ref",
invocation=ToolInvocation(inputs={"i": CollectionInput(type="collection", ref="C\\_PAIRED")}),
output="o",
),
},
)
},
)
result = then.as_latex()
assert "\\text{mapOver}(C, '\\text{paired}')" in result
assert "\\text{list}" in result
assert "C\\_PAIRED" in result
def test_pydantic_parse_invalid_then_yaml():
"""Structured InvalidThen parses from dict (simulating YAML load)."""
data = {
"type": "invalid",
"invocation": {"inputs": {"i": {"type": "collection", "ref": "C"}}},
}
then = InvalidThen.model_validate(data)
assert isinstance(then.invocation.inputs["i"], CollectionInput)
assert then.invocation.inputs["i"].ref == "C"
def test_pydantic_parse_reduction_then_yaml():
"""Structured ReductionThen parses from dict."""
data = {
"type": "reduction",
"invocation": {"inputs": {"i": {"type": "collection", "ref": "C"}}},
"produces": {"o": {"type": "dataset"}},
}
then = ReductionThen.model_validate(data)
assert isinstance(then.produces["o"], DatasetOutput)
def test_example_accepts_structured_then():
"""Example model accepts structured then expression."""
data = {
"label": "TEST",
"then": {
"type": "invalid",
"invocation": {"inputs": {"i": {"type": "collection", "ref": "C"}}},
},
"is_valid": False,
}
ex = Example.model_validate(data)
assert isinstance(ex.then, InvalidThen)
assert ex.is_valid is False
def test_example_rejects_string_then():
"""Example model no longer accepts string then after migration."""
with pytest.raises(ValidationError):
Example(label="TEST", then="tool(i=C) -> {o: dataset}")
# --- Smoke Tests (Steps 3-4) ---
def test_all_then_as_latex_succeeds():
"""Every then expression produces valid LaTeX without error."""
root = _load_root_model()
for entry in root.root:
if isinstance(entry, ExampleEntry) and entry.example.then:
latex = entry.example.then.as_latex()
assert isinstance(latex, str)
assert len(latex) > 0, f"{entry.example.label} produced empty LaTeX"
def test_generate_docs_succeeds():
"""generate_docs() runs to completion without errors."""
generate_docs()
# --- Unit Tests for Uncovered Models (Step 5) ---
def test_ellipsis_marker_as_latex():
m = EllipsisMarker(type="ellipsis")
assert m.as_latex() == "..."
def test_dataset_list_input_as_latex():
d = DatasetListInput(type="dataset_list", refs=["d_1", "...", "d_n"])
assert d.as_latex() == "[d_1,...,d_n]"
def test_nested_elements_as_latex():
ne = NestedElements.model_validate(
{
"type": "nested_elements",
"elements": {
"forward": {
"type": "tool_output_ref",
"invocation": {"inputs": {"i": {"type": "dataset", "ref": "d_f"}}},
"output": "o",
},
"reverse": {
"type": "tool_output_ref",
"invocation": {"inputs": {"i": {"type": "dataset", "ref": "d_r"}}},
"output": "o",
},
},
}
)
result = ne.as_latex()
assert "\\text{forward}=" in result
assert "\\text{reverse}=" in result
assert "tool(i=d_f)[o]" in result
def test_collection_output_with_ellipsis_as_latex():
co = CollectionOutput.model_validate(
{
"type": "collection",
"collection_type": "list",
"elements": {
"el1": {
"type": "tool_output_ref",
"invocation": {"inputs": {"i": {"type": "dataset", "ref": "d_1"}}},
"output": "o",
},
"...": {"type": "ellipsis"},
"eln": {
"type": "tool_output_ref",
"invocation": {"inputs": {"i": {"type": "dataset", "ref": "d_n"}}},
"output": "o",
},
},
}
)
result = co.as_latex()
assert "\\text{list}" in result
assert ",...," in result
def test_map_over_input_compound_sub_collection_type():
mi = MapOverInput(type="map_over", collection="C", sub_collection_type="list:paired_or_unpaired")
result = mi.as_latex()
assert "\\text{list}:\\text{paired\\_or\\_unpaired}" in result
def test_map_over_then_produces_dataset():
then = MapOverThen(
type="map_over",
invocation=ToolInvocation(inputs={"i": MapOverInput(type="map_over", collection="C")}),
produces={"o": DatasetOutput(type="dataset")},
)
result = then.as_latex()
assert "\\mapsto" in result
assert "\\text{dataset}" in result
test/unit/tool_util/test_cwl_util.py
+104
View File
@@ -151,3 +151,107 @@ def test_galactic_job_json_collection_element_filetype():
for target in captured_targets:
assert isinstance(target, FileLiteralTarget)
assert target.properties.get("filetype") == "fastqsanger"
def test_galactic_job_json_sample_sheet_collection_without_rows():
"""sample_sheet collection works without rows metadata."""
created_collections = []
def collection_create_func(element_identifiers, collection_type, rows=None, name=None):
created_collections.append(
{"element_identifiers": element_identifiers, "collection_type": collection_type, "rows": rows, "name": name}
)
return {"id": f"collection_{collection_type}"}
job = {
"input1": {
"class": "Collection",
"collection_type": "sample_sheet",
"elements": [
{"identifier": "el1", "class": "File", "contents": "element 1"},
{"identifier": "el2", "class": "File", "contents": "element 2"},
],
}
}
result_job, datasets = galactic_job_json(
job, ".", _mock_upload_func, collection_create_func, tool_or_workflow="workflow"
)
assert len(created_collections) == 1
coll = created_collections[0]
assert coll["collection_type"] == "sample_sheet"
assert coll["rows"] is None
assert len(coll["element_identifiers"]) == 2
assert coll["element_identifiers"][0]["name"] == "el1"
assert coll["element_identifiers"][1]["name"] == "el2"
def test_galactic_job_json_sample_sheet_collection_with_rows():
"""sample_sheet collection passes rows metadata through."""
created_collections = []
def collection_create_func(element_identifiers, collection_type, rows=None, name=None):
created_collections.append(
{"element_identifiers": element_identifiers, "collection_type": collection_type, "rows": rows, "name": name}
)
return {"id": f"collection_{collection_type}"}
job = {
"input1": {
"class": "Collection",
"collection_type": "sample_sheet",
"elements": [
{"identifier": "el1", "class": "File", "contents": "element 1"},
{"identifier": "el2", "class": "File", "contents": "element 2"},
],
"rows": {"el1": {"condition": "treatment"}, "el2": {"condition": "control"}},
}
}
result_job, datasets = galactic_job_json(
job, ".", _mock_upload_func, collection_create_func, tool_or_workflow="workflow"
)
assert len(created_collections) == 1
coll = created_collections[0]
assert coll["collection_type"] == "sample_sheet"
assert coll["rows"] == {"el1": {"condition": "treatment"}, "el2": {"condition": "control"}}
def test_galactic_job_json_sample_sheet_paired_collection():
"""sample_sheet:paired creates nested collection with paired sub-elements."""
created_collections = []
def collection_create_func(element_identifiers, collection_type, rows=None, name=None):
created_collections.append(
{"element_identifiers": element_identifiers, "collection_type": collection_type, "rows": rows, "name": name}
)
return {"id": f"collection_{collection_type}"}
job = {
"input1": {
"class": "Collection",
"collection_type": "sample_sheet:paired",
"elements": [
{
"identifier": "sample1",
"class": "Collection",
"type": "paired",
"elements": [
{"identifier": "forward", "class": "File", "contents": "fwd reads"},
{"identifier": "reverse", "class": "File", "contents": "rev reads"},
],
}
],
}
}
result_job, datasets = galactic_job_json(
job, ".", _mock_upload_func, collection_create_func, tool_or_workflow="workflow"
)
assert len(created_collections) == 1
coll = created_collections[0]
assert coll["collection_type"] == "sample_sheet:paired"
assert coll["rows"] is None
assert len(coll["element_identifiers"]) == 1
el = coll["element_identifiers"][0]
assert el["name"] == "sample1"
assert el["src"] == "new_collection"
assert el["collection_type"] == "paired"
assert len(el["element_identifiers"]) == 2