From f5f3f473f1ed1a8132240ba92f78b07699f535c9 Mon Sep 17 00:00:00 2001
From: Matthias Bernt <m.bernt@ufz.de>
Date: Wed, 8 Dec 2021 19:33:09 +0100
Subject: [PATCH 1/6] allow param_value filter to refer parameters with
 multiple values

For instance if we want to have a parameter B to select values
that have not been selected in a parameter A .. and A has
`multiple="true"`

```
<param name="A" type="select" multiple="true"/>
<param name="B" type="select">
    <options ...>
        <filter type="param_value" column="0" ref="A" keep="false"/>
    </options>
</param>
```
---
 lib/galaxy/tools/parameters/dynamic_options.py | 16 ++++++++++++----
 1 file changed, 12 insertions(+), 4 deletions(-)

diff --git a/lib/galaxy/tools/parameters/dynamic_options.py b/lib/galaxy/tools/parameters/dynamic_options.py
index 759457f4a6a..b49e2a62d20 100644
--- a/lib/galaxy/tools/parameters/dynamic_options.py
+++ b/lib/galaxy/tools/parameters/dynamic_options.py
@@ -157,6 +157,7 @@ class DataMetaFilter(Filter):
             self.column = d_option.column_spec_to_index(self.column)
         self.multiple = string_as_bool(elem.get("multiple", "False"))
         self.separator = elem.get("separator", ",")
+        log.error(f"data_meta.init: ref_name {self.ref_name} key {self.key} column {self.column} multiple {self.multiple} separator {self.separator}")
 
     def get_dependency_name(self):
         return self.ref_name
@@ -275,10 +276,17 @@ class ParamValueFilter(Filter):
             if not hasattr(ref, ref_attribute):
                 return []  # ref does not have attribute, so we cannot filter, return empty list
             ref = getattr(ref, ref_attribute)
-        ref = str(ref)
+        log.error(f"ParamValue ref {ref}")
+        if ref is None:
+            ref = []
+        elif isinstance(ref, list):
+            ref = [str(_) for _ in ref]
+        else:
+            ref = [str(ref)]
+        log.error(f"ParamValue ref {ref}")
         rval = []
         for fields in options:
-            if self.keep == (fields[self.column] == ref):
+            if self.keep == (fields[self.column] in ref):
                 rval.append(fields)
         return rval
 
@@ -666,10 +674,10 @@ class DynamicOptions:
             try:
                 datasets = _get_ref_data(other_values, self.dataset_ref_name)
             except KeyError:  # no such dataset
-                log.warning(f"could not create dynamic options from_dataset: {self.dataset_ref_name} unknown")
+                log.warning(f"{self.tool_param.name} could not create dynamic options from_dataset: {self.dataset_ref_name} unknown")
                 return []
             except ValueError:  # not a valid dataset
-                log.warning(f"could not create dynamic options from_dataset: {self.dataset_ref_name} not a data or collection parameter")
+                log.warning(f"{self.tool_param.name} could not create dynamic options from_dataset: {self.dataset_ref_name} not a data or collection parameter")
                 return []
 
             options = []

From c8090031b736927a2c178f43f662953081975952 Mon Sep 17 00:00:00 2001
From: Matthias Bernt <m.bernt@ufz.de>
Date: Wed, 8 Dec 2021 20:17:59 +0100
Subject: [PATCH 2/6] add test for param_value filter

---
 test/functional/tool-data/fasta_indexes.loc  |  2 +
 test/functional/tools/filter_param_value.xml | 51 ++++++++++++++++++++
 test/functional/tools/samples_tool_conf.xml  |  1 +
 3 files changed, 54 insertions(+)
 create mode 100644 test/functional/tools/filter_param_value.xml

diff --git a/test/functional/tool-data/fasta_indexes.loc b/test/functional/tool-data/fasta_indexes.loc
index 57f11daa9a9..b8776ac5d71 100644
--- a/test/functional/tool-data/fasta_indexes.loc
+++ b/test/functional/tool-data/fasta_indexes.loc
@@ -1,2 +1,4 @@
 hg19_value	hg19	hg19_name	hg19_path
 hg18_value	hg18	hg18_name	hg18_path
+hg38_value	hg38	hg38_name	hg38_path
+
diff --git a/test/functional/tools/filter_param_value.xml b/test/functional/tools/filter_param_value.xml
new file mode 100644
index 00000000000..d8ee13f619b
--- /dev/null
+++ b/test/functional/tools/filter_param_value.xml
@@ -0,0 +1,51 @@
+<tool id="filter_param_value" name="filter_param_value" version="0.1.0">
+    <description>Filter input with the param_value</description>
+    <command><![CDATA[
+        echo $select1 > '$output' && 
+        echo $select2 >> '$output'
+    ]]></command>
+    <inputs>
+        <!-- define 2 selects that are initialised with entries from a data table
+             and use param_value filters to ensure that disjoint elements are
+             selected (one of them is multiple="true" to ensure that also list of
+             elements can be used in the filter) -->
+        <param name="select1" type="select" multiple="true">
+            <options from_data_table="test_fasta_indexes">
+                <column name="value" index="0"/>
+                <column name="name" index="1"/>
+                <filter type="param_value" column="0" ref="select2" keep="false"/>
+            </options>
+        </param>
+        <param name="select2" type="select">
+            <options  from_data_table="test_fasta_indexes">
+                <column name="value" index="0"/>
+                <column name="name" index="1"/>
+                <filter type="param_value" column="0" ref="select1" keep="false"/>
+            </options>
+        </param>
+    </inputs>
+
+    <outputs>
+        <data format="txt" name="output" />
+    </outputs>
+
+    <tests>
+        <test expect_failure="false">
+            <param name="select1" value="hg18_value,hg19_value"/>
+            <param name="select2" value="hg38_value" />
+            <output name="output">
+                <assert_contents>
+                    <has_line line="hg18_value,hg19_value" />    
+                    <has_line line="hg38_value" />    
+                </assert_contents>
+            </output>
+        </test>
+        <test expect_failure="true">
+            <param name="select1" value="hg18_value,hg19_value"/>
+            <param name="select2" value="hg18_value" />
+        </test>
+    </tests>
+
+    <help>
+    </help>
+</tool>
diff --git a/test/functional/tools/samples_tool_conf.xml b/test/functional/tools/samples_tool_conf.xml
index 2b74faf87c9..f62d6acbbaf 100644
--- a/test/functional/tools/samples_tool_conf.xml
+++ b/test/functional/tools/samples_tool_conf.xml
@@ -47,6 +47,7 @@
   <tool file="inputs_as_json_with_paths.xml" />
   <tool file="inputs_as_json_with_staging_path_and_source_path.xml" />
   <tool file="filter_multiple_splitter.xml" />
+  <tool file="filter_param_value.xml" />
   <tool file="filter_static_regexp.xml" />
   <tool file="select_from_dataset.xml" />
   <tool file="select_from_dataset_optional.xml" />

From dddf2011779b718965f6136ba6211e12d01bb714 Mon Sep 17 00:00:00 2001
From: Matthias Bernt <m.bernt@ufz.de>
Date: Wed, 8 Dec 2021 20:25:01 +0100
Subject: [PATCH 3/6] remove debug messages

---
 lib/galaxy/tools/parameters/dynamic_options.py | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/lib/galaxy/tools/parameters/dynamic_options.py b/lib/galaxy/tools/parameters/dynamic_options.py
index b49e2a62d20..fcf17f71569 100644
--- a/lib/galaxy/tools/parameters/dynamic_options.py
+++ b/lib/galaxy/tools/parameters/dynamic_options.py
@@ -276,14 +276,12 @@ class ParamValueFilter(Filter):
             if not hasattr(ref, ref_attribute):
                 return []  # ref does not have attribute, so we cannot filter, return empty list
             ref = getattr(ref, ref_attribute)
-        log.error(f"ParamValue ref {ref}")
         if ref is None:
             ref = []
         elif isinstance(ref, list):
             ref = [str(_) for _ in ref]
         else:
             ref = [str(ref)]
-        log.error(f"ParamValue ref {ref}")
         rval = []
         for fields in options:
             if self.keep == (fields[self.column] in ref):
@@ -674,10 +672,10 @@ class DynamicOptions:
             try:
                 datasets = _get_ref_data(other_values, self.dataset_ref_name)
             except KeyError:  # no such dataset
-                log.warning(f"{self.tool_param.name} could not create dynamic options from_dataset: {self.dataset_ref_name} unknown")
+                log.warning(f"Parameter {self.tool_param.name}: could not create dynamic options from_dataset: {self.dataset_ref_name} unknown")
                 return []
             except ValueError:  # not a valid dataset
-                log.warning(f"{self.tool_param.name} could not create dynamic options from_dataset: {self.dataset_ref_name} not a data or collection parameter")
+                log.warning(f"Parameter {self.tool_param.name}: could not create dynamic options from_dataset: {self.dataset_ref_name} not a data or collection parameter")
                 return []
 
             options = []

From 07d66b8ae01ee9da86178a8db3bb30c6d23e0e97 Mon Sep 17 00:00:00 2001
From: Matthias Bernt <m.bernt@ufz.de>
Date: Thu, 9 Dec 2021 11:10:34 +0100
Subject: [PATCH 4/6] do not add hg38 to test data table

interferes with another test
---
 test/functional/tool-data/fasta_indexes.loc  | 2 +-
 test/functional/tools/filter_param_value.xml | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/test/functional/tool-data/fasta_indexes.loc b/test/functional/tool-data/fasta_indexes.loc
index b8776ac5d71..f733d49f42c 100644
--- a/test/functional/tool-data/fasta_indexes.loc
+++ b/test/functional/tool-data/fasta_indexes.loc
@@ -1,4 +1,4 @@
 hg19_value	hg19	hg19_name	hg19_path
 hg18_value	hg18	hg18_name	hg18_path
-hg38_value	hg38	hg38_name	hg38_path
+mm10_value	mm10	mm10_name	mm10_path
 
diff --git a/test/functional/tools/filter_param_value.xml b/test/functional/tools/filter_param_value.xml
index d8ee13f619b..4069395db35 100644
--- a/test/functional/tools/filter_param_value.xml
+++ b/test/functional/tools/filter_param_value.xml
@@ -32,11 +32,11 @@
     <tests>
         <test expect_failure="false">
             <param name="select1" value="hg18_value,hg19_value"/>
-            <param name="select2" value="hg38_value" />
+            <param name="select2" value="mm10_value" />
             <output name="output">
                 <assert_contents>
                     <has_line line="hg18_value,hg19_value" />    
-                    <has_line line="hg38_value" />    
+                    <has_line line="mm10_value" />    
                 </assert_contents>
             </output>
         </test>

From b53f29465fe1004baed44f9ef48d064afa8b73ba Mon Sep 17 00:00:00 2001
From: Matthias Bernt <m.bernt@ufz.de>
Date: Thu, 9 Dec 2021 11:11:09 +0100
Subject: [PATCH 5/6] comment forward reference

seems that it is not possible to refer other input parameters
that are defined below in the xml
---
 test/functional/tools/filter_param_value.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/functional/tools/filter_param_value.xml b/test/functional/tools/filter_param_value.xml
index 4069395db35..248d07976b7 100644
--- a/test/functional/tools/filter_param_value.xml
+++ b/test/functional/tools/filter_param_value.xml
@@ -13,7 +13,7 @@
             <options from_data_table="test_fasta_indexes">
                 <column name="value" index="0"/>
                 <column name="name" index="1"/>
-                <filter type="param_value" column="0" ref="select2" keep="false"/>
+                <!-- <filter type="param_value" column="0" ref="select2" keep="false"/> -->
             </options>
         </param>
         <param name="select2" type="select">

From 2e33be78acf4a7a0bf343d78b4f52b9a6edc88a3 Mon Sep 17 00:00:00 2001
From: Matthias Bernt <m.bernt@ufz.de>
Date: Thu, 9 Dec 2021 23:11:02 +0100
Subject: [PATCH 6/6] make param_value filter work with multi data inputs

ie data with multiple=true and collections
---
 .../tools/parameters/dynamic_options.py       |  30 +++--
 test/functional/tools/filter_param_value.xml  |   6 +-
 .../filter_param_value_ref_attribute.xml      | 116 ++++++++++++++++++
 test/functional/tools/samples_tool_conf.xml   |   1 +
 4 files changed, 142 insertions(+), 11 deletions(-)
 create mode 100644 test/functional/tools/filter_param_value_ref_attribute.xml

diff --git a/lib/galaxy/tools/parameters/dynamic_options.py b/lib/galaxy/tools/parameters/dynamic_options.py
index fcf17f71569..b10af39e191 100644
--- a/lib/galaxy/tools/parameters/dynamic_options.py
+++ b/lib/galaxy/tools/parameters/dynamic_options.py
@@ -272,19 +272,31 @@ class ParamValueFilter(Filter):
         if trans is not None and trans.workflow_building_mode:
             return []
         ref = other_values.get(self.ref_name, None)
-        for ref_attribute in self.ref_attribute:
-            if not hasattr(ref, ref_attribute):
-                return []  # ref does not have attribute, so we cannot filter, return empty list
-            ref = getattr(ref, ref_attribute)
         if ref is None:
             ref = []
-        elif isinstance(ref, list):
-            ref = [str(_) for _ in ref]
-        else:
-            ref = [str(ref)]
+
+        # - for HDCAs the list of contained HDAs is extracted
+        # - single values are transformed in a single eleent list
+        # - remaining cases are already lists (select and data parameters with multiple=true)
+        if isinstance(ref, HistoryDatasetCollectionAssociation):
+            ref = ref.to_hda_representative(multiple=True)
+        elif not isinstance(ref, list):
+            ref = [ref]
+
+        ref_values = []
+        for r in ref:
+            for ref_attribute in self.ref_attribute:
+                # ref does not have attribute, so we cannot filter,
+                # but other refs might have it
+                if not hasattr(r, ref_attribute):
+                    break
+                r = getattr(r, ref_attribute)
+            ref_values.append(r)
+        ref_values = [str(_) for _ in ref_values]
+
         rval = []
         for fields in options:
-            if self.keep == (fields[self.column] in ref):
+            if self.keep == (fields[self.column] in ref_values):
                 rval.append(fields)
         return rval
 
diff --git a/test/functional/tools/filter_param_value.xml b/test/functional/tools/filter_param_value.xml
index 248d07976b7..9dc13e8a5a3 100644
--- a/test/functional/tools/filter_param_value.xml
+++ b/test/functional/tools/filter_param_value.xml
@@ -13,11 +13,13 @@
             <options from_data_table="test_fasta_indexes">
                 <column name="value" index="0"/>
                 <column name="name" index="1"/>
-                <!-- <filter type="param_value" column="0" ref="select2" keep="false"/> -->
+                <!-- unfortunatelly this does not work (bug?): at the moment ane can
+                     not refer to other inputs that are defined below)
+                    <filter type="param_value" column="0" ref="select2" keep="false"/> -->
             </options>
         </param>
         <param name="select2" type="select">
-            <options  from_data_table="test_fasta_indexes">
+            <options from_data_table="test_fasta_indexes">
                 <column name="value" index="0"/>
                 <column name="name" index="1"/>
                 <filter type="param_value" column="0" ref="select1" keep="false"/>
diff --git a/test/functional/tools/filter_param_value_ref_attribute.xml b/test/functional/tools/filter_param_value_ref_attribute.xml
new file mode 100644
index 00000000000..68439df520f
--- /dev/null
+++ b/test/functional/tools/filter_param_value_ref_attribute.xml
@@ -0,0 +1,116 @@
+<tool id="filter_param_value_ref_attribute" name="filter_param_value_ref_attribute" version="0.1.0">
+    <description>Filter input with the param_value</description>
+    <command><![CDATA[
+        #if $select_single
+            echo $select_single >> '$output' &&
+        #end if
+        #if $select_mult
+            echo $select_mult >> '$output' &&
+        #end if
+        #if $select_coll
+            echo $select_coll >> '$output' &&
+        #end if
+        true
+    ]]></command>
+    <inputs>
+        <!-- this tests the param_value filter with the ref_attribute attribute 
+             (so ref is dataset(s) or a dataset collection), the following pairs
+             test param_value filter refering:
+             - dataset 
+             - multiple datasets
+             - a collection
+
+             in each case the data and the select input are optional to allow
+             to test them separately -->
+
+        <!-- 1. dataset (here with non-default keep) -->
+        <param name="data_single" type="data" format="bed" optional="true"/>
+        <param name="select_single" type="select" multiple="true" optional="true">
+            <options from_data_table="test_fasta_indexes">
+                <column name="value" index="0"/>
+                <column name="name" index="1"/>
+                <filter type="param_value" column="1" ref="data_single" ref_attribute="metadata.dbkey" keep="false"/>
+            </options>
+        </param>
+        
+        <!-- 2. same, but with a data input accepting multiple datasets 
+            (which may have different dbkeys .. but we can not specify them in a test) -->
+        <param name="data_mult" format="bed" type="data" multiple="true" optional="true"/>
+        <param name="select_mult" type="select" multiple="true" optional="true">
+            <options from_data_table="test_fasta_indexes">
+                <column name="value" index="0"/>
+                <column name="name" index="1"/>
+                <filter type="param_value" column="1" ref="data_mult" ref_attribute="metadata.dbkey"/>
+            </options>
+        </param>
+        
+        <!-- 3. same, but with a collection input (elements may have different dbkeys) -->
+        <param name="data_coll" format="bed" type="data_collection" collection_type="list" optional="true"/>
+        <param name="select_coll" type="select" multiple="true" optional="true">
+            <options from_data_table="test_fasta_indexes">
+                <column name="value" index="0"/>
+                <column name="name" index="1"/>
+                <filter type="param_value" column="1" ref="data_coll" ref_attribute="metadata.dbkey"/>
+            </options>
+        </param>
+    </inputs>
+
+    <outputs>
+        <data format="txt" name="output" />
+    </outputs>
+
+    <tests>
+        <test expect_failure="false">
+            <param name="data_single" value="1.bed" ftype="bed" dbkey="hg19"/>
+            <param name="select_single" value="hg18_value,mm10_value"/>
+            <output name="output">
+                <assert_contents>
+                    <has_line line="hg18_value,mm10_value"/>
+                </assert_contents>
+            </output>
+        </test>
+        <test expect_failure="true">
+            <param name="data_single" value="1.bed" ftype="bed" dbkey="hg19"/>
+            <param name="select_single" value="hg19_value"/>
+        </test>
+        <test expect_failure="false">
+            <param name="data_mult" value="1.bed,2.bed" dbkey="hg19"/>
+            <param name="select_mult" value="hg19_value"/>
+            <output name="output">
+                <assert_contents>
+                    <has_line line="hg19_value"/>
+                </assert_contents>
+            </output>
+        </test>
+        <test expect_failure="true">
+            <param name="data_mult" value="1.bed,2.bed" dbkey="hg19"/>
+            <param name="select_mult" value="hg18_value"/>
+        </test>
+        <test expect_failure="false">
+            <param name="data_coll">
+                <collection type="list">
+                    <element name="element1" ftype="bed" value="1.bed" dbkey="hg18"/>
+                    <element name="element2" ftype="bed" value="2.bed" dbkey="hg19"/>
+               </collection>
+            </param>
+            <param name="select_coll" value="hg18_value,hg19_value"/>
+            <output name="output">
+                <assert_contents>
+                    <has_line line="hg18_value,hg19_value"/>
+                </assert_contents>
+            </output>
+        </test>
+        <test expect_failure="true">
+            <param name="data_coll">
+                <collection type="list">
+                    <element name="element1" ftype="bed" value="1.bed" dbkey="hg18"/>
+                    <element name="element2" ftype="bed" value="2.bed" dbkey="hg19"/>
+               </collection>
+            </param>
+            <param name="select_coll" value="mm10_value"/>
+        </test>
+    </tests>
+
+    <help>
+    </help>
+</tool>
diff --git a/test/functional/tools/samples_tool_conf.xml b/test/functional/tools/samples_tool_conf.xml
index f62d6acbbaf..92870be5a6e 100644
--- a/test/functional/tools/samples_tool_conf.xml
+++ b/test/functional/tools/samples_tool_conf.xml
@@ -48,6 +48,7 @@
   <tool file="inputs_as_json_with_staging_path_and_source_path.xml" />
   <tool file="filter_multiple_splitter.xml" />
   <tool file="filter_param_value.xml" />
+  <tool file="filter_param_value_ref_attribute.xml" />
   <tool file="filter_static_regexp.xml" />
   <tool file="select_from_dataset.xml" />
   <tool file="select_from_dataset_optional.xml" />