From 6563e387a5d001bb6078469b7ad6a13f9bbed8aa Mon Sep 17 00:00:00 2001
From: John Chilton <jmchilton@gmail.com>
Date: Wed, 16 Aug 2017 12:54:27 -0400
Subject: [PATCH] Allow discovering dataset collections in galaxy.json in a
 nested fashion.

Fixes #2658.
---
 lib/galaxy/tools/parameters/output_collect.py | 17 ++++
 ...ates_dynamic_nested_from_json_elements.xml | 84 +++++++++++++++++++
 test/functional/tools/samples_tool_conf.xml   |  1 +
 3 files changed, 102 insertions(+)
 create mode 100644 test/functional/tools/collection_creates_dynamic_nested_from_json_elements.xml

diff --git a/lib/galaxy/tools/parameters/output_collect.py b/lib/galaxy/tools/parameters/output_collect.py
index 9b30b937abb..530e0400b52 100644
--- a/lib/galaxy/tools/parameters/output_collect.py
+++ b/lib/galaxy/tools/parameters/output_collect.py
@@ -93,8 +93,25 @@ class ToolProvidedMetadata(object):
 
     def get_new_datasets(self, output_name):
         datasets = self.tool_provided_job_metadata.get(output_name, {}).get("datasets", [])
+        if not datasets:
+            elements = self.tool_provided_job_metadata.get(output_name, {}).get("elements", [])
+            if elements:
+                datasets = self._elements_to_datasets(elements)
         return datasets
 
+    def _elements_to_datasets(self, elements, level=0):
+        for element in elements:
+            extra_kwds = {"identifier_%d" % level: element["name"]}
+            if "elements" in element:
+                for inner_element in self._elements_to_datasets(element["elements"], level=level + 1):
+                    dataset = extra_kwds.copy()
+                    dataset.update(inner_element)
+                    yield dataset
+            else:
+                dataset = extra_kwds
+                extra_kwds.update(element)
+                yield extra_kwds
+
 
 def collect_dynamic_collections(
     tool,
diff --git a/test/functional/tools/collection_creates_dynamic_nested_from_json_elements.xml b/test/functional/tools/collection_creates_dynamic_nested_from_json_elements.xml
new file mode 100644
index 00000000000..c0110ce6983
--- /dev/null
+++ b/test/functional/tools/collection_creates_dynamic_nested_from_json_elements.xml
@@ -0,0 +1,84 @@
+<tool id="collection_creates_dynamic_nested_from_json_elements" name="collection_creates_dynamic_nested_from_json_elements" version="0.1.0" profile="17.09">
+  <command>
+    echo "A" > oe1_ie1.fq ;
+    echo "B" > oe1_ie2.fq ;
+    echo "C" > oe2_ie1.fq ;
+    echo "D" > oe2_ie2.fq ;
+    echo "E" > oe3_ie1.fq ;
+    echo "F" > oe3_ie2.fq ;
+    cp $c1 galaxy.json
+  </command>
+  <configfiles>
+      <configfile name="c1">{"list_output": {
+    "elements": [
+      {"name": "oe1", "elements": [
+        {"name": "ie1", "filename": "oe1_ie1.fq"},
+        {"name": "ie2", "filename": "oe1_ie2.fq"}
+      ]},
+      {"name": "oe2", "elements": [
+        {"name": "ie1", "filename": "oe2_ie1.fq"},
+        {"name": "ie2", "filename": "oe2_ie2.fq"}
+      ]},
+      {"name": "oe3", "elements": [
+        {"name": "ie1", "filename": "oe3_ie1.fq"},
+        {"name": "ie2", "filename": "oe3_ie2.fq"}
+      ]}
+]}}</configfile>
+  </configfiles>
+  <inputs>
+    <param name="foo" type="text" label="Dummy Parameter" />
+  </inputs>
+  <outputs>
+    <collection name="list_output" type="list:list" label="Duplicate List">
+      <!-- Use named regex group to grab pattern
+           <identifier_0>_<identifier_1>.fq. Here identifier_0 is the list
+           identifier of the outer list and identifier_1 is the list identifier
+           of the inner list (for instance oe1_ie2.fq in above example).
+      -->
+      <discover_datasets from_provided_metadata="true" ext="fastqsanger" visible="true" />
+    </collection>
+  </outputs>
+  <tests>
+    <test>
+      <param name="foo" value="bar" />
+      <output_collection name="list_output" type="list:list">
+        <element name="oe1">
+          <element name="ie1">
+            <assert_contents>
+              <has_text_matching expression="^A\n$" />
+            </assert_contents>
+          </element>
+          <element name="ie2">
+            <assert_contents>
+              <has_text_matching expression="^B\n$" />
+            </assert_contents>
+          </element>
+        </element>
+        <element name="oe2">
+          <element name="ie1">
+            <assert_contents>
+              <has_text_matching expression="^C\n$" />
+            </assert_contents>
+          </element>
+          <element name="ie2">
+            <assert_contents>
+              <has_text_matching expression="^D\n$" />
+            </assert_contents>
+          </element>
+        </element>
+        <element name="oe3">
+          <element name="ie1">
+            <assert_contents>
+              <has_text_matching expression="^E\n$" />
+            </assert_contents>
+          </element>
+          <element name="ie2">
+            <assert_contents>
+              <has_text_matching expression="^F\n$" />
+            </assert_contents>
+          </element>
+        </element>
+      </output_collection>
+    </test>
+  </tests>
+</tool>
diff --git a/test/functional/tools/samples_tool_conf.xml b/test/functional/tools/samples_tool_conf.xml
index 4fa4feee5f0..eb0f66eb36b 100644
--- a/test/functional/tools/samples_tool_conf.xml
+++ b/test/functional/tools/samples_tool_conf.xml
@@ -105,6 +105,7 @@
   <tool file="collection_split_on_column.xml" />
   <tool file="collection_creates_dynamic_nested.xml" />
   <tool file="collection_creates_dynamic_nested_from_json.xml" />
+  <tool file="collection_creates_dynamic_nested_from_json_elements.xml" />
   <tool file="collection_creates_dynamic_list_of_pairs.xml" />
   <tool file="collection_type_source.xml" />
   <tool file="collection_creates_list_fail.xml" />