Merge pull request #4505 from jmchilton/composite_upload_refine

Tests and clarifications for composite uploads.

lib/galaxy/tools/xsd/galaxy.xsd

@@ -1062,18 +1062,18 @@ of ``type`` ``data``.</xs:documentation>
   </xs:complexType>
   <xs:complexType name="TestCompositeData">
     <xs:annotation>
-      <xs:documentation xml:lang="en">Define extra composite input files for test input.</xs:documentation>
+      <xs:documentation xml:lang="en">Define extra composite input files for test
+input. The specified ``ftype`` on the parent ``param`` should specify a composite
+datatype with defined static composite files. The order of the defined composite
+files on the datatype must match the order specified with these elements and All
+non-optional composite inputs must be specified as part of the ``param``.
+</xs:documentation>
     </xs:annotation>
     <xs:attribute name="value" type="xs:string" use="required">
       <xs:annotation>
         <xs:documentation xml:lang="en">Path relative to test-data of composite file.</xs:documentation>
       </xs:annotation>
     </xs:attribute>
-    <xs:attribute name="ftype" type="xs:string">
-      <xs:annotation>
-        <xs:documentation xml:lang="en">Optional datatype of composite file for test input.</xs:documentation>
-      </xs:annotation>
-    </xs:attribute>
   </xs:complexType>
   <xs:complexType name="TestCollection">
     <xs:annotation>

test/api/test_tools.py

@@ -6,7 +6,8 @@ from base.populators import (
     DatasetCollectionPopulator,
     DatasetPopulator,
     LibraryPopulator,
-    skip_without_tool
+    skip_without_tool,
+    skip_without_datatype,
 )
 from galaxy.tools.verify.test_data import TestDataResolver
 
@@ -119,6 +120,71 @@ class ToolsTestCase(api.ApiTestCase):
         rdata_metadata = self._upload_and_get_details(open(rdata_path, "rb"), file_type="auto")
         self.assertEquals(rdata_metadata["file_ext"], "rdata")
 
+    @skip_without_datatype("velvet")
+    def test_composite_datatype(self):
+        with self.dataset_populator.test_history() as history_id:
+            dataset = self._velvet_upload(history_id, extra_inputs={
+                "files_1|url_paste": "roadmaps content",
+                "files_1|type": "upload_dataset",
+                "files_2|url_paste": "log content",
+                "files_2|type": "upload_dataset",
+            })
+
+            roadmaps_content = self._get_roadmaps_content(history_id, dataset)
+            assert roadmaps_content.strip() == "roadmaps content", roadmaps_content
+
+    @skip_without_datatype("velvet")
+    def test_composite_datatype_space_to_tab(self):
+        # Like previous test but set one upload with space_to_tab to True to
+        # verify that works.
+        with self.dataset_populator.test_history() as history_id:
+            dataset = self._velvet_upload(history_id, extra_inputs={
+                "files_1|url_paste": "roadmaps content",
+                "files_1|type": "upload_dataset",
+                "files_1|space_to_tab": "Yes",
+                "files_2|url_paste": "log content",
+                "files_2|type": "upload_dataset",
+            })
+
+            roadmaps_content = self._get_roadmaps_content(history_id, dataset)
+            assert roadmaps_content.strip() == "roadmaps\tcontent", roadmaps_content
+
+    @skip_without_datatype("velvet")
+    def test_composite_datatype_posix_lines(self):
+        # Like previous test but set one upload with space_to_tab to True to
+        # verify that works.
+        with self.dataset_populator.test_history() as history_id:
+            dataset = self._velvet_upload(history_id, extra_inputs={
+                "files_1|url_paste": "roadmaps\rcontent",
+                "files_1|type": "upload_dataset",
+                "files_1|space_to_tab": "Yes",
+                "files_2|url_paste": "log\rcontent",
+                "files_2|type": "upload_dataset",
+            })
+
+            roadmaps_content = self._get_roadmaps_content(history_id, dataset)
+            assert roadmaps_content.strip() == "roadmaps\ncontent", roadmaps_content
+
+    def _velvet_upload(self, history_id, extra_inputs):
+        payload = self.dataset_populator.upload_payload(
+            history_id,
+            "sequences content",
+            file_type="velvet",
+            extra_inputs=extra_inputs,
+        )
+        run_response = self.dataset_populator.tools_post(payload)
+        self.dataset_populator.wait_for_tool_run(history_id, run_response)
+        datasets = run_response.json()["outputs"]
+
+        assert len(datasets) == 1
+        dataset = datasets[0]
+
+        return dataset
+
+    def _get_roadmaps_content(self, history_id, dataset):
+        roadmaps_content = self.dataset_populator.get_history_dataset_content(history_id, dataset=dataset, filename="Roadmaps")
+        return roadmaps_content
+
     def test_unzip_collection(self):
         with self.dataset_populator.test_history() as history_id:
             hdca_id = self.__build_pair(history_id, ["123", "456"])

test/base/interactor.py

@@ -188,10 +188,7 @@ class GalaxyInteractorApi(object):
                 file_name = self.functional_test_case.get_filename(composite_file.get('value'), shed_tool_id=shed_tool_id)
                 files["files_%s|file_data" % i] = open(file_name, 'rb')
                 tool_input.update({
-                    # "files_%d|NAME" % i: name,
                     "files_%d|type" % i: "upload_dataset",
-                    # TODO:
-                    # "files_%d|space_to_tab" % i: composite_file.get( 'space_to_tab', False )
                 })
             name = test_data['name']
         else:

test/base/populators.py

@@ -2,6 +2,7 @@ import contextlib
 import json
 import time
 
+from functools import wraps
 from operator import itemgetter
 
 import requests
@@ -26,8 +27,9 @@ DEFAULT_TIMEOUT = 60  # Secs to wait for state to turn ok
 
 
 def skip_without_tool(tool_id):
-    """ Decorate an API test method as requiring a specific tool,
-    have nose skip the test case is the tool is unavailable.
+    """Decorate an API test method as requiring a specific tool.
+
+    Have test framework skip the test case is the tool is unavailable.
     """
 
     def method_wrapper(method):
@@ -39,21 +41,46 @@ def skip_without_tool(tool_id):
             tool_ids = [itemgetter("id")(_) for _ in tools]
             return tool_ids
 
+        @wraps(method)
         def wrapped_method(api_test_case, *args, **kwargs):
-            if tool_id not in get_tool_ids(api_test_case):
-                from nose.plugins.skip import SkipTest
-                raise SkipTest()
-
+            _raise_skip_if(tool_id not in get_tool_ids(api_test_case))
             return method(api_test_case, *args, **kwargs)
 
-        # Must preserve method name so nose can detect and report tests by
-        # name.
-        wrapped_method.__name__ = method.__name__
         return wrapped_method
 
     return method_wrapper
 
 
+def skip_without_datatype(extension):
+    """Decorate an API test method as requiring a specific datatype.
+
+    Have test framework skip the test case is the tool is unavailable.
+    """
+
+    def has_datatype(api_test_case):
+        index_response = api_test_case.galaxy_interactor.get("datatypes")
+        assert index_response.status_code == 200, "Failed to fetch datatypes for target Galaxy."
+        datatypes = index_response.json()
+        assert isinstance(datatypes, list)
+        return extension in datatypes
+
+    def method_wrapper(method):
+        @wraps(method)
+        def wrapped_method(api_test_case, *args, **kwargs):
+            _raise_skip_if(not has_datatype(api_test_case))
+            method(api_test_case, *args, **kwargs)
+
+        return wrapped_method
+
+    return method_wrapper
+
+
+def _raise_skip_if(check):
+    if check:
+        from nose.plugins.skip import SkipTest
+        raise SkipTest()
+
+
 # Deprecated mixin, use dataset populator instead.
 # TODO: Rework existing tests to target DatasetPopulator in a setup method instead.
 class TestsDatasets:
@@ -80,15 +107,23 @@ class BaseDatasetPopulator(object):
     """
 
     def new_dataset(self, history_id, content='TestData123', wait=False, **kwds):
+        run_response = self.new_dataset_request(history_id, content=content, wait=wait, **kwds)
+        return run_response.json()["outputs"][0]
+
+    def new_dataset_request(self, history_id, content='TestData123', wait=False, **kwds):
         payload = self.upload_payload(history_id, content, **kwds)
-        run_response = self._post("tools", data=payload)
-        run = run_response.json()
+        run_response = self.tools_post(payload)
         if wait:
-            assert run_response.status_code == 200, run
-            job = run["jobs"][0]
-            self.wait_for_job(job["id"])
-            self.wait_for_history(history_id, assert_ok=True)
-        return run["outputs"][0]
+            self.wait_for_tool_run(history_id, run_response)
+        return run_response
+
+    def wait_for_tool_run(self, history_id, run_response):
+        run = run_response.json()
+        assert run_response.status_code == 200, run
+        job = run["jobs"][0]
+        self.wait_for_job(job["id"])
+        self.wait_for_history(history_id, assert_ok=True)
+        return run_response
 
     def wait_for_history(self, history_id, assert_ok=False, timeout=DEFAULT_TIMEOUT):
         try:
@@ -144,6 +179,7 @@ class BaseDatasetPopulator(object):
             upload_params["files_0|space_to_tab"] = kwds["space_to_tab"]
         if "auto_decompress" in kwds:
             upload_params["files_0|auto_decompress"] = kwds["auto_decompress"]
+        upload_params.update(kwds.get("extra_inputs", {}))
         return self.run_tool_payload(
             tool_id='upload1',
             inputs=upload_params,
@@ -163,15 +199,25 @@ class BaseDatasetPopulator(object):
             **kwds
         )
 
-    def run_tool(self, tool_id, inputs, history_id, **kwds):
+    def run_tool(self, tool_id, inputs, history_id, assert_ok=True, **kwds):
         payload = self.run_tool_payload(tool_id, inputs, history_id, **kwds)
-        tool_response = self._post("tools", data=payload)
-        api_asserts.assert_status_code_is(tool_response, 200)
-        return tool_response.json()
+        tool_response = self.tools_post(payload)
+        if assert_ok:
+            api_asserts.assert_status_code_is(tool_response, 200)
+            return tool_response.json()
+        else:
+            return tool_response
 
-    def get_history_dataset_content(self, history_id, wait=True, **kwds):
+    def tools_post(self, payload):
+        tool_response = self._post("tools", data=payload)
+        return tool_response
+
+    def get_history_dataset_content(self, history_id, wait=True, filename=None, **kwds):
         dataset_id = self.__history_content_id(history_id, wait=wait, **kwds)
-        display_response = self.__get_contents_request(history_id, "/%s/display" % dataset_id)
+        data = {}
+        if filename:
+            data["filename"] = filename
+        display_response = self.__get_contents_request(history_id, "/%s/display" % dataset_id, data=data)
         assert display_response.status_code == 200, display_response.content
         return display_response.content
 
@@ -212,11 +258,11 @@ class BaseDatasetPopulator(object):
                 history_content_id = history_contents[-1]["id"]
         return history_content_id
 
-    def __get_contents_request(self, history_id, suffix=""):
+    def __get_contents_request(self, history_id, suffix="", data={}):
         url = "histories/%s/contents" % history_id
         if suffix:
             url = "%s%s" % (url, suffix)
-        return self._get(url)
+        return self._get(url, data=data)
 
 
 class DatasetPopulator(BaseDatasetPopulator):
@@ -231,8 +277,8 @@ class DatasetPopulator(BaseDatasetPopulator):
 
         return self.galaxy_interactor.post(route, data, files=files)
 
-    def _get(self, route):
-        return self.galaxy_interactor.get(route)
+    def _get(self, route, data={}):
+        return self.galaxy_interactor.get(route, data=data)
 
     def _summarize_history(self, history_id):
         self.galaxy_interactor._summarize_history(history_id)
@@ -305,8 +351,8 @@ class WorkflowPopulator(BaseWorkflowPopulator, ImporterGalaxyInterface):
     def _post(self, route, data={}):
         return self.galaxy_interactor.post(route, data)
 
-    def _get(self, route):
-        return self.galaxy_interactor.get(route)
+    def _get(self, route, data={}):
+        return self.galaxy_interactor.get(route, data=data)
 
     # Required for ImporterGalaxyInterface interface - so we can recurisvely import
     # nested workflows.
@@ -526,8 +572,8 @@ def wait_on_state(state_func, skip_states=["running", "queued", "new", "ready"],
 class GiPostGetMixin:
     """Mixin for adapting Galaxy testing populators helpers to bioblend."""
 
-    def _get(self, route):
-        return self._gi.make_get_request(self.__url(route))
+    def _get(self, route, data={}):
+        return self._gi.make_get_request(self.__url(route), data)
 
     def _post(self, route, data={}):
         data = data.copy()

test/functional/tools/composite.xml

@@ -10,9 +10,9 @@
   <tests>
     <test>
       <param name="input" value="velveth_test1/output.html" ftype="velvet" >
-        <composite_data value='velveth_test1/Sequences' ftype="Sequences"/>
-        <composite_data value='velveth_test1/Roadmaps' ftype="Roadmaps"/>
-        <composite_data value='velveth_test1/Log'/>
+        <composite_data value="velveth_test1/Sequences"/>
+        <composite_data value="velveth_test1/Roadmaps"/>
+        <composite_data value="velveth_test1/Log"/>
       </param>
       <output name="unused_reads_fasta" file="velveth_test1/Sequences" compare="diff"/>
     </test>

test/functional/tools/composite_output.xml

@@ -10,9 +10,9 @@
   <tests>
     <test>
       <param name="input" value="velveth_test1/output.html" ftype="velvet" >
-        <composite_data value='velveth_test1/Sequences' ftype="Sequences"/>
-        <composite_data value='velveth_test1/Roadmaps' ftype="Roadmaps"/>
-        <composite_data value='velveth_test1/Log'/>
+        <composite_data value="velveth_test1/Sequences" />
+        <composite_data value="velveth_test1/Roadmaps" />
+        <composite_data value="velveth_test1/Log" />
       </param>
       <output name="output" file="velveth_test1/output.html">
         <extra_files type="file" name="Sequences" value="velveth_test1/Sequences" />

test/functional/tools/composite_output_tests.xml

@@ -14,15 +14,15 @@
   <tests>
     <test>
       <param name="input" value="velveth_test1/output.html" ftype="velvet" >
-        <composite_data value='velveth_test1/Sequences' ftype="Sequences"/>
-        <composite_data value='velveth_test1/Roadmaps' ftype="Roadmaps"/>
-        <composite_data value='velveth_test1/Log'/>
+        <composite_data value="velveth_test1/Sequences" />
+        <composite_data value="velveth_test1/Roadmaps" />
+        <composite_data value="velveth_test1/Log"/>
       </param>
       <output name="output" file="velveth_test1/output.html">
         <extra_files type="file" name="Sequences" value="velveth_test1/Sequences" />
         <extra_files type="file" name="Roadmaps" value="velveth_test1/Roadmaps" />
         <extra_files type="file" name="Log" value="composite_output_expected_log" />
-        <extra_files type="file" name="md5out" md5="f2b33fb7b3d0eb95090a16060e6a24f9" /><!-- md5sum or "1 2 3" -->
+        <extra_files type="file" name="md5out" md5="f2b33fb7b3d0eb95090a16060e6a24f9" /><!-- md5sum of "1 2 3" -->
       </output>
     </test>
   </tests>

test/functional/tools/metadata.xml

@@ -10,9 +10,9 @@
   <tests>
     <test>
       <param name="input" value="velveth_test1/output.html" ftype="velvet" >
-        <composite_data value='velveth_test1/Sequences' ftype="Sequences"/>
-        <composite_data value='velveth_test1/Roadmaps' ftype="Roadmaps"/>
-        <composite_data value='velveth_test1/Log'/>
+        <composite_data value="velveth_test1/Sequences" />
+        <composite_data value="velveth_test1/Roadmaps" />
+        <composite_data value="velveth_test1/Log"/>
         <metadata name="base_name" value="Example Metadata" />
       </param>
       <!-- This ouptut tests setting input metadata above -->

tools/sr_assembly/velvetg.xml

@@ -189,9 +189,9 @@
   <tests>
     <test>
       <param name="input" value="velveth_test1/output.html" ftype="velvet" >
-        <composite_data value='velveth_test1/Sequences' ftype="Sequences"/>
-        <composite_data value='velveth_test1/Roadmaps' ftype="Roadmaps"/>
-        <composite_data value='velveth_test1/Log'/>
+        <composite_data value="velveth_test1/Sequences"/>
+        <composite_data value="velveth_test1/Roadmaps"/>
+        <composite_data value="velveth_test1/Log"/>
       </param>
       <param name="afg" value="yes" />
       <param name="generate_unused" value="yes" />