From 5a7a15d161bececf0a3cd93714e15768b329d95f Mon Sep 17 00:00:00 2001 From: John Chilton Date: Mon, 28 Aug 2017 09:49:41 -0400 Subject: [PATCH] Test and clarifications for composite uploads. - Add an API test for datatype-defined composite uploads - including exercising newline conversion and the space_to_tab parameter. - Add a test decorator skip_without_datatype to mirror skip_without_tool for this test, improve both decorators. - The ftype parameter in the composite test tools does nothing - drop it and drop it from the XSD spec. - Slightly improve the documentation for these composite_data elements in the XSD. --- lib/galaxy/tools/xsd/galaxy.xsd | 12 ++-- test/api/test_tools.py | 68 ++++++++++++++++++- test/base/interactor.py | 3 - test/base/populators.py | 45 +++++++++--- test/functional/tools/composite.xml | 6 +- test/functional/tools/composite_output.xml | 6 +- .../tools/composite_output_tests.xml | 8 +-- test/functional/tools/metadata.xml | 6 +- tools/sr_assembly/velvetg.xml | 6 +- 9 files changed, 125 insertions(+), 35 deletions(-) diff --git a/lib/galaxy/tools/xsd/galaxy.xsd b/lib/galaxy/tools/xsd/galaxy.xsd index d32b0f60e5e..40ae2fa7fe3 100644 --- a/lib/galaxy/tools/xsd/galaxy.xsd +++ b/lib/galaxy/tools/xsd/galaxy.xsd @@ -1062,18 +1062,18 @@ of ``type`` ``data``. - Define extra composite input files for test input. + Define extra composite input files for test +input. The specified ``ftype`` on the parent ``param`` should specify a composite +datatype with defined static composite files. The order of the defined composite +files on the datatype must match the order specified with these elements and All +non-optional composite inputs must be specified as part of the ``param``. + Path relative to test-data of composite file. - - - Optional datatype of composite file for test input. - - diff --git a/test/api/test_tools.py b/test/api/test_tools.py index 435b54d477f..0dcfe546bac 100644 --- a/test/api/test_tools.py +++ b/test/api/test_tools.py @@ -6,7 +6,8 @@ from base.populators import ( DatasetCollectionPopulator, DatasetPopulator, LibraryPopulator, - skip_without_tool + skip_without_tool, + skip_without_datatype, ) from galaxy.tools.verify.test_data import TestDataResolver @@ -119,6 +120,71 @@ class ToolsTestCase(api.ApiTestCase): rdata_metadata = self._upload_and_get_details(open(rdata_path, "rb"), file_type="auto") self.assertEquals(rdata_metadata["file_ext"], "rdata") + @skip_without_datatype("velvet") + def test_composite_datatype(self): + with self.dataset_populator.test_history() as history_id: + dataset = self._velvet_upload(history_id, extra_inputs={ + "files_1|url_paste": "roadmaps content", + "files_1|type": "upload_dataset", + "files_2|url_paste": "log content", + "files_2|type": "upload_dataset", + }) + + roadmaps_content = self._get_roadmaps_content(history_id, dataset) + assert roadmaps_content.strip() == "roadmaps content", roadmaps_content + + @skip_without_datatype("velvet") + def test_composite_datatype_space_to_tab(self): + # Like previous test but set one upload with space_to_tab to True to + # verify that works. + with self.dataset_populator.test_history() as history_id: + dataset = self._velvet_upload(history_id, extra_inputs={ + "files_1|url_paste": "roadmaps content", + "files_1|type": "upload_dataset", + "files_1|space_to_tab": "Yes", + "files_2|url_paste": "log content", + "files_2|type": "upload_dataset", + }) + + roadmaps_content = self._get_roadmaps_content(history_id, dataset) + assert roadmaps_content.strip() == "roadmaps\tcontent", roadmaps_content + + @skip_without_datatype("velvet") + def test_composite_datatype_posix_lines(self): + # Like previous test but set one upload with space_to_tab to True to + # verify that works. + with self.dataset_populator.test_history() as history_id: + dataset = self._velvet_upload(history_id, extra_inputs={ + "files_1|url_paste": "roadmaps\rcontent", + "files_1|type": "upload_dataset", + "files_1|space_to_tab": "Yes", + "files_2|url_paste": "log\rcontent", + "files_2|type": "upload_dataset", + }) + + roadmaps_content = self._get_roadmaps_content(history_id, dataset) + assert roadmaps_content.strip() == "roadmaps\ncontent", roadmaps_content + + def _velvet_upload(self, history_id, extra_inputs): + payload = self.dataset_populator.upload_payload( + history_id, + "sequences content", + file_type="velvet", + extra_inputs=extra_inputs, + ) + run_response = self.dataset_populator.tools_post(payload) + self.dataset_populator.wait_for_tool_run(history_id, run_response) + datasets = run_response.json()["outputs"] + + assert len(datasets) == 1 + dataset = datasets[0] + + return dataset + + def _get_roadmaps_content(self, history_id, dataset): + roadmaps_content = self.dataset_populator.get_history_dataset_content(history_id, dataset=dataset, filename="Roadmaps") + return roadmaps_content + def test_unzip_collection(self): with self.dataset_populator.test_history() as history_id: hdca_id = self.__build_pair(history_id, ["123", "456"]) diff --git a/test/base/interactor.py b/test/base/interactor.py index 1433169854c..736527477f5 100644 --- a/test/base/interactor.py +++ b/test/base/interactor.py @@ -188,10 +188,7 @@ class GalaxyInteractorApi(object): file_name = self.functional_test_case.get_filename(composite_file.get('value'), shed_tool_id=shed_tool_id) files["files_%s|file_data" % i] = open(file_name, 'rb') tool_input.update({ - # "files_%d|NAME" % i: name, "files_%d|type" % i: "upload_dataset", - # TODO: - # "files_%d|space_to_tab" % i: composite_file.get( 'space_to_tab', False ) }) name = test_data['name'] else: diff --git a/test/base/populators.py b/test/base/populators.py index 4c1a9d58ced..169b9935cfb 100644 --- a/test/base/populators.py +++ b/test/base/populators.py @@ -2,6 +2,7 @@ import contextlib import json import time +from functools import wraps from operator import itemgetter import requests @@ -26,8 +27,9 @@ DEFAULT_TIMEOUT = 60 # Secs to wait for state to turn ok def skip_without_tool(tool_id): - """ Decorate an API test method as requiring a specific tool, - have nose skip the test case is the tool is unavailable. + """Decorate an API test method as requiring a specific tool. + + Have test framework skip the test case is the tool is unavailable. """ def method_wrapper(method): @@ -39,21 +41,46 @@ def skip_without_tool(tool_id): tool_ids = [itemgetter("id")(_) for _ in tools] return tool_ids + @wraps(method) def wrapped_method(api_test_case, *args, **kwargs): - if tool_id not in get_tool_ids(api_test_case): - from nose.plugins.skip import SkipTest - raise SkipTest() - + _raise_skip_if(tool_id not in get_tool_ids(api_test_case)) return method(api_test_case, *args, **kwargs) - # Must preserve method name so nose can detect and report tests by - # name. - wrapped_method.__name__ = method.__name__ return wrapped_method return method_wrapper +def skip_without_datatype(extension): + """Decorate an API test method as requiring a specific datatype. + + Have test framework skip the test case is the tool is unavailable. + """ + + def has_datatype(api_test_case): + index_response = api_test_case.galaxy_interactor.get("datatypes") + assert index_response.status_code == 200, "Failed to fetch datatypes for target Galaxy." + datatypes = index_response.json() + assert isinstance(datatypes, list) + return extension in datatypes + + def method_wrapper(method): + @wraps(method) + def wrapped_method(api_test_case, *args, **kwargs): + _raise_skip_if(not has_datatype(api_test_case)) + method(api_test_case, *args, **kwargs) + + return wrapped_method + + return method_wrapper + + +def _raise_skip_if(check): + if check: + from nose.plugins.skip import SkipTest + raise SkipTest() + + # Deprecated mixin, use dataset populator instead. # TODO: Rework existing tests to target DatasetPopulator in a setup method instead. class TestsDatasets: diff --git a/test/functional/tools/composite.xml b/test/functional/tools/composite.xml index 8f8edbe58f3..f4c5e5bfc1b 100644 --- a/test/functional/tools/composite.xml +++ b/test/functional/tools/composite.xml @@ -10,9 +10,9 @@ - - - + + + diff --git a/test/functional/tools/composite_output.xml b/test/functional/tools/composite_output.xml index 6bb1bf1e78f..cc29995c1e3 100644 --- a/test/functional/tools/composite_output.xml +++ b/test/functional/tools/composite_output.xml @@ -10,9 +10,9 @@ - - - + + + diff --git a/test/functional/tools/composite_output_tests.xml b/test/functional/tools/composite_output_tests.xml index d12a5beb5c2..686743264d0 100644 --- a/test/functional/tools/composite_output_tests.xml +++ b/test/functional/tools/composite_output_tests.xml @@ -14,15 +14,15 @@ - - - + + + - + diff --git a/test/functional/tools/metadata.xml b/test/functional/tools/metadata.xml index cc68946ec0d..e9268b9f01c 100644 --- a/test/functional/tools/metadata.xml +++ b/test/functional/tools/metadata.xml @@ -10,9 +10,9 @@ - - - + + + diff --git a/tools/sr_assembly/velvetg.xml b/tools/sr_assembly/velvetg.xml index 9feaed56a19..4c27e829bd7 100644 --- a/tools/sr_assembly/velvetg.xml +++ b/tools/sr_assembly/velvetg.xml @@ -189,9 +189,9 @@ - - - + + +