From 6fd77db920d088ba6ebc64a4ac8ab7524d5ce654 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Tue, 22 Nov 2016 18:09:34 +0100 Subject: [PATCH 1/5] Preliminary fix for mapping tools with paired collection input over list:paired collections. --- lib/galaxy/managers/collections.py | 8 +++- lib/galaxy/tools/actions/__init__.py | 10 ++++- ...ed_collection_map_over_structured_like.xml | 44 +++++++++++++++++++ test/functional/tools/samples_tool_conf.xml | 1 + 4 files changed, 61 insertions(+), 2 deletions(-) create mode 100644 test/functional/tools/paired_collection_map_over_structured_like.xml diff --git a/lib/galaxy/managers/collections.py b/lib/galaxy/managers/collections.py index 24dbdff1c13..aadaf226439 100644 --- a/lib/galaxy/managers/collections.py +++ b/lib/galaxy/managers/collections.py @@ -126,7 +126,13 @@ class DatasetCollectionManager( object ): if collection_type_description.has_subcollections( ): # Nested collection - recursively create collections and update identifiers. self.__recursively_create_collections( trans, element_identifiers ) - elements = self.__load_elements( trans, element_identifiers ) + new_collection = False + for element_identifier in element_identifiers: + if element_identifier.get("src") == "new_collection" and element_identifier.get('collection_type') == '': + new_collection = True + elements = self.__load_elements(trans, element_identifier['element_identifiers']) + if not new_collection: + elements = self.__load_elements( trans, element_identifiers ) # else if elements is set, it better be an ordered dict! if elements is not self.ELEMENTS_UNINITIALIZED: diff --git a/lib/galaxy/tools/actions/__init__.py b/lib/galaxy/tools/actions/__init__.py index e4ff04a46c2..ddefd7ffe52 100644 --- a/lib/galaxy/tools/actions/__init__.py +++ b/lib/galaxy/tools/actions/__init__.py @@ -770,7 +770,15 @@ def determine_output_format(output, parameter_context, input_datasets, input_dat if collection_name in input_dataset_collections: try: input_collection = input_dataset_collections[collection_name][0][0] - input_dataset = input_collection.collection[element_index].element_object + input_collection_collection = input_collection.collection + try: + input_element = input_collection_collection[element_index] + except KeyError: + for element in input_collection_collection.dataset_elements: + if element.element_identifier == element_index: + input_element = element + break + input_dataset = input_element.element_object input_extension = input_dataset.ext ext = input_extension except Exception as e: diff --git a/test/functional/tools/paired_collection_map_over_structured_like.xml b/test/functional/tools/paired_collection_map_over_structured_like.xml new file mode 100644 index 00000000000..fef12be4696 --- /dev/null +++ b/test/functional/tools/paired_collection_map_over_structured_like.xml @@ -0,0 +1,44 @@ + + + + cat '${input1.forward}' > '${list_output.forward}'; + cat '${input1.reverse}' > '${list_output.reverse}' + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/test/functional/tools/samples_tool_conf.xml b/test/functional/tools/samples_tool_conf.xml index 54e6ad0fc30..c78d1595a7d 100644 --- a/test/functional/tools/samples_tool_conf.xml +++ b/test/functional/tools/samples_tool_conf.xml @@ -78,6 +78,7 @@ + From 1aba7549debca136048d2850dae28b1185e30469 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Tue, 22 Nov 2016 18:47:48 +0100 Subject: [PATCH 2/5] Add inherit_format tag to paired_collection_map_over_structured_like tool --- .../tools/paired_collection_map_over_structured_like.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/functional/tools/paired_collection_map_over_structured_like.xml b/test/functional/tools/paired_collection_map_over_structured_like.xml index fef12be4696..406a6833c0a 100644 --- a/test/functional/tools/paired_collection_map_over_structured_like.xml +++ b/test/functional/tools/paired_collection_map_over_structured_like.xml @@ -10,7 +10,7 @@ - + From 55eac47d0fe9083c84afc613f451bfa68649b4d8 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Tue, 22 Nov 2016 20:17:13 +0100 Subject: [PATCH 3/5] Fix test tool -- functional test framework appears to not support mapping over test-data --- ...ed_collection_map_over_structured_like.xml | 22 +++++-------------- 1 file changed, 5 insertions(+), 17 deletions(-) diff --git a/test/functional/tools/paired_collection_map_over_structured_like.xml b/test/functional/tools/paired_collection_map_over_structured_like.xml index 406a6833c0a..cbd03f31df3 100644 --- a/test/functional/tools/paired_collection_map_over_structured_like.xml +++ b/test/functional/tools/paired_collection_map_over_structured_like.xml @@ -18,26 +18,14 @@ - - - - - - - + + + - - - - - - - - - - + + From a39941a3c57f0cf1d0db848debfed4d54c321627 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Tue, 22 Nov 2016 21:40:02 +0100 Subject: [PATCH 4/5] Add failing test for structured_like paired input --- test/api/test_tools.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/test/api/test_tools.py b/test/api/test_tools.py index 2db9ed5e2c8..cdf0cbb6030 100644 --- a/test/api/test_tools.py +++ b/test/api/test_tools.py @@ -846,6 +846,15 @@ class ToolsTestCase( api.ApiTestCase ): } self._check_simple_cat1_over_nested_collections( history_id, inputs ) + @skip_without_tool( "paired_collection_map_over_structured_like" ) + def test_paired_input_map_over_nested_collections( self ): + history_id = self.dataset_populator.new_history() + hdca_id = self.__build_pair( history_id, ["123", "456"] ) + inputs = { + "input1": { 'batch': True, 'values': [ dict( src="hdca", id=hdca_id ) ] }, + } + create = self._run( "paired_collection_map_over_structured_like", history_id, inputs, assert_ok=True ) + def _check_simple_cat1_over_nested_collections( self, history_id, inputs ): create = self._run_cat1( history_id, inputs=inputs, assert_ok=True ) outputs = create[ 'outputs' ] From cbd9f68f760178b113b624078372f955b486d32e Mon Sep 17 00:00:00 2001 From: John Chilton Date: Fri, 16 Dec 2016 09:17:08 -0500 Subject: [PATCH 5/5] Fixup test case for #3209. --- test/api/test_tools.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/test/api/test_tools.py b/test/api/test_tools.py index cdf0cbb6030..c5db4f6a1d7 100644 --- a/test/api/test_tools.py +++ b/test/api/test_tools.py @@ -849,11 +849,20 @@ class ToolsTestCase( api.ApiTestCase ): @skip_without_tool( "paired_collection_map_over_structured_like" ) def test_paired_input_map_over_nested_collections( self ): history_id = self.dataset_populator.new_history() - hdca_id = self.__build_pair( history_id, ["123", "456"] ) + hdca_id = self.__build_nested_list( history_id ) inputs = { - "input1": { 'batch': True, 'values': [ dict( src="hdca", id=hdca_id ) ] }, + "input1": { 'batch': True, 'values': [ dict( map_over_type='paired', src="hdca", id=hdca_id ) ] }, } + self.dataset_populator.wait_for_history( history_id, assert_ok=True ) create = self._run( "paired_collection_map_over_structured_like", history_id, inputs, assert_ok=True ) + jobs = create[ 'jobs' ] + implicit_collections = create[ 'implicit_collections' ] + self.assertEquals( len( jobs ), 2 ) + self.assertEquals( len( implicit_collections ), 1 ) + implicit_collection = implicit_collections[ 0 ] + assert implicit_collection[ "collection_type" ] == "list:paired", implicit_collection + outer_elements = implicit_collection[ "elements" ] + assert len( outer_elements ) == 2 def _check_simple_cat1_over_nested_collections( self, history_id, inputs ): create = self._run_cat1( history_id, inputs=inputs, assert_ok=True )