From f469db3d7e26ad2aeb7ee2d8fab7e094f4a339bd Mon Sep 17 00:00:00 2001 From: John Chilton Date: Tue, 19 May 2015 12:35:14 -0400 Subject: [PATCH 1/7] Fixes for boolean & select handling in test data elements. - Allow boolean "truevalue"/"falsevalue" to be used in plain (non-conditional test element) parameters (see https://trello.com/c/iGk3f1pE). - Fix conditionals to use processed values - so this fix and 5b6c092aa0a85455b4daacd318541e92bd08b876 are applid to these elements as well. (http://osdir.com/ml/general/2015-05/msg24402.html). This also adds tests to verify these behaviors. Run most relevant tests with commands: ./run_tests.sh -framework -id simple_constructs ./run_tests.sh -framework -id multi_select ./run_tests.sh -framework -id boolean_conditional --- lib/galaxy/tools/test.py | 83 ++++++++++++------- test/functional/tools/boolean_conditional.xml | 53 ++++++++++++ test/functional/tools/samples_tool_conf.xml | 1 + test/functional/tools/simple_constructs.xml | 12 +++ 4 files changed, 119 insertions(+), 30 deletions(-) create mode 100644 test/functional/tools/boolean_conditional.xml diff --git a/lib/galaxy/tools/test.py b/lib/galaxy/tools/test.py index babdcb84fe2..07837255dc2 100644 --- a/lib/galaxy/tools/test.py +++ b/lib/galaxy/tools/test.py @@ -71,9 +71,8 @@ class ToolTestBuilder( object ): # No explicit value for param in test case, determine from default query_value = test_param.checked else: - # Test case supplied value, check cases against this. - query_value = string_as_bool( declared_value ) - matches_declared_value = lambda case_value: string_as_bool(case_value) == query_value + query_value = _process_bool_param_value( test_param, declared_value ) + matches_declared_value = lambda case_value: _process_bool_param_value( test_param, case_value ) == query_value elif isinstance(test_param, galaxy.tools.parameters.basic.SelectToolParameter): if declared_value is not None: # Test case supplied explicit value to check against. @@ -166,7 +165,8 @@ class ToolTestBuilder( object ): # as a new instance with value defined and hence enter # an infinite loop - hence the "case_value is not None" # check. - expanded_inputs[ case_context.for_state() ] = expanded_case_value + processed_value = _process_simple_value( value.test_param, expanded_case_value ) + expanded_inputs[ case_context.for_state() ] = processed_value elif isinstance( value, galaxy.tools.parameters.grouping.Section ): context = ParamContext( name=value.name, parent_context=parent_context ) for r_name, r_value in value.inputs.iteritems(): @@ -203,33 +203,8 @@ class ToolTestBuilder( object ): for ( name, value, extra ) in collection_def.collect_inputs(): require_file( name, value, extra, self.required_files ) processed_value = collection_def - elif isinstance( value, galaxy.tools.parameters.basic.SelectToolParameter ) and hasattr( value, 'static_options' ): - # Tests may specify values as either raw value or the value - # as they appear in the list - the API doesn't and shouldn't - # accept the text value - so we need to convert the text - # into the form value. - def process_param_value( param_value ): - found_value = False - value_for_text = None - if value.static_options: - for (text, opt_value, selected) in value.static_options: - if param_value == opt_value: - found_value = True - if value_for_text is None and param_value == text: - value_for_text = opt_value - if not found_value and value_for_text is not None: - processed_value = value_for_text - else: - processed_value = param_value - return processed_value - # Do replacement described above for lists or singleton - # values. - if isinstance( param_value, list ): - processed_value = map( process_param_value, param_value ) - else: - processed_value = process_param_value( param_value ) else: - processed_value = param_value + processed_value = _process_simple_value( value, param_value ) expanded_inputs[ context.for_state() ] = processed_value return expanded_inputs @@ -240,6 +215,54 @@ class ToolTestBuilder( object ): return require_file( name, value, extra, self.required_files ) +def _process_simple_value( param, param_value ): + if isinstance( param, galaxy.tools.parameters.basic.SelectToolParameter ) and hasattr( param, 'static_options' ): + # Tests may specify values as either raw value or the value + # as they appear in the list - the API doesn't and shouldn't + # accept the text value - so we need to convert the text + # into the form value. + def process_param_value( param_value ): + found_value = False + value_for_text = None + if param.static_options: + for (text, opt_value, selected) in param.static_options: + if param_value == opt_value: + found_value = True + if value_for_text is None and param_value == text: + value_for_text = opt_value + if not found_value and value_for_text is not None: + processed_value = value_for_text + else: + processed_value = param_value + return processed_value + # Do replacement described above for lists or singleton + # values. + if isinstance( param_value, list ): + processed_value = map( process_param_value, param_value ) + else: + processed_value = process_param_value( param_value ) + elif isinstance( param, galaxy.tools.parameters.basic.BooleanToolParameter ): + # Like above, tests may use the tool define values of simply + # true/false. + processed_value = _process_bool_param_value( param, param_value ) + else: + processed_value = param_value + return processed_value + + +def _process_bool_param_value( param, param_value ): + assert isinstance( param, galaxy.tools.parameters.basic.BooleanToolParameter ) + if isinstance( param_value, list ): + param_value = param_value[0] + if param.truevalue == param_value: + processed_value = True + elif param.falsevalue == param_value: + processed_value = False + else: + processed_value = string_as_bool( param_value ) + return processed_value + + @nottest def test_data_iter( required_files ): for fname, extra in required_files: diff --git a/test/functional/tools/boolean_conditional.xml b/test/functional/tools/boolean_conditional.xml new file mode 100644 index 00000000000..df699e267a8 --- /dev/null +++ b/test/functional/tools/boolean_conditional.xml @@ -0,0 +1,53 @@ + + + echo "$p1.p1val" >> $out_file1; + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/test/functional/tools/samples_tool_conf.xml b/test/functional/tools/samples_tool_conf.xml index c4c86f5272f..948793afa9b 100644 --- a/test/functional/tools/samples_tool_conf.xml +++ b/test/functional/tools/samples_tool_conf.xml @@ -3,6 +3,7 @@ + diff --git a/test/functional/tools/simple_constructs.xml b/test/functional/tools/simple_constructs.xml index d67c0c282ee..6fdde0ff154 100644 --- a/test/functional/tools/simple_constructs.xml +++ b/test/functional/tools/simple_constructs.xml @@ -61,5 +61,17 @@ + + + + + + + + + + + + From 182fbd1bc61f8ff8bd2cf29c23fca47c8eac7002 Mon Sep 17 00:00:00 2001 From: Martin Cech Date: Tue, 19 May 2015 16:53:25 -0400 Subject: [PATCH 2/7] fix the data source tools for modendcode with proper URLs --- tools/data_source/fly_modencode.xml | 2 +- tools/data_source/worm_modencode.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/data_source/fly_modencode.xml b/tools/data_source/fly_modencode.xml index 7b14302ebec..70e080eec48 100644 --- a/tools/data_source/fly_modencode.xml +++ b/tools/data_source/fly_modencode.xml @@ -2,7 +2,7 @@ server data_source.py $output $__app__.config.output_size_limit - + go to modENCODE fly server $GALAXY_URL diff --git a/tools/data_source/worm_modencode.xml b/tools/data_source/worm_modencode.xml index 074792cccf7..b2ade3a0e51 100644 --- a/tools/data_source/worm_modencode.xml +++ b/tools/data_source/worm_modencode.xml @@ -2,7 +2,7 @@ server data_source.py $output $__app__.config.output_size_limit - + go to modENCODE worm server $GALAXY_URL From 0d3c849d4f90e30c793f1ae61203daf7842a0989 Mon Sep 17 00:00:00 2001 From: abretaud Date: Fri, 24 Apr 2015 14:37:43 +0200 Subject: [PATCH 3/7] perf optimization: store tools' section in a dict instead of looping through the whole list of sections/tools for each tool (cherry picked from commit b8261db255759a4725fd30de280fae19ee1a375b) --- lib/galaxy/tools/toolbox/base.py | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/lib/galaxy/tools/toolbox/base.py b/lib/galaxy/tools/toolbox/base.py index 9656dd147ed..3b441704af8 100644 --- a/lib/galaxy/tools/toolbox/base.py +++ b/lib/galaxy/tools/toolbox/base.py @@ -53,6 +53,7 @@ class AbstractToolBox( object, Dictifiable, ManagesIntegratedToolPanelMixin ): # shed_tool_conf.xml file. self._dynamic_tool_confs = [] self._tools_by_id = {} + self._integrated_section_by_tool = {} # Tool lineages can contain chains of related tools with different ids # so each will be present once in the above dictionary. The following # dictionary can instead hold multiple tools with different versions. @@ -217,19 +218,10 @@ class AbstractToolBox( object, Dictifiable, ManagesIntegratedToolPanelMixin ): def get_integrated_section_for_tool( self, tool ): tool_id = tool.id - for key, item_type, item in self._integrated_tool_panel.panel_items_iter(): - if item: - if item_type == panel_item_types.TOOL: - if item.id == tool_id: - return '', '' - if item_type == panel_item_types.SECTION: - section_id = item.id or '' - section_name = item.name or '' - for section_key, section_item_type, section_item in item.panel_items_iter(): - if section_item_type == panel_item_types.TOOL: - if section_item: - if section_item.id == tool_id: - return section_id, section_name + + if tool_id in self._integrated_section_by_tool: + return self._integrated_section_by_tool[tool_id] + return None, None def __resolve_tool_path(self, tool_path, config_filename): @@ -310,6 +302,7 @@ class AbstractToolBox( object, Dictifiable, ManagesIntegratedToolPanelMixin ): tool_id = key.replace( 'tool_', '', 1 ) if tool_id in self._tools_by_id: self.__add_tool_to_tool_panel( val, self._tool_panel, section=False ) + self._integrated_section_by_tool[tool_id] = '', '' elif item_type == panel_item_types.WORKFLOW: workflow_id = key.replace( 'workflow_', '', 1 ) if workflow_id in self._workflows_by_id: @@ -331,6 +324,7 @@ class AbstractToolBox( object, Dictifiable, ManagesIntegratedToolPanelMixin ): tool_id = section_key.replace( 'tool_', '', 1 ) if tool_id in self._tools_by_id: self.__add_tool_to_tool_panel( section_val, section, section=True ) + self._integrated_section_by_tool[tool_id] = key, val.name elif section_item_type == panel_item_types.WORKFLOW: workflow_id = section_key.replace( 'workflow_', '', 1 ) if workflow_id in self._workflows_by_id: @@ -561,6 +555,9 @@ class AbstractToolBox( object, Dictifiable, ManagesIntegratedToolPanelMixin ): if tool_key in itegrated_items: del itegrated_items[ tool_key ] + if tool_id in self._integrated_section_by_tool: + del self._integrated_section_by_tool[ tool_id ] + if section_key: _, tool_section = self.get_section( section_key ) if tool_section: @@ -934,6 +931,10 @@ class AbstractToolBox( object, Dictifiable, ManagesIntegratedToolPanelMixin ): break if tool_id in self.data_manager_tools: del self.data_manager_tools[ tool_id ] + + if tool_id in self._integrated_section_by_tool: + del self._integrated_section_by_tool[ tool_id ] + # TODO: do we need to manually remove from the integrated panel here? message = "Removed the tool:
" message += "name: %s
" % tool.name From c5675307231b1e15d95e2ca35b00f9be5d7ff3f6 Mon Sep 17 00:00:00 2001 From: abretaud Date: Mon, 27 Apr 2015 11:44:30 +0200 Subject: [PATCH 4/7] handle adding/removing tools from toolshed (cherry picked from commit 911acd7a6ed364f2124e8673a02849f7922a0325) --- lib/galaxy/tools/toolbox/base.py | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/lib/galaxy/tools/toolbox/base.py b/lib/galaxy/tools/toolbox/base.py index 3b441704af8..1a04fdf4b74 100644 --- a/lib/galaxy/tools/toolbox/base.py +++ b/lib/galaxy/tools/toolbox/base.py @@ -546,18 +546,22 @@ class AbstractToolBox( object, Dictifiable, ManagesIntegratedToolPanelMixin ): has_elems.insert( tool_panel_index, replacement_tool_key, replacement_tool_version ) + self._integrated_section_by_tool[ tool_id ] = available_tool_section_id, available_tool_section_name else: del has_elems[ tool_key ] + + if tool_id in self._integrated_section_by_tool: + del self._integrated_section_by_tool[ tool_id ] else: del has_elems[ tool_key ] + + if tool_id in self._integrated_section_by_tool: + del self._integrated_section_by_tool[ tool_id ] if remove_from_config: itegrated_items = integrated_has_elems.panel_items() if tool_key in itegrated_items: del itegrated_items[ tool_key ] - if tool_id in self._integrated_section_by_tool: - del self._integrated_section_by_tool[ tool_id ] - if section_key: _, tool_section = self.get_section( section_key ) if tool_section: @@ -678,6 +682,14 @@ class AbstractToolBox( object, Dictifiable, ManagesIntegratedToolPanelMixin ): index=sub_index, internal=True, ) + + # Ensure each tool's section is stored + for section_key, section_item_type, section_item in integrated_elems.panel_items_iter(): + if section_item_type == panel_item_types.TOOL: + if section_item: + tool_id = section_key.replace( 'tool_', '', 1 ) + self._integrated_section_by_tool[tool_id] = integrated_section.id, integrated_section.name + if load_panel_dict: self._tool_panel[ key ] = section # Always load sections into the integrated_tool_panel. @@ -931,10 +943,6 @@ class AbstractToolBox( object, Dictifiable, ManagesIntegratedToolPanelMixin ): break if tool_id in self.data_manager_tools: del self.data_manager_tools[ tool_id ] - - if tool_id in self._integrated_section_by_tool: - del self._integrated_section_by_tool[ tool_id ] - # TODO: do we need to manually remove from the integrated panel here? message = "Removed the tool:
" message += "name: %s
" % tool.name From 93c51fa25986d0cc2c0939879fc9e29e5e82568a Mon Sep 17 00:00:00 2001 From: Daniel Blankenberg Date: Wed, 20 May 2015 12:39:59 -0400 Subject: [PATCH 5/7] Update default dbkey for fly_modencode.xml and bump version. dm3 is UCSC for release 5 --- tools/data_source/fly_modencode.xml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tools/data_source/fly_modencode.xml b/tools/data_source/fly_modencode.xml index 70e080eec48..3d3f5b8e533 100644 --- a/tools/data_source/fly_modencode.xml +++ b/tools/data_source/fly_modencode.xml @@ -1,5 +1,5 @@ - + server data_source.py $output $__app__.config.output_size_limit @@ -7,15 +7,15 @@ - + - + - + From af6172d14d343e2e53fb757c93bbbab03edbfc6b Mon Sep 17 00:00:00 2001 From: Daniel Blankenberg Date: Wed, 20 May 2015 12:42:05 -0400 Subject: [PATCH 6/7] Update default dbkey for worm_modencode.xml and bump version. ce10 is UCSC for WS220 --- tools/data_source/worm_modencode.xml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tools/data_source/worm_modencode.xml b/tools/data_source/worm_modencode.xml index b2ade3a0e51..b021f4a0ddf 100644 --- a/tools/data_source/worm_modencode.xml +++ b/tools/data_source/worm_modencode.xml @@ -1,5 +1,5 @@ - + server data_source.py $output $__app__.config.output_size_limit @@ -7,15 +7,15 @@ - + - + - + From 8d2f9f77ebbdd4d2134fcd55304bb4aa1313f10b Mon Sep 17 00:00:00 2001 From: Daniel Blankenberg Date: Wed, 20 May 2015 12:47:01 -0400 Subject: [PATCH 7/7] Update modencode dbkeys in gbrowse_build_sites.txt --- tool-data/shared/gbrowse/gbrowse_build_sites.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tool-data/shared/gbrowse/gbrowse_build_sites.txt b/tool-data/shared/gbrowse/gbrowse_build_sites.txt index 289af693b69..df21421f090 100644 --- a/tool-data/shared/gbrowse/gbrowse_build_sites.txt +++ b/tool-data/shared/gbrowse/gbrowse_build_sites.txt @@ -18,7 +18,7 @@ wormbase_ws225 WormBase WS225 http://ws225.wormbase.org/db/gb2/gbrowse/ caeAng1, tair tair http://arabidopsis.org/cgi-bin/gbrowse/ arabidopsis_tair8,arabidopsis,Arabidopsis_thaliana_TAIR10 arabidopsis_tair8,arabidopsis_tair9,arabidopsis #modENCODE -modencode modENCODE http://modencode.oicr.on.ca/fgb2/gbrowse/ ce6,dm2 worm,fly +modencode modENCODE http://gbrowse.modencode.org/fgb2/gbrowse/ ce10,dm3 worm,fly #Saccharomyces Genome Database (SGD) sgd_yeast Saccharomyces Genome Database http://browse.yeastgenome.org/fgb2/gbrowse/ Saccharomyces_cerevisiae_S288C_SGD2010 scgenome