From d5d404c912340013aab34fe3350a2570010b0898 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Tue, 29 Dec 2015 15:09:49 +0000 Subject: [PATCH] Implement special config file to dump parameters as json. ``` ``` Version is specified as 1, because the Galaxy tool plumbing doesn't really distinguish between parameter is '', parameter was explicitly supplied as None, parameter was absent, etc... - everything is a just a string in some ways. This distinction may be more important with JSON and for instance the CWL will require more percision, so we may want to revise Galaxy's tool handling in a backward compatible way someday. --- lib/galaxy/tools/__init__.py | 15 +++- lib/galaxy/tools/evaluation.py | 23 ++++-- lib/galaxy/tools/parameters/wrapped_json.py | 72 +++++++++++++++++++ test/functional/tools/inputs_as_json.xml | 79 +++++++++++++++++++++ test/functional/tools/samples_tool_conf.xml | 1 + 5 files changed, 184 insertions(+), 6 deletions(-) create mode 100644 lib/galaxy/tools/parameters/wrapped_json.py create mode 100644 test/functional/tools/inputs_as_json.xml diff --git a/lib/galaxy/tools/__init__.py b/lib/galaxy/tools/__init__.py index 8c12d282533..d80b279f87b 100755 --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -641,11 +641,22 @@ class Tool( object, Dictifiable ): root = tool_source.root conf_parent_elem = root.find("configfiles") if conf_parent_elem is not None: + inputs_elem = conf_parent_elem.find( "inputs" ) + if inputs_elem is not None: + name = inputs_elem.get( "name" ) + filename = inputs_elem.get( "filename", None ) + format = inputs_elem.get("format", "json") + version_str = inputs_elem.get("version", "1") + if not version_str: + raise ValueError("inputs configfile tag must define a version attribute") + version = int(version_str) + content = dict(format=format, version=version) + self.config_files.append( ( name, filename, content ) ) for conf_elem in conf_parent_elem.findall( "configfile" ): name = conf_elem.get( "name" ) filename = conf_elem.get( "filename", None ) - text = conf_elem.text - self.config_files.append( ( name, filename, text ) ) + content = conf_elem.text + self.config_files.append( ( name, filename, content ) ) def __parse_trackster_conf(self, tool_source): self.trackster_conf = None diff --git a/lib/galaxy/tools/evaluation.py b/lib/galaxy/tools/evaluation.py index 72da979a5d9..7f13c1a7df1 100644 --- a/lib/galaxy/tools/evaluation.py +++ b/lib/galaxy/tools/evaluation.py @@ -1,3 +1,4 @@ +import json import os import tempfile @@ -20,6 +21,7 @@ from galaxy.tools.parameters.basic import ( DataCollectionToolParameter, SelectToolParameter, ) +from galaxy.tools.parameters import wrapped_json from galaxy.tools.parameters.grouping import Conditional, Repeat, Section from galaxy.tools import global_tool_errors from galaxy.jobs.datasets import dataset_path_rewrites @@ -472,7 +474,8 @@ class ToolEvaluator( object ): """ param_dict = self.param_dict config_filenames = [] - for name, filename, template_text in self.tool.config_files: + for name, filename, content in self.tool.config_files: + config_text, is_template = self.__build_config_file_text(content) # If a particular filename was forced by the config use it directory = self.local_working_directory if filename is not None: @@ -480,7 +483,7 @@ class ToolEvaluator( object ): else: fd, config_filename = tempfile.mkstemp( dir=directory ) os.close( fd ) - self.__write_workdir_file( config_filename, template_text, param_dict ) + self.__write_workdir_file( config_filename, config_text, param_dict, is_template=is_template ) self.__register_extra_file( name, config_filename ) config_filenames.append( config_filename ) return config_filenames @@ -526,8 +529,20 @@ class ToolEvaluator( object ): else: return None - def __write_workdir_file( self, config_filename, template, context ): - value = fill_template( template, context=context ) + def __build_config_file_text( self, content ): + if isinstance( content, basestring ): + return content, True + + assert content["format"] == "json" + assert content["version"] == 1 + + return json.dumps(wrapped_json.json_wrap(self.tool.inputs, self.param_dict)), False + + def __write_workdir_file( self, config_filename, content, context, is_template=True ): + if is_template: + value = fill_template( content, context=context ) + else: + value = content with open( config_filename, "w" ) as f: f.write( value ) # For running jobs as the actual user, ensure the config file is globally readable diff --git a/lib/galaxy/tools/parameters/wrapped_json.py b/lib/galaxy/tools/parameters/wrapped_json.py new file mode 100644 index 00000000000..e49cee48622 --- /dev/null +++ b/lib/galaxy/tools/parameters/wrapped_json.py @@ -0,0 +1,72 @@ +import logging +log = logging.getLogger(__name__) + + +def json_wrap(inputs, input_values, as_dict=None, handle_files="SKIP"): + if as_dict is None: + as_dict = {} + + for input in inputs.itervalues(): + input_name = input.name + input_type = input.type + value = input_values[input_name] + if input_type == "repeat": + repeat_job_value = [] + for d in input_values[input.name]: + repeat_instance_job_value = {} + json_wrap(input.inputs, d, repeat_instance_job_value) + repeat_job_value.append(repeat_instance_job_value) + as_dict[input_name] = repeat_job_value + if input_type == "conditional": + values = input_values[input_name] + current = values["__current_case__"] + conditional_job_value = {} + json_wrap(input.cases[current].inputs, values, conditional_job_value) + as_dict[input_name] = conditional_job_value + if input_type == "section": + values = input_values[input_name] + section_job_value = {} + json_wrap(input.inputs, values, section_job_value) + as_dict[input_name] = section_job_value + elif input_type == "data" and input.multiple: + if handle_files == "SKIP": + continue + raise NotImplementedError() + elif input_type == "data": + if handle_files == "SKIP": + continue + raise NotImplementedError() + elif input_type == "data_collection": + if handle_files == "SKIP": + continue + raise NotImplementedError() + elif input_type == "select" or input_type == "text": + value = input_values[input_name] + json_value = _cast_if_not_none(value, str) + as_dict[input_name] = json_value + elif input_type == "float": + value = input_values[input_name] + json_value = _cast_if_not_none(value, float, empty_to_none=True) + as_dict[input_name] = json_value + elif input_type == "integer": + value = input_values[input_name] + json_value = _cast_if_not_none(value, int, empty_to_none=True) + as_dict[input_name] = json_value + elif input_type == "boolean": + value = input_values[input_name] + json_value = _cast_if_not_none(value, bool) + as_dict[input_name] = json_value + else: + raise NotImplementedError("input_type [%s] not implemented" % input_type) + return as_dict + + +def _cast_if_not_none(value, cast_to, empty_to_none=False): + # log.debug("value [%s], type[%s]" % (value, type(value))) + if value is None or (empty_to_none and str(value) == ''): + return None + else: + return cast_to(value) + + +__all__ = ['json_wrap'] diff --git a/test/functional/tools/inputs_as_json.xml b/test/functional/tools/inputs_as_json.xml new file mode 100644 index 00000000000..a4f5a4de91c --- /dev/null +++ b/test/functional/tools/inputs_as_json.xml @@ -0,0 +1,79 @@ + + + + + + + python $check_inputs $inputs $test_case + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/test/functional/tools/samples_tool_conf.xml b/test/functional/tools/samples_tool_conf.xml index 509a37c78ae..3d7be1d164b 100644 --- a/test/functional/tools/samples_tool_conf.xml +++ b/test/functional/tools/samples_tool_conf.xml @@ -15,6 +15,7 @@ +