Implement special config file to dump parameters as json.

```
    <configfiles>
        <inputs name="inputs" format="json" version="1" />
    </configfiles>
```

Version is specified as 1, because the Galaxy tool plumbing doesn't really distinguish between parameter is '', parameter was explicitly supplied as None, parameter was absent, etc... - everything is a just a string in some ways. This distinction may be more important with JSON and for instance the CWL will require more percision, so we may want to revise Galaxy's tool handling in a backward compatible way someday.
This commit is contained in:
John Chilton
2015-12-29 15:09:49 +00:00
parent 14a618ee3a
commit d5d404c912
5 changed files with 184 additions and 6 deletions
+13 -2
View File
@@ -641,11 +641,22 @@ class Tool( object, Dictifiable ):
root = tool_source.root
conf_parent_elem = root.find("configfiles")
if conf_parent_elem is not None:
inputs_elem = conf_parent_elem.find( "inputs" )
if inputs_elem is not None:
name = inputs_elem.get( "name" )
filename = inputs_elem.get( "filename", None )
format = inputs_elem.get("format", "json")
version_str = inputs_elem.get("version", "1")
if not version_str:
raise ValueError("inputs configfile tag must define a version attribute")
version = int(version_str)
content = dict(format=format, version=version)
self.config_files.append( ( name, filename, content ) )
for conf_elem in conf_parent_elem.findall( "configfile" ):
name = conf_elem.get( "name" )
filename = conf_elem.get( "filename", None )
text = conf_elem.text
self.config_files.append( ( name, filename, text ) )
content = conf_elem.text
self.config_files.append( ( name, filename, content ) )
def __parse_trackster_conf(self, tool_source):
self.trackster_conf = None
+19 -4
View File
@@ -1,3 +1,4 @@
import json
import os
import tempfile
@@ -20,6 +21,7 @@ from galaxy.tools.parameters.basic import (
DataCollectionToolParameter,
SelectToolParameter,
)
from galaxy.tools.parameters import wrapped_json
from galaxy.tools.parameters.grouping import Conditional, Repeat, Section
from galaxy.tools import global_tool_errors
from galaxy.jobs.datasets import dataset_path_rewrites
@@ -472,7 +474,8 @@ class ToolEvaluator( object ):
"""
param_dict = self.param_dict
config_filenames = []
for name, filename, template_text in self.tool.config_files:
for name, filename, content in self.tool.config_files:
config_text, is_template = self.__build_config_file_text(content)
# If a particular filename was forced by the config use it
directory = self.local_working_directory
if filename is not None:
@@ -480,7 +483,7 @@ class ToolEvaluator( object ):
else:
fd, config_filename = tempfile.mkstemp( dir=directory )
os.close( fd )
self.__write_workdir_file( config_filename, template_text, param_dict )
self.__write_workdir_file( config_filename, config_text, param_dict, is_template=is_template )
self.__register_extra_file( name, config_filename )
config_filenames.append( config_filename )
return config_filenames
@@ -526,8 +529,20 @@ class ToolEvaluator( object ):
else:
return None
def __write_workdir_file( self, config_filename, template, context ):
value = fill_template( template, context=context )
def __build_config_file_text( self, content ):
if isinstance( content, basestring ):
return content, True
assert content["format"] == "json"
assert content["version"] == 1
return json.dumps(wrapped_json.json_wrap(self.tool.inputs, self.param_dict)), False
def __write_workdir_file( self, config_filename, content, context, is_template=True ):
if is_template:
value = fill_template( content, context=context )
else:
value = content
with open( config_filename, "w" ) as f:
f.write( value )
# For running jobs as the actual user, ensure the config file is globally readable
@@ -0,0 +1,72 @@
import logging
log = logging.getLogger(__name__)
def json_wrap(inputs, input_values, as_dict=None, handle_files="SKIP"):
if as_dict is None:
as_dict = {}
for input in inputs.itervalues():
input_name = input.name
input_type = input.type
value = input_values[input_name]
if input_type == "repeat":
repeat_job_value = []
for d in input_values[input.name]:
repeat_instance_job_value = {}
json_wrap(input.inputs, d, repeat_instance_job_value)
repeat_job_value.append(repeat_instance_job_value)
as_dict[input_name] = repeat_job_value
if input_type == "conditional":
values = input_values[input_name]
current = values["__current_case__"]
conditional_job_value = {}
json_wrap(input.cases[current].inputs, values, conditional_job_value)
as_dict[input_name] = conditional_job_value
if input_type == "section":
values = input_values[input_name]
section_job_value = {}
json_wrap(input.inputs, values, section_job_value)
as_dict[input_name] = section_job_value
elif input_type == "data" and input.multiple:
if handle_files == "SKIP":
continue
raise NotImplementedError()
elif input_type == "data":
if handle_files == "SKIP":
continue
raise NotImplementedError()
elif input_type == "data_collection":
if handle_files == "SKIP":
continue
raise NotImplementedError()
elif input_type == "select" or input_type == "text":
value = input_values[input_name]
json_value = _cast_if_not_none(value, str)
as_dict[input_name] = json_value
elif input_type == "float":
value = input_values[input_name]
json_value = _cast_if_not_none(value, float, empty_to_none=True)
as_dict[input_name] = json_value
elif input_type == "integer":
value = input_values[input_name]
json_value = _cast_if_not_none(value, int, empty_to_none=True)
as_dict[input_name] = json_value
elif input_type == "boolean":
value = input_values[input_name]
json_value = _cast_if_not_none(value, bool)
as_dict[input_name] = json_value
else:
raise NotImplementedError("input_type [%s] not implemented" % input_type)
return as_dict
def _cast_if_not_none(value, cast_to, empty_to_none=False):
# log.debug("value [%s], type[%s]" % (value, type(value)))
if value is None or (empty_to_none and str(value) == ''):
return None
else:
return cast_to(value)
__all__ = ['json_wrap']
+79
View File
@@ -0,0 +1,79 @@
<tool id="inputs_as_json" name="inputs_as_json" version="1.0.0">
<configfiles>
<inputs name="inputs" format="json" version="1" />
<configfile name="check_inputs"><![CDATA[
import json
import sys
input_json_path = sys.argv[1]
test_case = sys.argv[2]
as_dict = json.load(open(input_json_path, "r"))
def assert_equals(x, y):
assert x == y, "%s != %s" % (x, y)
if test_case == "1":
assert_equals(as_dict["test_case"], 1)
assert_equals(as_dict["text_test"], "foo")
assert_equals(as_dict["booltest"], True)
assert_equals(as_dict["inttest"], 12456)
assert_equals(as_dict["floattest"], 6.789)
assert_equals(as_dict["radio_select"], "a_radio")
elif test_case == "2":
assert_equals(as_dict["test_case"], 2)
assert_equals(as_dict["text_test"], "bar")
assert_equals(as_dict["booltest"], False)
assert_equals(as_dict["inttest"], None)
assert_equals(as_dict["floattest"], 1.0)
assert_equals(as_dict["radio_select"], "a_radio")
with open("output", "w") as f:
f.write("okay\n")
]]></configfile>
</configfiles>
<command>
python $check_inputs $inputs $test_case
</command>
<inputs>
<param name="test_case" type="integer" value="0" />
<param name="text_test" type="text" />
<param name="booltest" truevalue="booltrue" falsevalue="boolfalse" checked="false" type="boolean" />
<param name="inttest" type="integer" optional="true" />
<param name="floattest" value="1.0" type="float" />
<param name="radio_select" type="select" display="radio">
<option value="a_radio" selected="true">A Radio</option>
<option value="b_radio">B Radio</option>
<option value="c_radio">C Radio</option>
</param>
</inputs>
<outputs>
<data name="out_file1" from_work_dir="output" format="txt" />
</outputs>
<tests>
<test>
<param name="test_case" value="1" />
<param name="text_test" value="foo" />
<param name="booltest" value="true" />
<param name="inttest" value="12456" />
<param name="floattest" value="6.789" />
<output name="out_file1">
<assert_contents>
<has_line line="okay" />
</assert_contents>
</output>
</test>
<test>
<param name="test_case" value="2" />
<param name="text_test" value="bar" />
<param name="booltest" value="false" />
<!-- Testing null integers -->
<!-- <param name="inttest" value="12456" /> -->
<output name="out_file1">
<assert_contents>
<has_line line="okay" />
</assert_contents>
</output>
</test>
</tests>
</tool>
@@ -15,6 +15,7 @@
<tool file="multi_output.xml" />
<tool file="multi_output_configured.xml" />
<tool file="multi_output_assign_primary.xml" />
<tool file="inputs_as_json.xml" />
<tool file="dbkey_filter_input.xml" />
<tool file="dbkey_filter_multi_input.xml" />
<tool file="composite_output.xml" />