opensource/galaxy
3
0
Fork 0
mirror of https://github.com/galaxyproject/galaxy.git synced 2026-09-24 16:30:27 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #1658 from einon/pass_lists_as_file

Browse Source 
Optionally pass dataset lists as a single file within tool wrappers
This commit is contained in:
Martin Cech
2016-02-05 11:45:13 -05:00
parent dfceb5c144 66350fb871
commit 75f1b58ecb
6 changed files with 63 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/galaxy/tools/evaluation.py
+7 -3
View File
@@ -130,7 +130,7 @@ class ToolEvaluator( object ):
param_dict.update( incoming )
input_dataset_paths = dataset_path_rewrites( input_paths )
self.__populate_wrappers(param_dict, input_dataset_paths)
self.__populate_wrappers(param_dict, input_dataset_paths, job_working_directory)
self.__populate_input_dataset_wrappers(param_dict, input_datasets, input_dataset_paths)
self.__populate_output_dataset_wrappers(param_dict, output_datasets, output_paths, job_working_directory)
self.__populate_output_collection_wrappers(param_dict, output_collections, output_paths, job_working_directory)
@@ -165,18 +165,20 @@ class ToolEvaluator( object ):
do_walk( inputs, input_values )
def __populate_wrappers(self, param_dict, input_dataset_paths):
def __populate_wrappers(self, param_dict, input_dataset_paths, job_working_directory):
def wrap_input( input_values, input ):
if isinstance( input, DataToolParameter ) and input.multiple:
value = input_values[ input.name ]
dataset_instances = DatasetListWrapper.to_dataset_instances( value )
input_values[ input.name ] = \
DatasetListWrapper( dataset_instances,
DatasetListWrapper( job_working_directory,
dataset_instances,
dataset_paths=input_dataset_paths,
datatypes_registry=self.app.datatypes_registry,
tool=self.tool,
name=input.name )
elif isinstance( input, DataToolParameter ):
# FIXME: We're populating param_dict with conversions when
# wrapping values, this should happen as a separate
@@ -234,6 +236,7 @@ class ToolEvaluator( object ):
name=input.name
)
wrapper = DatasetCollectionWrapper(
job_working_directory,
dataset_collection,
**wrapper_kwds
)
@@ -306,6 +309,7 @@ class ToolEvaluator( object ):
name=name
)
wrapper = DatasetCollectionWrapper(
job_working_directory,
out_collection,
**wrapper_kwds
)
lib/galaxy/tools/parameters/basic.py
+1 -1
View File
@@ -1114,7 +1114,7 @@ class GenomeBuildParameter( SelectToolParameter ):
'options' : options,
'value' : value,
'display' : self.display,
'multiple' : self.multiple
'multiple' : self.multiple,
})
return d
lib/galaxy/tools/parameters/wrapped.py
+3 -1
View File
@@ -58,7 +58,8 @@ class WrappedParameters( object ):
value = input_values[ input.name ]
dataset_instances = DatasetListWrapper.to_dataset_instances( value )
input_values[ input.name ] = \
DatasetListWrapper( dataset_instances,
DatasetListWrapper( None,
dataset_instances,
datatypes_registry=trans.app.datatypes_registry,
tool=tool,
name=input.name )
@@ -72,6 +73,7 @@ class WrappedParameters( object ):
input_values[ input.name ] = SelectToolParameterWrapper( input, input_values[ input.name ], tool.app, other_values=incoming )
elif isinstance( input, DataCollectionToolParameter ):
input_values[ input.name ] = DatasetCollectionWrapper(
None,
input_values[ input.name ],
datatypes_registry=trans.app.datatypes_registry,
tool=tool,
lib/galaxy/tools/wrappers.py
+14 -3
View File
@@ -1,4 +1,6 @@
import os
import pipes
import tempfile
from galaxy import exceptions
from galaxy.util.none_like import NoneDataset
from galaxy.util import odict
@@ -263,11 +265,18 @@ class HasDatasets:
wrapper_kwds[ "dataset_path" ] = dataset_paths[ real_path ]
return DatasetFilenameWrapper( dataset, **wrapper_kwds )
def paths_as_file(self, sep="\n"):
handle, filepath = tempfile.mkstemp(prefix="gx_file_list", dir=self.job_working_directory)
contents = sep.join(map(str, self))
os.write(handle, contents)
os.close(handle)
return filepath
class DatasetListWrapper( list, ToolParameterValueWrapper, HasDatasets ):
"""
"""
def __init__( self, datasets, dataset_paths=[], **kwargs ):
def __init__( self, job_working_directory, datasets, dataset_paths=[], **kwargs ):
if not isinstance(datasets, list):
datasets = [datasets]
@@ -279,6 +288,7 @@ class DatasetListWrapper( list, ToolParameterValueWrapper, HasDatasets ):
return self._dataset_wrapper( dataset, dataset_paths, **kwargs )
list.__init__( self, map( to_wrapper, datasets ) )
self.job_working_directory = job_working_directory
@staticmethod
def to_dataset_instances( dataset_instance_sources ):
@@ -300,8 +310,9 @@ class DatasetListWrapper( list, ToolParameterValueWrapper, HasDatasets ):
class DatasetCollectionWrapper( ToolParameterValueWrapper, HasDatasets ):
def __init__( self, has_collection, dataset_paths=[], **kwargs ):
def __init__( self, job_working_directory, has_collection, dataset_paths=[], **kwargs ):
super(DatasetCollectionWrapper, self).__init__()
self.job_working_directory = job_working_directory
if has_collection is None:
self.__input_supplied = False
@@ -330,7 +341,7 @@ class DatasetCollectionWrapper( ToolParameterValueWrapper, HasDatasets ):
element_identifier = dataset_collection_element.element_identifier
if dataset_collection_element.is_collection:
element_wrapper = DatasetCollectionWrapper( dataset_collection_element, dataset_paths, **kwargs )
element_wrapper = DatasetCollectionWrapper(job_working_directory, dataset_collection_element, dataset_paths, **kwargs )
else:
element_wrapper = self._dataset_wrapper( element_object, dataset_paths, **kwargs)
test/functional/tools/paths_as_file.xml
+37
View File
@@ -0,0 +1,37 @@
<tool id="paths_as_file" name="paths_as_file" version="0.1.0">
<configfiles>
<configfile name="check_paths_file"><![CDATA[
import sys
paths_file = sys.argv[1]
sep = sys.argv[2]
if sep == "NEWLINE":
sep = "\n"
with open(paths_file, "r") as f:
paths = f.read()
assert paths == sep.join(sys.argv[3:])
]]></configfile>
</configfiles>
<command detect_errors="exit_code"><![CDATA[
python $check_paths_file $inputs.paths_as_file NEWLINE #for $f in $inputs# ${f} #end for#
&&
python $check_paths_file $inputs.paths_as_file(sep=',') ',' #for $f in $inputs# ${f} #end for#
&&
printf 'All Done' > $out1
]]></command>
<inputs>
<param name="inputs" type="data" format="txt" multiple="true" label="Data 1" />
</inputs>
<outputs>
<data format="txt" name="out1" />
</outputs>
<tests>
<test>
<param name="inputs" value="simple_line.txt,simple_line_alternative.txt" />
<output name="out1">
<assert_contents>
<has_line line="All Done" />
</assert_contents>
</output>
</test>
</tests>
</tool>
test/functional/tools/samples_tool_conf.xml
+1
View File
@@ -44,6 +44,7 @@
<tool file="parallelism_optional.xml" />
<tool file="implicit_default_conds.xml" />
<tool file="multi_data_param.xml" />
<tool file="paths_as_file.xml" />
<tool file="column_param.xml" />
<tool file="column_multi_param.xml" />
<tool file="special_params.xml" />