mirror of
https://github.com/galaxyproject/galaxy.git
synced 2026-09-24 16:30:27 +08:00
Provide $input.serialize(), allow wrap_json to serialize collections
This commit is contained in:
mvdbeek
@@ -5307,7 +5307,9 @@ A contrived example of a tool that uses this is the test tool
|
||||
[inputs_as_json.xml](https://github.com/galaxyproject/galaxy/blob/dev/test/functional/tools/inputs_as_json.xml).
|
||||
|
||||
By default this file will not contain paths for data or collection inputs. To include simple
|
||||
paths for data parameters set the ``data_style`` attribute to ``paths`` (see [inputs_as_json_with_paths.xml](https://github.com/galaxyproject/galaxy/blob/dev/test/functional/tools/inputs_as_json_with_paths.xml) for an example).
|
||||
paths for data or collection inputs set the ``data_style`` attribute to ``paths`` (see [inputs_as_json_with_paths.xml](https://github.com/galaxyproject/galaxy/blob/dev/test/functional/tools/inputs_as_json_with_paths.xml) for an example).
|
||||
To include a dictionary with staging paths, paths and metadata files set the ``data_style`` attribute to ``staging_path_and_source_path``.
|
||||
An example tool that uses ``staging_path_and_source_path`` is [inputs_as_json_with_staging_path_and_source_path.xml](https://github.com/galaxyproject/galaxy/blobl/dev/test/functional/tools/inputs_as_json_with_staging_path_and_source_path.xml)
|
||||
|
||||
For tools with profile >= 20.05 a select with ``multiple="true"`` is rendered as an array which is empty if nothing is selected. For older profile versions select lists are rendered as comma separated strings or a literal ``null`` in case nothing is selected.
|
||||
]]></xs:documentation>
|
||||
@@ -5329,7 +5331,7 @@ response to this directive.
|
||||
</xs:attribute>
|
||||
<xs:attribute name="data_style" type="xs:string">
|
||||
<xs:annotation>
|
||||
<xs:documentation xml:lang="en">Set to 'paths' to include dataset paths in the resulting file.</xs:documentation>
|
||||
<xs:documentation xml:lang="en">Set to 'paths' to include dataset paths in the resulting file. Set to 'staging_path_and_source_path' to include a staging path, a source path and all metadata files.</xs:documentation>
|
||||
</xs:annotation>
|
||||
</xs:attribute>
|
||||
</xs:extension>
|
||||
|
||||
@@ -22,15 +22,39 @@ def json_wrap(inputs, input_values, profile, as_dict=None, handle_files="skip"):
|
||||
return as_dict
|
||||
|
||||
|
||||
def data_input_to_path(v):
|
||||
path = _cast_if_not_none(v, str)
|
||||
if path == "None":
|
||||
path = None
|
||||
return path
|
||||
|
||||
|
||||
def data_collection_input_to_path(v):
|
||||
return v.all_paths
|
||||
|
||||
|
||||
def data_collection_input_to_staging_path_and_source_path(v):
|
||||
staging_paths = v.all_staging_paths
|
||||
source_paths = v.all_paths
|
||||
metadata_files = v.all_metadata_files
|
||||
return [
|
||||
{'staging_path': staging_path,
|
||||
'source_path': source_path,
|
||||
'metadata_files': [{'staging_path': f"{staging_path}.{mf[0]}", 'source_path': mf[1]} for mf in metadata_files]
|
||||
} for staging_path, source_path, metadata_files in zip(staging_paths, source_paths, metadata_files)]
|
||||
|
||||
|
||||
def data_input_to_staging_path_and_source_path(v):
|
||||
return {
|
||||
'staging_path': v.staging_path,
|
||||
'source_path': data_input_to_path(v),
|
||||
'metadata_files': [{'staging_path': f"{v.staging_path}.{mf[0]}", 'source_path': mf[1]} for mf in v.all_metadata_files]
|
||||
}
|
||||
|
||||
|
||||
def _json_wrap_input(input, value_wrapper, profile, handle_files="skip"):
|
||||
input_type = input.type
|
||||
|
||||
def _data_input_to_path(v):
|
||||
path = _cast_if_not_none(v, str)
|
||||
if path == "None":
|
||||
path = None
|
||||
return path
|
||||
|
||||
if input_type == "repeat":
|
||||
repeat_job_value = []
|
||||
for d in value_wrapper:
|
||||
@@ -58,14 +82,18 @@ def _json_wrap_input(input, value_wrapper, profile, handle_files="skip"):
|
||||
json_value = section_job_value
|
||||
elif input_type == "data" and input.multiple:
|
||||
if handle_files == "paths":
|
||||
json_value = list(map(_data_input_to_path, value_wrapper))
|
||||
json_value = [data_input_to_path(v) for v in value_wrapper]
|
||||
elif handle_files == 'staging_path_and_source_path':
|
||||
json_value = [data_input_to_staging_path_and_source_path(v) for v in value_wrapper]
|
||||
elif handle_files == "skip":
|
||||
return SKIP_INPUT
|
||||
else:
|
||||
raise NotImplementedError()
|
||||
elif input_type == "data":
|
||||
if handle_files == "paths":
|
||||
json_value = _data_input_to_path(value_wrapper)
|
||||
json_value = data_input_to_path(value_wrapper)
|
||||
elif handle_files == 'staging_path_and_source_path':
|
||||
json_value = data_input_to_staging_path_and_source_path(value_wrapper)
|
||||
elif handle_files == "skip":
|
||||
return SKIP_INPUT
|
||||
elif handle_files == "OBJECT":
|
||||
@@ -84,6 +112,10 @@ def _json_wrap_input(input, value_wrapper, profile, handle_files="skip"):
|
||||
elif input_type == "data_collection":
|
||||
if handle_files == "skip":
|
||||
return SKIP_INPUT
|
||||
elif handle_files == "paths":
|
||||
return data_collection_input_to_path(value_wrapper)
|
||||
elif handle_files == "staging_path_and_source_path":
|
||||
return data_collection_input_to_staging_path_and_source_path(value_wrapper)
|
||||
raise NotImplementedError()
|
||||
elif input_type in ["text", "color", "hidden"]:
|
||||
if getattr(input, "optional", False) and value_wrapper is not None and value_wrapper.value is None:
|
||||
@@ -113,6 +145,8 @@ def _json_wrap_input(input, value_wrapper, profile, handle_files="skip"):
|
||||
json_value = [int(_) for _ in _cast_if_not_none(value_wrapper.value, list)]
|
||||
else:
|
||||
json_value = [_cast_if_not_none(value_wrapper.value, int)]
|
||||
elif input_type == "directory_uri":
|
||||
json_value = _cast_if_not_none(value_wrapper, str)
|
||||
else:
|
||||
raise NotImplementedError(f"input_type [{input_type}] not implemented")
|
||||
|
||||
|
||||
@@ -6,6 +6,10 @@ from functools import total_ordering
|
||||
|
||||
from galaxy import exceptions
|
||||
from galaxy.model.none_like import NoneDataset
|
||||
from galaxy.tools.parameters.wrapped_json import (
|
||||
data_collection_input_to_staging_path_and_source_path,
|
||||
data_input_to_staging_path_and_source_path,
|
||||
)
|
||||
from galaxy.util import filesystem_safe_string
|
||||
from galaxy.util.object_wrapper import wrap_with_safe_string
|
||||
|
||||
@@ -312,7 +316,7 @@ class DatasetFilenameWrapper(ToolParameterValueWrapper):
|
||||
return f"{self.element_identifier}.{self.file_ext}"
|
||||
|
||||
@property
|
||||
def name_and_ext_filesystem_safe(self):
|
||||
def staging_path(self):
|
||||
"""
|
||||
Strip leading dots, unicode null chars, replace `/` with `_`, truncate at 255 characters.
|
||||
|
||||
@@ -324,7 +328,10 @@ class DatasetFilenameWrapper(ToolParameterValueWrapper):
|
||||
|
||||
@property
|
||||
def all_metadata_files(self):
|
||||
return self.unsanitized.get_metadata_file_paths_and_extensions()
|
||||
return self.unsanitized.get_metadata_file_paths_and_extensions() if self else []
|
||||
|
||||
def serialize(self):
|
||||
return data_input_to_staging_path_and_source_path(self) if self else {}
|
||||
|
||||
@property
|
||||
def is_collection(self):
|
||||
@@ -374,6 +381,8 @@ class DatasetFilenameWrapper(ToolParameterValueWrapper):
|
||||
# instead of just returning a non-existent
|
||||
# path like DiskObjectStore.
|
||||
raise
|
||||
elif key == 'serialize':
|
||||
return self.serialize
|
||||
else:
|
||||
return getattr(self.dataset, key)
|
||||
|
||||
@@ -440,6 +449,9 @@ class DatasetListWrapper(list, ToolParameterValueWrapper, HasDatasets):
|
||||
self._dataset_elements_cache[group] = wrappers
|
||||
return self._dataset_elements_cache[group]
|
||||
|
||||
def serialize(self):
|
||||
return [v.serialize() for v in self]
|
||||
|
||||
def __str__(self):
|
||||
return ','.join(map(str, self))
|
||||
|
||||
@@ -531,11 +543,14 @@ class DatasetCollectionWrapper(ToolParameterValueWrapper, HasDatasets):
|
||||
@property
|
||||
def element_identifiers_extensions_paths_and_metadata_files(self):
|
||||
if self._element_identifiers_extensions_paths_and_metadata_files is None:
|
||||
self._element_identifiers_extensions_paths_and_metadata_files = self.collection.element_identifiers_extensions_paths_and_metadata_files
|
||||
if self.collection:
|
||||
self._element_identifiers_extensions_paths_and_metadata_files = self.collection.element_identifiers_extensions_paths_and_metadata_files
|
||||
else:
|
||||
return []
|
||||
return self._element_identifiers_extensions_paths_and_metadata_files
|
||||
|
||||
@property
|
||||
def all_element_identifiers_and_extensions_filesystem_safe(self):
|
||||
def all_staging_paths(self):
|
||||
safe_element_identifiers = []
|
||||
for element_identifiers, extension, *_ in self.element_identifiers_extensions_paths_and_metadata_files:
|
||||
datatype = self.datatypes_registry.get_datatype_by_extension(extension)
|
||||
@@ -547,6 +562,9 @@ class DatasetCollectionWrapper(ToolParameterValueWrapper, HasDatasets):
|
||||
safe_element_identifiers.append(f'{os.path.sep.join(current_element_identifiers)}.{extension}')
|
||||
return safe_element_identifiers
|
||||
|
||||
def serialize(self):
|
||||
return data_collection_input_to_staging_path_and_source_path(self)
|
||||
|
||||
@property
|
||||
def is_input_supplied(self):
|
||||
return self.__input_supplied
|
||||
|
||||
@@ -33,6 +33,7 @@ if test_case == "1":
|
||||
assert_equals(as_dict["section_example"]["section_text"], "section_default")
|
||||
assert_equals(as_dict["data_input"], None)
|
||||
assert_equals(as_dict["multiple_data_input"], [None])
|
||||
assert as_dict["collection_input"] == []
|
||||
elif test_case == "2":
|
||||
assert_equals(as_dict["test_case"], 2)
|
||||
assert_equals(as_dict["text_test"], "bar")
|
||||
@@ -50,6 +51,11 @@ elif test_case == "2":
|
||||
multiple_data_input = as_dict["multiple_data_input"]
|
||||
assert type(multiple_data_input) is list
|
||||
assert len(multiple_data_input) == 2
|
||||
assert as_dict["collection_input"] == []
|
||||
elif test_case == "3":
|
||||
collection_input = as_dict["collection_input"]
|
||||
assert type(collection_input) is list
|
||||
assert len(collection_input) == 1
|
||||
|
||||
with open("output", "w") as f:
|
||||
f.write("okay\n")
|
||||
@@ -69,6 +75,7 @@ with open("output", "w") as f:
|
||||
</param>
|
||||
<param name="data_input" type="data" optional="true" />
|
||||
<param name="multiple_data_input" type="data" optional="true" multiple="true" />
|
||||
<param name="collection_input" type="data_collection" optional="true"/>
|
||||
<repeat name="repeat" title="Repeat" min="1">
|
||||
<param name="r" type="color" />
|
||||
</repeat>
|
||||
@@ -133,6 +140,19 @@ with open("output", "w") as f:
|
||||
</assert_contents>
|
||||
</output>
|
||||
</test>
|
||||
<test>
|
||||
<param name="test_case" value="3" />
|
||||
<param name="text_test" value="foo" />
|
||||
<param name="collection_input">
|
||||
<collection type="list:list">
|
||||
<element name="list">
|
||||
<collection type="list">
|
||||
<element name="element1" value="simple_line.txt" ftype="txt" />
|
||||
</collection>
|
||||
</element>
|
||||
</collection>
|
||||
</param>
|
||||
</test>
|
||||
</tests>
|
||||
<help>
|
||||
Test tool demonstrating the special inputs config file.
|
||||
|
||||
@@ -0,0 +1,46 @@
|
||||
<tool id="inputs_as_json_with_staging_paths" name="inputs_as_json_with_staging_paths" version="1.0.0" profile="21.09">
|
||||
<command detect_errors="exit_code">
|
||||
python '$check_inputs' '$inputs'
|
||||
</command>
|
||||
<configfiles>
|
||||
<inputs name="inputs" data_style="staging_path_and_source_path" />
|
||||
<configfile name="check_inputs"><![CDATA[
|
||||
import json
|
||||
import sys
|
||||
|
||||
input_json_path = sys.argv[1]
|
||||
as_dict = json.load(open(input_json_path, "r"))
|
||||
|
||||
collection_input_with_staging_details = as_dict["collection_input"]
|
||||
assert collection_input_with_staging_details[0]['staging_path'] == "list/element1.bam"
|
||||
assert collection_input_with_staging_details[0]['metadata_files'][0]['staging_path'] == "list/element1.bam.bai"
|
||||
|
||||
with open("output", "w") as f:
|
||||
f.write("okay\n")
|
||||
]]></configfile>
|
||||
</configfiles>
|
||||
<inputs>
|
||||
<param name="data_input" type="data" optional="true" />
|
||||
<param name="multiple_data_input" type="data" optional="true" multiple="true" />
|
||||
<param name="collection_input" type="data_collection" optional="true"/>
|
||||
</inputs>
|
||||
<outputs>
|
||||
<data name="out_file1" from_work_dir="output" format="txt" />
|
||||
</outputs>
|
||||
<tests>
|
||||
<test>
|
||||
<param name="collection_input">
|
||||
<collection type="list:list">
|
||||
<element name="list">
|
||||
<collection type="list">
|
||||
<element name="element1" value="1.bam" ftype="bam" />
|
||||
</collection>
|
||||
</element>
|
||||
</collection>
|
||||
</param>
|
||||
</test>
|
||||
</tests>
|
||||
<help>
|
||||
Test tool demonstrating the special inputs config file.
|
||||
</help>
|
||||
</tool>
|
||||
@@ -44,6 +44,7 @@
|
||||
<tool file="inputs_as_json.xml" />
|
||||
<tool file="inputs_as_json_profile.xml" />
|
||||
<tool file="inputs_as_json_with_paths.xml" />
|
||||
<tool file="inputs_as_json_with_staging_path_and_source_path.xml" />
|
||||
<tool file="filter_multiple_splitter.xml" />
|
||||
<tool file="filter_static_regexp.xml" />
|
||||
<tool file="select_from_dataset.xml" />
|
||||
|
||||
@@ -18,7 +18,7 @@ def check_for_duplicate_name(files_to_export):
|
||||
seen = set()
|
||||
duplicates = set()
|
||||
for entry in files_to_export:
|
||||
name = entry['name']
|
||||
name = entry['staging_path']
|
||||
if name in seen:
|
||||
duplicates.add(name)
|
||||
seen.add(name)
|
||||
@@ -42,25 +42,24 @@ def main(argv=None):
|
||||
exit_code = 0
|
||||
file_sources = get_file_sources(args.file_sources)
|
||||
directory_uri = args.directory_uri
|
||||
if not directory_uri.endswith("/"):
|
||||
directory_uri = f"{directory_uri}/"
|
||||
export_metadata_files = args.export_metadata_files
|
||||
with open(args.files_to_export) as f:
|
||||
files_to_export = json.load(f)
|
||||
counter = 0
|
||||
check_for_duplicate_name(files_to_export)
|
||||
for entry in files_to_export:
|
||||
name = entry["name"]
|
||||
real_data_path = entry["real_data_path"]
|
||||
if directory_uri.endswith("/"):
|
||||
target_uri = directory_uri + name
|
||||
else:
|
||||
target_uri = directory_uri + "/" + name
|
||||
name = entry["staging_path"]
|
||||
real_data_path = entry["source_path"]
|
||||
target_uri = f"{directory_uri}{name}"
|
||||
if write_if_not_exists(file_sources, target_uri, real_data_path):
|
||||
exit_code = 1
|
||||
if export_metadata_files:
|
||||
metadata_files = entry.get('metadata_files', [])
|
||||
for extension, path in metadata_files:
|
||||
metadata_file_uri = f"{target_uri}.{extension}"
|
||||
if write_if_not_exists(file_sources, metadata_file_uri, path):
|
||||
for metadata_file in metadata_files:
|
||||
metadata_file_uri = f"{directory_uri}{metadata_file['staging_path']}"
|
||||
if write_if_not_exists(file_sources, metadata_file_uri, metadata_file['source_path']):
|
||||
exit_code = 1
|
||||
counter += 1
|
||||
print(f"{counter} out of {len(files_to_export)} files have been exported.\n")
|
||||
|
||||
@@ -18,11 +18,15 @@ python '$__tool_directory__/export_remote.py'
|
||||
<configfile name="files_to_export">#import json
|
||||
#from galaxy.util import filesystem_safe_string
|
||||
#if $export_type.export_type_selector == "datasets_auto":
|
||||
#set $fileconfig = json.dumps([{"real_data_path": str(infile), "name": infile.name_and_ext_filesystem_safe, "metadata_files": infile.all_metadata_files} for infile in $infiles])
|
||||
#set $fileconfig = json.dumps($export_type.infiles.serialize())
|
||||
#else if $export_type.export_type_selector == "collection_auto":
|
||||
#set $fileconfig = json.dumps([{"real_data_path": data_path, "name": identifier_and_extension, "metadata_files": metadata_files} for data_path, identifier_and_extension, metadata_files in zip($export_type.incollection.all_paths, $export_type.incollection.all_element_identifiers_and_extensions_filesystem_safe, $export_type.incollection.all_metadata_files)])
|
||||
#set $fileconfig = json.dumps($export_type.incollection.serialize())
|
||||
#else
|
||||
#set $fileconfig = json.dumps([{"real_data_path": str(dataset["infile"]), "name": filesystem_safe_string(str(dataset["name"]), 255, invalid_chars=()), "metadata_files": dataset['infile'].all_metadata_files} for dataset in $export_type.datasets])
|
||||
#set $fileconfig = [d['infile'].serialize() for d in $export_type.datasets]
|
||||
#for $i, $dataset in enumerate($export_type.datasets):
|
||||
#silent $fileconfig[$i]['staging_path'] = filesystem_safe_string(str(dataset["name"]), 255, invalid_chars=())
|
||||
#end for
|
||||
#silent fileconfig = json.dumps($fileconfig)
|
||||
#end if
|
||||
$fileconfig
|
||||
</configfile>
|
||||
|
||||
Reference in New Issue
Block a user