diff --git a/lib/galaxy/tool_util/xsd/galaxy.xsd b/lib/galaxy/tool_util/xsd/galaxy.xsd
index 3e5e3aa52ad..1b2ff9c63db 100644
--- a/lib/galaxy/tool_util/xsd/galaxy.xsd
+++ b/lib/galaxy/tool_util/xsd/galaxy.xsd
@@ -5307,7 +5307,9 @@ A contrived example of a tool that uses this is the test tool
[inputs_as_json.xml](https://github.com/galaxyproject/galaxy/blob/dev/test/functional/tools/inputs_as_json.xml).
By default this file will not contain paths for data or collection inputs. To include simple
-paths for data parameters set the ``data_style`` attribute to ``paths`` (see [inputs_as_json_with_paths.xml](https://github.com/galaxyproject/galaxy/blob/dev/test/functional/tools/inputs_as_json_with_paths.xml) for an example).
+paths for data or collection inputs set the ``data_style`` attribute to ``paths`` (see [inputs_as_json_with_paths.xml](https://github.com/galaxyproject/galaxy/blob/dev/test/functional/tools/inputs_as_json_with_paths.xml) for an example).
+To include a dictionary with staging paths, paths and metadata files set the ``data_style`` attribute to ``staging_path_and_source_path``.
+An example tool that uses ``staging_path_and_source_path`` is [inputs_as_json_with_staging_path_and_source_path.xml](https://github.com/galaxyproject/galaxy/blobl/dev/test/functional/tools/inputs_as_json_with_staging_path_and_source_path.xml)
For tools with profile >= 20.05 a select with ``multiple="true"`` is rendered as an array which is empty if nothing is selected. For older profile versions select lists are rendered as comma separated strings or a literal ``null`` in case nothing is selected.
]]>
@@ -5329,7 +5331,7 @@ response to this directive.
- Set to 'paths' to include dataset paths in the resulting file.
+ Set to 'paths' to include dataset paths in the resulting file. Set to 'staging_path_and_source_path' to include a staging path, a source path and all metadata files.
diff --git a/lib/galaxy/tools/parameters/wrapped_json.py b/lib/galaxy/tools/parameters/wrapped_json.py
index 3ddf7ffafd1..954e78123d1 100644
--- a/lib/galaxy/tools/parameters/wrapped_json.py
+++ b/lib/galaxy/tools/parameters/wrapped_json.py
@@ -22,15 +22,39 @@ def json_wrap(inputs, input_values, profile, as_dict=None, handle_files="skip"):
return as_dict
+def data_input_to_path(v):
+ path = _cast_if_not_none(v, str)
+ if path == "None":
+ path = None
+ return path
+
+
+def data_collection_input_to_path(v):
+ return v.all_paths
+
+
+def data_collection_input_to_staging_path_and_source_path(v):
+ staging_paths = v.all_staging_paths
+ source_paths = v.all_paths
+ metadata_files = v.all_metadata_files
+ return [
+ {'staging_path': staging_path,
+ 'source_path': source_path,
+ 'metadata_files': [{'staging_path': f"{staging_path}.{mf[0]}", 'source_path': mf[1]} for mf in metadata_files]
+ } for staging_path, source_path, metadata_files in zip(staging_paths, source_paths, metadata_files)]
+
+
+def data_input_to_staging_path_and_source_path(v):
+ return {
+ 'staging_path': v.staging_path,
+ 'source_path': data_input_to_path(v),
+ 'metadata_files': [{'staging_path': f"{v.staging_path}.{mf[0]}", 'source_path': mf[1]} for mf in v.all_metadata_files]
+ }
+
+
def _json_wrap_input(input, value_wrapper, profile, handle_files="skip"):
input_type = input.type
- def _data_input_to_path(v):
- path = _cast_if_not_none(v, str)
- if path == "None":
- path = None
- return path
-
if input_type == "repeat":
repeat_job_value = []
for d in value_wrapper:
@@ -58,14 +82,18 @@ def _json_wrap_input(input, value_wrapper, profile, handle_files="skip"):
json_value = section_job_value
elif input_type == "data" and input.multiple:
if handle_files == "paths":
- json_value = list(map(_data_input_to_path, value_wrapper))
+ json_value = [data_input_to_path(v) for v in value_wrapper]
+ elif handle_files == 'staging_path_and_source_path':
+ json_value = [data_input_to_staging_path_and_source_path(v) for v in value_wrapper]
elif handle_files == "skip":
return SKIP_INPUT
else:
raise NotImplementedError()
elif input_type == "data":
if handle_files == "paths":
- json_value = _data_input_to_path(value_wrapper)
+ json_value = data_input_to_path(value_wrapper)
+ elif handle_files == 'staging_path_and_source_path':
+ json_value = data_input_to_staging_path_and_source_path(value_wrapper)
elif handle_files == "skip":
return SKIP_INPUT
elif handle_files == "OBJECT":
@@ -84,6 +112,10 @@ def _json_wrap_input(input, value_wrapper, profile, handle_files="skip"):
elif input_type == "data_collection":
if handle_files == "skip":
return SKIP_INPUT
+ elif handle_files == "paths":
+ return data_collection_input_to_path(value_wrapper)
+ elif handle_files == "staging_path_and_source_path":
+ return data_collection_input_to_staging_path_and_source_path(value_wrapper)
raise NotImplementedError()
elif input_type in ["text", "color", "hidden"]:
if getattr(input, "optional", False) and value_wrapper is not None and value_wrapper.value is None:
@@ -113,6 +145,8 @@ def _json_wrap_input(input, value_wrapper, profile, handle_files="skip"):
json_value = [int(_) for _ in _cast_if_not_none(value_wrapper.value, list)]
else:
json_value = [_cast_if_not_none(value_wrapper.value, int)]
+ elif input_type == "directory_uri":
+ json_value = _cast_if_not_none(value_wrapper, str)
else:
raise NotImplementedError(f"input_type [{input_type}] not implemented")
diff --git a/lib/galaxy/tools/wrappers.py b/lib/galaxy/tools/wrappers.py
index 296fa801bfd..9751d423211 100644
--- a/lib/galaxy/tools/wrappers.py
+++ b/lib/galaxy/tools/wrappers.py
@@ -6,6 +6,10 @@ from functools import total_ordering
from galaxy import exceptions
from galaxy.model.none_like import NoneDataset
+from galaxy.tools.parameters.wrapped_json import (
+ data_collection_input_to_staging_path_and_source_path,
+ data_input_to_staging_path_and_source_path,
+)
from galaxy.util import filesystem_safe_string
from galaxy.util.object_wrapper import wrap_with_safe_string
@@ -312,7 +316,7 @@ class DatasetFilenameWrapper(ToolParameterValueWrapper):
return f"{self.element_identifier}.{self.file_ext}"
@property
- def name_and_ext_filesystem_safe(self):
+ def staging_path(self):
"""
Strip leading dots, unicode null chars, replace `/` with `_`, truncate at 255 characters.
@@ -324,7 +328,10 @@ class DatasetFilenameWrapper(ToolParameterValueWrapper):
@property
def all_metadata_files(self):
- return self.unsanitized.get_metadata_file_paths_and_extensions()
+ return self.unsanitized.get_metadata_file_paths_and_extensions() if self else []
+
+ def serialize(self):
+ return data_input_to_staging_path_and_source_path(self) if self else {}
@property
def is_collection(self):
@@ -374,6 +381,8 @@ class DatasetFilenameWrapper(ToolParameterValueWrapper):
# instead of just returning a non-existent
# path like DiskObjectStore.
raise
+ elif key == 'serialize':
+ return self.serialize
else:
return getattr(self.dataset, key)
@@ -440,6 +449,9 @@ class DatasetListWrapper(list, ToolParameterValueWrapper, HasDatasets):
self._dataset_elements_cache[group] = wrappers
return self._dataset_elements_cache[group]
+ def serialize(self):
+ return [v.serialize() for v in self]
+
def __str__(self):
return ','.join(map(str, self))
@@ -531,11 +543,14 @@ class DatasetCollectionWrapper(ToolParameterValueWrapper, HasDatasets):
@property
def element_identifiers_extensions_paths_and_metadata_files(self):
if self._element_identifiers_extensions_paths_and_metadata_files is None:
- self._element_identifiers_extensions_paths_and_metadata_files = self.collection.element_identifiers_extensions_paths_and_metadata_files
+ if self.collection:
+ self._element_identifiers_extensions_paths_and_metadata_files = self.collection.element_identifiers_extensions_paths_and_metadata_files
+ else:
+ return []
return self._element_identifiers_extensions_paths_and_metadata_files
@property
- def all_element_identifiers_and_extensions_filesystem_safe(self):
+ def all_staging_paths(self):
safe_element_identifiers = []
for element_identifiers, extension, *_ in self.element_identifiers_extensions_paths_and_metadata_files:
datatype = self.datatypes_registry.get_datatype_by_extension(extension)
@@ -547,6 +562,9 @@ class DatasetCollectionWrapper(ToolParameterValueWrapper, HasDatasets):
safe_element_identifiers.append(f'{os.path.sep.join(current_element_identifiers)}.{extension}')
return safe_element_identifiers
+ def serialize(self):
+ return data_collection_input_to_staging_path_and_source_path(self)
+
@property
def is_input_supplied(self):
return self.__input_supplied
diff --git a/test/functional/tools/inputs_as_json_with_paths.xml b/test/functional/tools/inputs_as_json_with_paths.xml
index ea68dbf9da5..47e245280bc 100644
--- a/test/functional/tools/inputs_as_json_with_paths.xml
+++ b/test/functional/tools/inputs_as_json_with_paths.xml
@@ -33,6 +33,7 @@ if test_case == "1":
assert_equals(as_dict["section_example"]["section_text"], "section_default")
assert_equals(as_dict["data_input"], None)
assert_equals(as_dict["multiple_data_input"], [None])
+ assert as_dict["collection_input"] == []
elif test_case == "2":
assert_equals(as_dict["test_case"], 2)
assert_equals(as_dict["text_test"], "bar")
@@ -50,6 +51,11 @@ elif test_case == "2":
multiple_data_input = as_dict["multiple_data_input"]
assert type(multiple_data_input) is list
assert len(multiple_data_input) == 2
+ assert as_dict["collection_input"] == []
+elif test_case == "3":
+ collection_input = as_dict["collection_input"]
+ assert type(collection_input) is list
+ assert len(collection_input) == 1
with open("output", "w") as f:
f.write("okay\n")
@@ -69,6 +75,7 @@ with open("output", "w") as f:
+
@@ -133,6 +140,19 @@ with open("output", "w") as f:
+
+
+
+
+
+
+
+
+
+
+
+
+
Test tool demonstrating the special inputs config file.
diff --git a/test/functional/tools/inputs_as_json_with_staging_path_and_source_path.xml b/test/functional/tools/inputs_as_json_with_staging_path_and_source_path.xml
new file mode 100644
index 00000000000..6ef634017f7
--- /dev/null
+++ b/test/functional/tools/inputs_as_json_with_staging_path_and_source_path.xml
@@ -0,0 +1,46 @@
+
+
+ python '$check_inputs' '$inputs'
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ Test tool demonstrating the special inputs config file.
+
+
diff --git a/test/functional/tools/samples_tool_conf.xml b/test/functional/tools/samples_tool_conf.xml
index c28818edd11..7288d7f71ed 100644
--- a/test/functional/tools/samples_tool_conf.xml
+++ b/test/functional/tools/samples_tool_conf.xml
@@ -44,6 +44,7 @@
+
diff --git a/tools/data_export/export_remote.py b/tools/data_export/export_remote.py
index 31182042104..8044b8bb861 100644
--- a/tools/data_export/export_remote.py
+++ b/tools/data_export/export_remote.py
@@ -18,7 +18,7 @@ def check_for_duplicate_name(files_to_export):
seen = set()
duplicates = set()
for entry in files_to_export:
- name = entry['name']
+ name = entry['staging_path']
if name in seen:
duplicates.add(name)
seen.add(name)
@@ -42,25 +42,24 @@ def main(argv=None):
exit_code = 0
file_sources = get_file_sources(args.file_sources)
directory_uri = args.directory_uri
+ if not directory_uri.endswith("/"):
+ directory_uri = f"{directory_uri}/"
export_metadata_files = args.export_metadata_files
with open(args.files_to_export) as f:
files_to_export = json.load(f)
counter = 0
check_for_duplicate_name(files_to_export)
for entry in files_to_export:
- name = entry["name"]
- real_data_path = entry["real_data_path"]
- if directory_uri.endswith("/"):
- target_uri = directory_uri + name
- else:
- target_uri = directory_uri + "/" + name
+ name = entry["staging_path"]
+ real_data_path = entry["source_path"]
+ target_uri = f"{directory_uri}{name}"
if write_if_not_exists(file_sources, target_uri, real_data_path):
exit_code = 1
if export_metadata_files:
metadata_files = entry.get('metadata_files', [])
- for extension, path in metadata_files:
- metadata_file_uri = f"{target_uri}.{extension}"
- if write_if_not_exists(file_sources, metadata_file_uri, path):
+ for metadata_file in metadata_files:
+ metadata_file_uri = f"{directory_uri}{metadata_file['staging_path']}"
+ if write_if_not_exists(file_sources, metadata_file_uri, metadata_file['source_path']):
exit_code = 1
counter += 1
print(f"{counter} out of {len(files_to_export)} files have been exported.\n")
diff --git a/tools/data_export/export_remote.xml b/tools/data_export/export_remote.xml
index 39b8ac256fa..3ffaacfa4c4 100644
--- a/tools/data_export/export_remote.xml
+++ b/tools/data_export/export_remote.xml
@@ -18,11 +18,15 @@ python '$__tool_directory__/export_remote.py'
#import json
#from galaxy.util import filesystem_safe_string
#if $export_type.export_type_selector == "datasets_auto":
-#set $fileconfig = json.dumps([{"real_data_path": str(infile), "name": infile.name_and_ext_filesystem_safe, "metadata_files": infile.all_metadata_files} for infile in $infiles])
+#set $fileconfig = json.dumps($export_type.infiles.serialize())
#else if $export_type.export_type_selector == "collection_auto":
-#set $fileconfig = json.dumps([{"real_data_path": data_path, "name": identifier_and_extension, "metadata_files": metadata_files} for data_path, identifier_and_extension, metadata_files in zip($export_type.incollection.all_paths, $export_type.incollection.all_element_identifiers_and_extensions_filesystem_safe, $export_type.incollection.all_metadata_files)])
+#set $fileconfig = json.dumps($export_type.incollection.serialize())
#else
-#set $fileconfig = json.dumps([{"real_data_path": str(dataset["infile"]), "name": filesystem_safe_string(str(dataset["name"]), 255, invalid_chars=()), "metadata_files": dataset['infile'].all_metadata_files} for dataset in $export_type.datasets])
+#set $fileconfig = [d['infile'].serialize() for d in $export_type.datasets]
+#for $i, $dataset in enumerate($export_type.datasets):
+ #silent $fileconfig[$i]['staging_path'] = filesystem_safe_string(str(dataset["name"]), 255, invalid_chars=())
+#end for
+#silent fileconfig = json.dumps($fileconfig)
#end if
$fileconfig