diff --git a/lib/galaxy/metadata/set_metadata.py b/lib/galaxy/metadata/set_metadata.py
index 39964d21bdb..afaae7602cd 100644
--- a/lib/galaxy/metadata/set_metadata.py
+++ b/lib/galaxy/metadata/set_metadata.py
@@ -139,6 +139,7 @@ def set_metadata_portable():
version_string = ""
export_store = None
+ final_job_state = Job.states.OK
if extended_metadata_collection:
tool_dict = metadata_params["tool"]
stdio_exit_code_dicts, stdio_regex_dicts = tool_dict["stdio_exit_codes"], tool_dict["stdio_regexes"]
@@ -198,7 +199,7 @@ def set_metadata_portable():
if os.path.exists(COMMAND_VERSION_FILENAME):
version_string = open(COMMAND_VERSION_FILENAME).read()
- job_context = ExpressionContext(dict(stdout=tool_stdout, stderr=tool_stderr))
+ expression_context = ExpressionContext(dict(stdout=tool_stdout, stderr=tool_stderr))
# Load outputs.
export_store = store.DirectoryModelExportStore('metadata/outputs_populated', serialize_dataset_objects=True, for_edit=True, strip_metadata_files=False, serialize_jobs=False)
@@ -208,6 +209,27 @@ def set_metadata_portable():
# Remove in 21.09, this should only happen for jobs that started on <= 20.09 and finish now
import_model_store = None
+ job_context = SessionlessJobContext(
+ metadata_params,
+ tool_provided_metadata,
+ object_store,
+ export_store,
+ import_model_store,
+ os.path.join(tool_job_working_directory, "working"),
+ final_job_state=final_job_state,
+ )
+
+ unnamed_id_to_path = {}
+ for unnamed_output_dict in job_context.tool_provided_metadata.get_unnamed_outputs():
+ destination = unnamed_output_dict["destination"]
+ elements = unnamed_output_dict["elements"]
+ destination_type = destination["type"]
+ if destination_type == 'hdas':
+ for element in elements:
+ filename = element.get('filename')
+ if filename:
+ unnamed_id_to_path[element['object_id']] = os.path.join(job_context.job_working_directory, filename)
+
for output_name, output_dict in outputs.items():
dataset_instance_id = output_dict["id"]
klass = getattr(galaxy.model, output_dict.get('model_class', 'HistoryDatasetAssociation'))
@@ -232,7 +254,7 @@ def set_metadata_portable():
# Same block as below...
set_meta_kwds = stringify_dictionary_keys(json.load(open(filename_kwds))) # load kwds; need to ensure our keywords are not unicode
try:
- dataset.dataset.external_filename = dataset_filename_override
+ dataset.dataset.external_filename = unnamed_id_to_path.get(dataset_instance_id, dataset_filename_override)
store_by = output_dict.get("object_store_store_by", legacy_object_store_store_by)
extra_files_dir_name = "dataset_%s_files" % getattr(dataset.dataset, store_by)
files_path = os.path.abspath(os.path.join(tool_job_working_directory, "working", extra_files_dir_name))
@@ -248,14 +270,17 @@ def set_metadata_portable():
setattr(dataset.metadata, metadata_name, metadata_file_override)
if output_dict.get("validate", False):
set_validated_state(dataset)
- set_meta(dataset, file_dict)
+ if dataset_instance_id not in unnamed_id_to_path:
+ # We're going to run through set_metadata in collect_dynamic_outputs with more contextual metadata,
+ # so skip set_meta here.
+ set_meta(dataset, file_dict)
if extended_metadata_collection:
meta = tool_provided_metadata.get_dataset_meta(output_name, dataset.dataset.id, dataset.dataset.uuid)
if meta:
- context = ExpressionContext(meta, job_context)
+ context = ExpressionContext(meta, expression_context)
else:
- context = job_context
+ context = expression_context
# Lazy and unattached
# if getattr(dataset, "hidden_beneath_collection_instance", None):
@@ -314,17 +339,6 @@ def set_metadata_portable():
if extended_metadata_collection:
# discover extra outputs...
-
- job_context = SessionlessJobContext(
- metadata_params,
- tool_provided_metadata,
- object_store,
- export_store,
- import_model_store,
- os.path.join(tool_job_working_directory, "working"),
- final_job_state=final_job_state,
- )
-
output_collections = {}
for name, output_collection in metadata_params["output_collections"].items():
output_collections[name] = import_model_store.sa_session.query(HistoryDatasetCollectionAssociation).find(output_collection["id"])
diff --git a/lib/galaxy/tool_util/verify/__init__.py b/lib/galaxy/tool_util/verify/__init__.py
index 8b21fb430c2..898878116f2 100644
--- a/lib/galaxy/tool_util/verify/__init__.py
+++ b/lib/galaxy/tool_util/verify/__init__.py
@@ -184,12 +184,12 @@ def _bam_to_sam(local_name, temp_name):
temp_local = tempfile.NamedTemporaryFile(suffix='.sam', prefix='local_bam_converted_to_sam_')
with tempfile.NamedTemporaryFile(suffix='.sam', prefix='history_bam_converted_to_sam_', delete=False) as temp:
try:
- pysam.view('-h', '-o%s' % temp_local.name, local_name)
+ pysam.view('-h', '--no-PG', '-o', temp_local.name, local_name, catch_stdout=False)
except Exception as e:
msg = "Converting local (test-data) BAM to SAM failed: %s" % unicodify(e)
raise Exception(msg)
try:
- pysam.view('-h', '-o%s' % temp.name, temp_name)
+ pysam.view('-h', '--no-PG', '-o', temp.name, temp_name, catch_stdout=False)
except Exception as e:
msg = "Converting history BAM to SAM failed: %s" % unicodify(e)
raise Exception(msg)
diff --git a/lib/galaxy_test/api/test_tools_upload.py b/lib/galaxy_test/api/test_tools_upload.py
index 4c3cc7dcb0a..4802b42b228 100644
--- a/lib/galaxy_test/api/test_tools_upload.py
+++ b/lib/galaxy_test/api/test_tools_upload.py
@@ -322,6 +322,28 @@ class ToolsUploadTestCase(ApiTestCase):
assert output0["state"] == "ok"
assert output1["state"] == "error"
+ @uses_test_history(require_new=False)
+ def test_fetch_bam_file_from_url_with_extension_set(self, history_id):
+ destination = {"type": "hdas"}
+ targets = [{
+ "destination": destination,
+ "items": [
+ {
+ "src": "url",
+ "url": "https://raw.githubusercontent.com/galaxyproject/galaxy/dev/test-data/1.bam",
+ "ext": "bam"
+ },
+ ]
+ }]
+ payload = {
+ "history_id": history_id,
+ "targets": json.dumps(targets),
+ }
+ fetch_response = self.dataset_populator.fetch(payload)
+ self._assert_status_code_is(fetch_response, 200)
+ outputs = fetch_response.json()["outputs"]
+ self.dataset_populator.get_history_dataset_details(history_id, dataset=outputs[0], assert_ok=True)
+
@skip_without_datatype("velvet")
def test_composite_datatype(self):
with self.dataset_populator.test_history() as history_id:
diff --git a/test/functional/tools/sam_to_bam.xml b/test/functional/tools/sam_to_bam.xml
index b01152a7670..3420ed73ec9 100644
--- a/test/functional/tools/sam_to_bam.xml
+++ b/test/functional/tools/sam_to_bam.xml
@@ -11,7 +11,7 @@ cat '$input1' > '$out_file1'
-
+