diff --git a/lib/galaxy/metadata/set_metadata.py b/lib/galaxy/metadata/set_metadata.py index 39964d21bdb..afaae7602cd 100644 --- a/lib/galaxy/metadata/set_metadata.py +++ b/lib/galaxy/metadata/set_metadata.py @@ -139,6 +139,7 @@ def set_metadata_portable(): version_string = "" export_store = None + final_job_state = Job.states.OK if extended_metadata_collection: tool_dict = metadata_params["tool"] stdio_exit_code_dicts, stdio_regex_dicts = tool_dict["stdio_exit_codes"], tool_dict["stdio_regexes"] @@ -198,7 +199,7 @@ def set_metadata_portable(): if os.path.exists(COMMAND_VERSION_FILENAME): version_string = open(COMMAND_VERSION_FILENAME).read() - job_context = ExpressionContext(dict(stdout=tool_stdout, stderr=tool_stderr)) + expression_context = ExpressionContext(dict(stdout=tool_stdout, stderr=tool_stderr)) # Load outputs. export_store = store.DirectoryModelExportStore('metadata/outputs_populated', serialize_dataset_objects=True, for_edit=True, strip_metadata_files=False, serialize_jobs=False) @@ -208,6 +209,27 @@ def set_metadata_portable(): # Remove in 21.09, this should only happen for jobs that started on <= 20.09 and finish now import_model_store = None + job_context = SessionlessJobContext( + metadata_params, + tool_provided_metadata, + object_store, + export_store, + import_model_store, + os.path.join(tool_job_working_directory, "working"), + final_job_state=final_job_state, + ) + + unnamed_id_to_path = {} + for unnamed_output_dict in job_context.tool_provided_metadata.get_unnamed_outputs(): + destination = unnamed_output_dict["destination"] + elements = unnamed_output_dict["elements"] + destination_type = destination["type"] + if destination_type == 'hdas': + for element in elements: + filename = element.get('filename') + if filename: + unnamed_id_to_path[element['object_id']] = os.path.join(job_context.job_working_directory, filename) + for output_name, output_dict in outputs.items(): dataset_instance_id = output_dict["id"] klass = getattr(galaxy.model, output_dict.get('model_class', 'HistoryDatasetAssociation')) @@ -232,7 +254,7 @@ def set_metadata_portable(): # Same block as below... set_meta_kwds = stringify_dictionary_keys(json.load(open(filename_kwds))) # load kwds; need to ensure our keywords are not unicode try: - dataset.dataset.external_filename = dataset_filename_override + dataset.dataset.external_filename = unnamed_id_to_path.get(dataset_instance_id, dataset_filename_override) store_by = output_dict.get("object_store_store_by", legacy_object_store_store_by) extra_files_dir_name = "dataset_%s_files" % getattr(dataset.dataset, store_by) files_path = os.path.abspath(os.path.join(tool_job_working_directory, "working", extra_files_dir_name)) @@ -248,14 +270,17 @@ def set_metadata_portable(): setattr(dataset.metadata, metadata_name, metadata_file_override) if output_dict.get("validate", False): set_validated_state(dataset) - set_meta(dataset, file_dict) + if dataset_instance_id not in unnamed_id_to_path: + # We're going to run through set_metadata in collect_dynamic_outputs with more contextual metadata, + # so skip set_meta here. + set_meta(dataset, file_dict) if extended_metadata_collection: meta = tool_provided_metadata.get_dataset_meta(output_name, dataset.dataset.id, dataset.dataset.uuid) if meta: - context = ExpressionContext(meta, job_context) + context = ExpressionContext(meta, expression_context) else: - context = job_context + context = expression_context # Lazy and unattached # if getattr(dataset, "hidden_beneath_collection_instance", None): @@ -314,17 +339,6 @@ def set_metadata_portable(): if extended_metadata_collection: # discover extra outputs... - - job_context = SessionlessJobContext( - metadata_params, - tool_provided_metadata, - object_store, - export_store, - import_model_store, - os.path.join(tool_job_working_directory, "working"), - final_job_state=final_job_state, - ) - output_collections = {} for name, output_collection in metadata_params["output_collections"].items(): output_collections[name] = import_model_store.sa_session.query(HistoryDatasetCollectionAssociation).find(output_collection["id"]) diff --git a/lib/galaxy/tool_util/verify/__init__.py b/lib/galaxy/tool_util/verify/__init__.py index 8b21fb430c2..898878116f2 100644 --- a/lib/galaxy/tool_util/verify/__init__.py +++ b/lib/galaxy/tool_util/verify/__init__.py @@ -184,12 +184,12 @@ def _bam_to_sam(local_name, temp_name): temp_local = tempfile.NamedTemporaryFile(suffix='.sam', prefix='local_bam_converted_to_sam_') with tempfile.NamedTemporaryFile(suffix='.sam', prefix='history_bam_converted_to_sam_', delete=False) as temp: try: - pysam.view('-h', '-o%s' % temp_local.name, local_name) + pysam.view('-h', '--no-PG', '-o', temp_local.name, local_name, catch_stdout=False) except Exception as e: msg = "Converting local (test-data) BAM to SAM failed: %s" % unicodify(e) raise Exception(msg) try: - pysam.view('-h', '-o%s' % temp.name, temp_name) + pysam.view('-h', '--no-PG', '-o', temp.name, temp_name, catch_stdout=False) except Exception as e: msg = "Converting history BAM to SAM failed: %s" % unicodify(e) raise Exception(msg) diff --git a/lib/galaxy_test/api/test_tools_upload.py b/lib/galaxy_test/api/test_tools_upload.py index 4c3cc7dcb0a..4802b42b228 100644 --- a/lib/galaxy_test/api/test_tools_upload.py +++ b/lib/galaxy_test/api/test_tools_upload.py @@ -322,6 +322,28 @@ class ToolsUploadTestCase(ApiTestCase): assert output0["state"] == "ok" assert output1["state"] == "error" + @uses_test_history(require_new=False) + def test_fetch_bam_file_from_url_with_extension_set(self, history_id): + destination = {"type": "hdas"} + targets = [{ + "destination": destination, + "items": [ + { + "src": "url", + "url": "https://raw.githubusercontent.com/galaxyproject/galaxy/dev/test-data/1.bam", + "ext": "bam" + }, + ] + }] + payload = { + "history_id": history_id, + "targets": json.dumps(targets), + } + fetch_response = self.dataset_populator.fetch(payload) + self._assert_status_code_is(fetch_response, 200) + outputs = fetch_response.json()["outputs"] + self.dataset_populator.get_history_dataset_details(history_id, dataset=outputs[0], assert_ok=True) + @skip_without_datatype("velvet") def test_composite_datatype(self): with self.dataset_populator.test_history() as history_id: diff --git a/test/functional/tools/sam_to_bam.xml b/test/functional/tools/sam_to_bam.xml index b01152a7670..3420ed73ec9 100644 --- a/test/functional/tools/sam_to_bam.xml +++ b/test/functional/tools/sam_to_bam.xml @@ -11,7 +11,7 @@ cat '$input1' > '$out_file1' - +