Merge branch 'release_21.01' into release_21.05

lib/galaxy/metadata/set_metadata.py

@@ -139,6 +139,7 @@ def set_metadata_portable():
     version_string = ""
 
     export_store = None
+    final_job_state = Job.states.OK
     if extended_metadata_collection:
         tool_dict = metadata_params["tool"]
         stdio_exit_code_dicts, stdio_regex_dicts = tool_dict["stdio_exit_codes"], tool_dict["stdio_regexes"]
@@ -198,7 +199,7 @@ def set_metadata_portable():
         if os.path.exists(COMMAND_VERSION_FILENAME):
             version_string = open(COMMAND_VERSION_FILENAME).read()
 
-        job_context = ExpressionContext(dict(stdout=tool_stdout, stderr=tool_stderr))
+        expression_context = ExpressionContext(dict(stdout=tool_stdout, stderr=tool_stderr))
 
         # Load outputs.
         export_store = store.DirectoryModelExportStore('metadata/outputs_populated', serialize_dataset_objects=True, for_edit=True, strip_metadata_files=False, serialize_jobs=False)
@@ -208,6 +209,27 @@ def set_metadata_portable():
         # Remove in 21.09, this should only happen for jobs that started on <= 20.09 and finish now
         import_model_store = None
 
+    job_context = SessionlessJobContext(
+        metadata_params,
+        tool_provided_metadata,
+        object_store,
+        export_store,
+        import_model_store,
+        os.path.join(tool_job_working_directory, "working"),
+        final_job_state=final_job_state,
+    )
+
+    unnamed_id_to_path = {}
+    for unnamed_output_dict in job_context.tool_provided_metadata.get_unnamed_outputs():
+        destination = unnamed_output_dict["destination"]
+        elements = unnamed_output_dict["elements"]
+        destination_type = destination["type"]
+        if destination_type == 'hdas':
+            for element in elements:
+                filename = element.get('filename')
+                if filename:
+                    unnamed_id_to_path[element['object_id']] = os.path.join(job_context.job_working_directory, filename)
+
     for output_name, output_dict in outputs.items():
         dataset_instance_id = output_dict["id"]
         klass = getattr(galaxy.model, output_dict.get('model_class', 'HistoryDatasetAssociation'))
@@ -232,7 +254,7 @@ def set_metadata_portable():
         # Same block as below...
         set_meta_kwds = stringify_dictionary_keys(json.load(open(filename_kwds)))  # load kwds; need to ensure our keywords are not unicode
         try:
-            dataset.dataset.external_filename = dataset_filename_override
+            dataset.dataset.external_filename = unnamed_id_to_path.get(dataset_instance_id, dataset_filename_override)
             store_by = output_dict.get("object_store_store_by", legacy_object_store_store_by)
             extra_files_dir_name = "dataset_%s_files" % getattr(dataset.dataset, store_by)
             files_path = os.path.abspath(os.path.join(tool_job_working_directory, "working", extra_files_dir_name))
@@ -248,14 +270,17 @@ def set_metadata_portable():
                 setattr(dataset.metadata, metadata_name, metadata_file_override)
             if output_dict.get("validate", False):
                 set_validated_state(dataset)
-            set_meta(dataset, file_dict)
+            if dataset_instance_id not in unnamed_id_to_path:
+                # We're going to run through set_metadata in collect_dynamic_outputs with more contextual metadata,
+                # so skip set_meta here.
+                set_meta(dataset, file_dict)
 
             if extended_metadata_collection:
                 meta = tool_provided_metadata.get_dataset_meta(output_name, dataset.dataset.id, dataset.dataset.uuid)
                 if meta:
-                    context = ExpressionContext(meta, job_context)
+                    context = ExpressionContext(meta, expression_context)
                 else:
-                    context = job_context
+                    context = expression_context
 
                 # Lazy and unattached
                 # if getattr(dataset, "hidden_beneath_collection_instance", None):
@@ -314,17 +339,6 @@ def set_metadata_portable():
 
     if extended_metadata_collection:
         # discover extra outputs...
-
-        job_context = SessionlessJobContext(
-            metadata_params,
-            tool_provided_metadata,
-            object_store,
-            export_store,
-            import_model_store,
-            os.path.join(tool_job_working_directory, "working"),
-            final_job_state=final_job_state,
-        )
-
         output_collections = {}
         for name, output_collection in metadata_params["output_collections"].items():
             output_collections[name] = import_model_store.sa_session.query(HistoryDatasetCollectionAssociation).find(output_collection["id"])

lib/galaxy/tool_util/verify/__init__.py

@@ -184,12 +184,12 @@ def _bam_to_sam(local_name, temp_name):
     temp_local = tempfile.NamedTemporaryFile(suffix='.sam', prefix='local_bam_converted_to_sam_')
     with tempfile.NamedTemporaryFile(suffix='.sam', prefix='history_bam_converted_to_sam_', delete=False) as temp:
         try:
-            pysam.view('-h', '-o%s' % temp_local.name, local_name)
+            pysam.view('-h', '--no-PG', '-o', temp_local.name, local_name, catch_stdout=False)
         except Exception as e:
             msg = "Converting local (test-data) BAM to SAM failed: %s" % unicodify(e)
             raise Exception(msg)
         try:
-            pysam.view('-h', '-o%s' % temp.name, temp_name)
+            pysam.view('-h', '--no-PG', '-o', temp.name, temp_name, catch_stdout=False)
         except Exception as e:
             msg = "Converting history BAM to SAM failed: %s" % unicodify(e)
             raise Exception(msg)

lib/galaxy_test/api/test_tools_upload.py

@@ -322,6 +322,28 @@ class ToolsUploadTestCase(ApiTestCase):
         assert output0["state"] == "ok"
         assert output1["state"] == "error"
 
+    @uses_test_history(require_new=False)
+    def test_fetch_bam_file_from_url_with_extension_set(self, history_id):
+        destination = {"type": "hdas"}
+        targets = [{
+            "destination": destination,
+            "items": [
+                {
+                    "src": "url",
+                    "url": "https://raw.githubusercontent.com/galaxyproject/galaxy/dev/test-data/1.bam",
+                    "ext": "bam"
+                },
+            ]
+        }]
+        payload = {
+            "history_id": history_id,
+            "targets": json.dumps(targets),
+        }
+        fetch_response = self.dataset_populator.fetch(payload)
+        self._assert_status_code_is(fetch_response, 200)
+        outputs = fetch_response.json()["outputs"]
+        self.dataset_populator.get_history_dataset_details(history_id, dataset=outputs[0], assert_ok=True)
+
     @skip_without_datatype("velvet")
     def test_composite_datatype(self):
         with self.dataset_populator.test_history() as history_id:

test/functional/tools/sam_to_bam.xml

@@ -11,7 +11,7 @@ cat '$input1' > '$out_file1'
     <tests>
         <test>
             <param name="input1" value="sam_with_header.sam" ftype="sam"/>
-            <output name="out_file1" file="bam_from_sam.bam"/>
+            <output name="out_file1" file="bam_from_sam.bam" ftype="bam"/>
         </test>
     </tests>
     <help>