Merge remote-tracking branch 'upstream/release_18.09' into dev

lib/galaxy/model/metadata.py

@@ -523,7 +523,7 @@ class FileParameter(MetadataParameter):
         if value:
             new_value = galaxy.model.MetadataFile(dataset=target_context.parent, name=self.spec.name)
             object_session(target_context.parent).add(new_value)
-            object_session(target_context.parent).flush([new_value])
+            object_session(target_context.parent).flush()
             shutil.copy(value.file_name, new_value.file_name)
             return self.unwrap(new_value)
         return None
@@ -573,7 +573,7 @@ class FileParameter(MetadataParameter):
         if object_session(dataset):
             mf = galaxy.model.MetadataFile(name=self.spec.name, dataset=dataset, **kwds)
             object_session(dataset).add(mf)
-            object_session(dataset).flush([dataset, mf])  # flush to assign id
+            object_session(dataset).flush()  # flush to assign id
             return mf
         else:
             # we need to make a tmp file that is accessable to the head node,
@@ -776,7 +776,7 @@ class JobExternalOutputMetadataWrapper(object):
                 json.dump(override_metadata, open(metadata_files.filename_override_metadata, 'wt+'))
                 # add to session and flush
                 sa_session.add(metadata_files)
-                sa_session.flush([metadata_files])
+                sa_session.flush()
             metadata_files_list.append(metadata_files)
         args = '"%s" "%s" %s %s' % (metadata_path_on_compute(datatypes_config),
                                     job_metadata,

test/api/test_tools.py

@@ -803,6 +803,25 @@ class ToolsTestCase(api.ApiTestCase):
         output1_content = self.dataset_populator.get_history_dataset_content(history_id, dataset=output1)
         self.assertEqual(output1_content.strip(), "Cat1Test\nCat2Test")
 
+    @skip_without_tool("mapper_two")
+    @uses_test_history(require_new=False)
+    def test_bam_state_regression(self, history_id):
+        # Test regression of https://github.com/galaxyproject/galaxy/issues/6856. With changes
+        # to metadata file flushing to optimize creating bam outputs and copying bam datasets
+        # we observed very subtle problems with HDA state changes on other files being flushed at
+        # the same time. This tests txt datasets finalized before and after the bam outputs as
+        # well as other bam files all flush properly during job completion.
+        new_dataset1 = self.dataset_populator.new_dataset(history_id, content='123\n456\n789')
+        inputs = {
+            'input1': dataset_to_param(new_dataset1),
+            'reference': dataset_to_param(new_dataset1),
+        }
+        outputs = self._run_and_get_outputs('mapper_two', history_id, inputs)
+        assert len(outputs) == 4
+        for output in outputs:
+            details = self.dataset_populator.get_history_dataset_details(history_id, dataset=output)
+            assert details["state"] == "ok"
+
     @skip_without_tool("cat1")
     @uses_test_history(require_new=False)
     def test_multirun_cat1(self, history_id):

test/functional/tools/for_workflows/mapper_two.xml

@@ -0,0 +1,22 @@
+<tool id="mapper_two" name="Mapper producing two BAMs" version="0.1.0">
+    <command>
+        echo 'text 0' > '$text0';
+        cp '$__tool_directory__/1.bam' '$out_file1';
+        cp '$__tool_directory__/1.bam' '$out_file2';
+        echo 'text 1' > '$text1'
+    </command>
+    <inputs>
+        <param name="input1" type="data" format="txt" label="Fastq Input"/>
+        <param name="reference" type="data" format="txt" label="Fasta Input"/>
+    </inputs>
+    <outputs>
+        <data name="text0" format="txt" />
+        <data name="out_file1" format="bam" />
+        <data name="out_file2" format="bam" />
+        <data name="text1" format="txt" />
+    </outputs>
+    <tests>
+    </tests>
+    <help>
+    </help>
+</tool>

test/functional/tools/samples_tool_conf.xml

@@ -158,6 +158,7 @@
   <tool file="for_workflows/pileup.xml" />
   <tool file="for_workflows/mapper.xml" />
   <tool file="for_workflows/mapper2.xml" />
+  <tool file="for_workflows/mapper_two.xml" />
   <tool file="for_workflows/split.xml" />
   <tool file="for_workflows/empty_list.xml" />
   <tool file="for_workflows/count_list.xml" />