From 2df6048550c9a2082eedfac80c86af6f28ef35c8 Mon Sep 17 00:00:00 2001
From: mvdbeek <m.vandenbeek@gmail.com>
Date: Tue, 10 Oct 2023 15:02:34 -0400
Subject: [PATCH] Make state change logic atomic

---
 lib/galaxy/jobs/__init__.py  |  5 +++--
 lib/galaxy/model/__init__.py | 23 ++++++++++++++++++++---
 2 files changed, 23 insertions(+), 5 deletions(-)

diff --git a/lib/galaxy/jobs/__init__.py b/lib/galaxy/jobs/__init__.py
index 2e7be38894f..e2d9cf90922 100644
--- a/lib/galaxy/jobs/__init__.py
+++ b/lib/galaxy/jobs/__init__.py
@@ -1548,9 +1548,10 @@ class MinimalJobWrapper(HasResourceParameters):
             return
         if info:
             job.info = info
-        job.set_state(state)
+        state_changed = job.set_state(state)
         self.sa_session.add(job)
-        job.update_output_states(self.app.application_stack.supports_skip_locked())
+        if state_changed:
+            job.update_output_states(self.app.application_stack.supports_skip_locked())
         if flush:
             with transaction(self.sa_session):
                 self.sa_session.commit()
diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py
index b16467a700f..28229cae905 100644
--- a/lib/galaxy/model/__init__.py
+++ b/lib/galaxy/model/__init__.py
@@ -1636,10 +1636,27 @@ class Job(Base, JobLike, UsesCreateAndUpdateTime, Dictifiable, Serializable):
 
     def set_state(self, state):
         """
-        Save state history
+        Save state history. Returns True if stat has changed, else False
         """
-        self.state = state
-        self.state_history.append(JobStateHistory(self))
+        if self.state == state:
+            # Nothing changed, no action needed
+            return False
+        session = object_session(self)
+        if session and self.id and not state in Job.terminal_states:
+            # generate statement that will not revert DELETING or DELETED back to anything non-terminal
+            rval = session.execute(update(Job.table).where(
+                Job.table.c.id == self.id,
+                ~Job.table.c.state.in_((Job.states.DELETING, Job.states.DELETED))
+            ).values(state=state))
+            if rval.rowcount == 1:
+                self.state_history.append(JobStateHistory(self))
+                return True
+            else:
+                return False
+        else:
+            self.state = state
+            self.state_history.append(JobStateHistory(self))
+            return True
 
     def get_param_values(self, app, ignore_errors=False):
         """