There's a brief window where we've committed a task to the database
but it doesn't have a task uuid yet
(https://github.com/galaxyproject/galaxy/blob/474a0536fadcddc1f6b328955e0cda1d6a281787/lib/galaxy/webapps/galaxy/services/histories.py#L397-L408).
This fails response validation as in
https://sentry.galaxyproject.org/share/issue/e0d4080044754c8aa60dd6bd8667dfcf/:
```
ExceptionGroup: unhandled errors in a TaskGroup
File "starlette/_utils.py", line 87, in collapse_excgroups
yield
File "starlette/middleware/base.py", line 190, in __call__
async with anyio.create_task_group() as task_group:
File "anyio/_backends/_asyncio.py", line 678, in __aexit__
raise BaseExceptionGroup(
ValidationError: 1 validation error for ExportTaskListResponse
0.task_uuid
UUID input should be a string, bytes or UUID object [type=uuid_type, input_value=None, input_type=NoneType]
For further information visit https://errors.pydantic.dev/2.5/v/uuid_type
File "starlette/applications.py", line 123, in __call__
await self.middleware_stack(scope, receive, send)
File "starlette/middleware/errors.py", line 186, in __call__
raise exc
File "starlette/middleware/errors.py", line 164, in __call__
await self.app(scope, receive, _send)
File "starlette_context/middleware/raw_middleware.py", line 92, in __call__
await self.app(scope, receive, send_wrapper)
File "starlette/middleware/base.py", line 189, in __call__
with collapse_excgroups():
File "contextlib.py", line 155, in __exit__
self.gen.throw(typ, value, traceback)
File "starlette/_utils.py", line 93, in collapse_excgroups
raise exc
File "starlette/middleware/base.py", line 191, in __call__
response = await self.dispatch_func(request, call_next)
File "galaxy/webapps/galaxy/fast_app.py", line 108, in add_x_frame_options
response = await call_next(request)
File "starlette/middleware/base.py", line 165, in call_next
raise app_exc
File "starlette/middleware/base.py", line 151, in coro
await self.app(scope, receive_or_disconnect, send_no_error)
File "starlette/middleware/exceptions.py", line 62, in __call__
await wrap_app_handling_exceptions(self.app, conn)(scope, receive, send)
File "starlette/_exception_handler.py", line 64, in wrapped_app
raise exc
File "starlette/_exception_handler.py", line 53, in wrapped_app
await app(scope, receive, sender)
File "starlette/routing.py", line 758, in __call__
await self.middleware_stack(scope, receive, send)
File "starlette/routing.py", line 778, in app
await route.handle(scope, receive, send)
File "starlette/routing.py", line 299, in handle
await self.app(scope, receive, send)
File "starlette/routing.py", line 79, in app
await wrap_app_handling_exceptions(app, request)(scope, receive, send)
File "starlette/_exception_handler.py", line 64, in wrapped_app
raise exc
File "starlette/_exception_handler.py", line 53, in wrapped_app
await app(scope, receive, sender)
File "starlette/routing.py", line 74, in app
response = await func(request)
File "fastapi/routing.py", line 278, in app
raw_response = await run_endpoint_function(
File "fastapi/routing.py", line 193, in run_endpoint_function
return await run_in_threadpool(dependant.call, **values)
File "starlette/concurrency.py", line 42, in run_in_threadpool
return await anyio.to_thread.run_sync(func, *args)
File "anyio/to_thread.py", line 56, in run_sync
return await get_async_backend().run_sync_in_worker_thread(
File "anyio/_backends/_asyncio.py", line 2144, in run_sync_in_worker_thread
return await future
File "anyio/_backends/_asyncio.py", line 851, in run
result = context.run(func, *args)
File "galaxy/webapps/galaxy/api/histories.py", line 503, in index_exports
return ExportTaskListResponse(root=exports)
```
Fixes:
```
ExceptionGroup: unhandled errors in a TaskGroup
File "starlette/_utils.py", line 87, in collapse_excgroups
yield
File "starlette/middleware/base.py", line 190, in __call__
async with anyio.create_task_group() as task_group:
File "anyio/_backends/_asyncio.py", line 678, in __aexit__
raise BaseExceptionGroup(
AttributeError: 'NoneType' object has no attribute 'id'
File "starlette/applications.py", line 123, in __call__
await self.middleware_stack(scope, receive, send)
File "starlette/middleware/errors.py", line 186, in __call__
raise exc
File "starlette/middleware/errors.py", line 164, in __call__
await self.app(scope, receive, _send)
File "starlette_context/middleware/raw_middleware.py", line 92, in __call__
await self.app(scope, receive, send_wrapper)
File "starlette/middleware/base.py", line 189, in __call__
with collapse_excgroups():
File "contextlib.py", line 155, in __exit__
self.gen.throw(typ, value, traceback)
File "starlette/_utils.py", line 93, in collapse_excgroups
raise exc
File "starlette/middleware/base.py", line 191, in __call__
response = await self.dispatch_func(request, call_next)
File "galaxy/webapps/galaxy/fast_app.py", line 108, in add_x_frame_options
response = await call_next(request)
File "starlette/middleware/base.py", line 165, in call_next
raise app_exc
File "starlette/middleware/base.py", line 151, in coro
await self.app(scope, receive_or_disconnect, send_no_error)
File "starlette/middleware/exceptions.py", line 62, in __call__
await wrap_app_handling_exceptions(self.app, conn)(scope, receive, send)
File "starlette/_exception_handler.py", line 64, in wrapped_app
raise exc
File "starlette/_exception_handler.py", line 53, in wrapped_app
await app(scope, receive, sender)
File "starlette/routing.py", line 758, in __call__
await self.middleware_stack(scope, receive, send)
File "starlette/routing.py", line 778, in app
await route.handle(scope, receive, send)
File "starlette/routing.py", line 299, in handle
await self.app(scope, receive, send)
File "starlette/routing.py", line 79, in app
await wrap_app_handling_exceptions(app, request)(scope, receive, send)
File "starlette/_exception_handler.py", line 64, in wrapped_app
raise exc
File "starlette/_exception_handler.py", line 53, in wrapped_app
await app(scope, receive, sender)
File "starlette/routing.py", line 74, in app
response = await func(request)
File "fastapi/routing.py", line 278, in app
raw_response = await run_endpoint_function(
File "fastapi/routing.py", line 193, in run_endpoint_function
return await run_in_threadpool(dependant.call, **values)
File "starlette/concurrency.py", line 42, in run_in_threadpool
return await anyio.to_thread.run_sync(func, *args)
File "anyio/to_thread.py", line 56, in run_sync
return await get_async_backend().run_sync_in_worker_thread(
File "anyio/_backends/_asyncio.py", line 2144, in run_sync_in_worker_thread
return await future
File "anyio/_backends/_asyncio.py", line 851, in run
result = context.run(func, *args)
File "galaxy/webapps/galaxy/api/workflows.py", line 1316, in index_invocations
invocations, total_matches = self.invocations_service.index(trans, invocation_payload, serialization_params)
File "galaxy/webapps/galaxy/services/invocations.py", line 111, in index
user_id = trans.user.id
```
from https://sentry.galaxyproject.org/share/issue/7d75e948d2db45039544435f1c4e1eda/
None isn't valid for `List[Any]`.
Fixes:
```
ValidationError
1 validation error for DataResult
data
Input should be a valid list [type=list_type, input_value=None, input_type=NoneType]
For further information visit https://errors.pydantic.dev/2.5/v/list_type
```
Here's the deadlock:
```
Traceback (most recent call last):
File "/home/runner/work/galaxy/galaxy/galaxy root/.venv/lib/python3.8/site-packages/sqlalchemy/engine/base.py", line 1910, in _execute_context
self.dialect.do_execute(
File "/home/runner/work/galaxy/galaxy/galaxy root/.venv/lib/python3.8/site-packages/sqlalchemy/engine/default.py", line 736, in do_execute
cursor.execute(statement, parameters)
psycopg2.errors.DeadlockDetected: deadlock detected
DETAIL: Process 317 waits for ShareLock on transaction 1057; blocked by process 318.
Process 318 waits for ShareLock on transaction 1056; blocked by process 317.
HINT: See server log for query details.
CONTEXT: while updating tuple (0,7) in relation "dataset"
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/home/runner/work/galaxy/galaxy/galaxy root/lib/galaxy/jobs/runners/__init__.py", line 203, in put
queue_job = job_wrapper.enqueue()
File "/home/runner/work/galaxy/galaxy/galaxy root/lib/galaxy/jobs/__init__.py", line 1589, in enqueue
self.change_state(model.Job.states.QUEUED, flush=False, job=job)
File "/home/runner/work/galaxy/galaxy/galaxy root/lib/galaxy/jobs/__init__.py", line 1547, in change_state
job.update_output_states(self.app.application_stack.supports_skip_locked())
File "/home/runner/work/galaxy/galaxy/galaxy root/lib/galaxy/model/__init__.py", line 2053, in update_output_states
sa_session.execute(statement, params)
File "/home/runner/work/galaxy/galaxy/galaxy root/.venv/lib/python3.8/site-packages/sqlalchemy/orm/session.py", line 1717, in execute
result = conn._execute_20(statement, params or {}, execution_options)
File "/home/runner/work/galaxy/galaxy/galaxy root/.venv/lib/python3.8/site-packages/sqlalchemy/engine/base.py", line 1710, in _execute_20
return meth(self, args_10style, kwargs_10style, execution_options)
File "/home/runner/work/galaxy/galaxy/galaxy root/.venv/lib/python3.8/site-packages/sqlalchemy/sql/elements.py", line 334, in _execute_on_connection
return connection._execute_clauseelement(
File "/home/runner/work/galaxy/galaxy/galaxy root/.venv/lib/python3.8/site-packages/sqlalchemy/engine/base.py", line 1577, in _execute_clauseelement
ret = self._execute_context(
File "/home/runner/work/galaxy/galaxy/galaxy root/.venv/lib/python3.8/site-packages/sqlalchemy/engine/base.py", line 1953, in _execute_context
self._handle_dbapi_exception(
File "/home/runner/work/galaxy/galaxy/galaxy root/.venv/lib/python3.8/site-packages/sqlalchemy/engine/base.py", line 2134, in _handle_dbapi_exception
util.raise_(
File "/home/runner/work/galaxy/galaxy/galaxy root/.venv/lib/python3.8/site-packages/sqlalchemy/util/compat.py", line 211, in raise_
raise exception
File "/home/runner/work/galaxy/galaxy/galaxy root/.venv/lib/python3.8/site-packages/sqlalchemy/engine/base.py", line 1910, in _execute_context
self.dialect.do_execute(
File "/home/runner/work/galaxy/galaxy/galaxy root/.venv/lib/python3.8/site-packages/sqlalchemy/engine/default.py", line 736, in do_execute
cursor.execute(statement, parameters)
sqlalchemy.exc.OperationalError: (psycopg2.errors.DeadlockDetected) deadlock detected
DETAIL: Process 317 waits for ShareLock on transaction 1057; blocked by process 318.
Process 318 waits for ShareLock on transaction 1056; blocked by process 317.
HINT: See server log for query details.
CONTEXT: while updating tuple (0,7) in relation "dataset"
[SQL:
UPDATE dataset
SET
state = %(state)s,
update_time = %(update_time)s
WHERE id IN (
SELECT hda.dataset_id FROM history_dataset_association hda
INNER JOIN job_to_output_dataset jtod
ON jtod.dataset_id = hda.id AND jtod.job_id = %(job_id)s
);
]
[parameters: {'state': 'queued', 'update_time': datetime.datetime(2024, 3, 7, 12, 29, 10, 229364), 'job_id': 3}]
(Background on this error at: https://sqlalche.me/e/14/e3q8)
```
The likely culprit for the deadlock is that __EXTRACT_DATASET__ deals
with the same dataset as the tool that created the collection
__EXTRACT_DATASET__ is running on, they might both be attempting to
update the output state.
My thinking is that by filtering on the job_id we're not going to change
state for the `__EXTRACT_DATASET__` change_state method.
mvdbeek