Merge pull request #22513 from mvdbeek/sse-notifications

Server-Sent Events for history + notification updates
This commit is contained in:
John Chilton
2026-04-30 11:01:37 -04:00
committed by GitHub
77 changed files with 5205 additions and 302 deletions
@@ -19,6 +19,10 @@ env:
YARN_INSTALL_OPTS: --frozen-lockfile
GALAXY_CONFIG_SQLALCHEMY_WARN_20: '1'
GALAXY_DEPENDENCIES_INSTALL_WEASYPRINT: '1'
# TEMP: shake down SSE/notification system across full UI surface — revert before merge
GALAXY_CONFIG_OVERRIDE_ENABLE_NOTIFICATION_SYSTEM: '1'
GALAXY_CONFIG_OVERRIDE_ENABLE_SSE_HISTORY_UPDATES: '1'
GALAXY_CONFIG_OVERRIDE_ENABLE_SSE_ENTRY_POINT_UPDATES: '1'
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
+4
View File
@@ -20,6 +20,10 @@ env:
GALAXY_TEST_SELENIUM_HEADLESS: 1
YARN_INSTALL_OPTS: --frozen-lockfile
GALAXY_CONFIG_SQLALCHEMY_WARN_20: '1'
# TEMP: shake down SSE/notification system across full UI surface — revert before merge
GALAXY_CONFIG_OVERRIDE_ENABLE_NOTIFICATION_SYSTEM: '1'
GALAXY_CONFIG_OVERRIDE_ENABLE_SSE_HISTORY_UPDATES: '1'
GALAXY_CONFIG_OVERRIDE_ENABLE_SSE_ENTRY_POINT_UPDATES: '1'
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
+4
View File
@@ -19,6 +19,10 @@ env:
GALAXY_TEST_SKIP_FLAKEY_TESTS_ON_ERROR: 1
YARN_INSTALL_OPTS: --frozen-lockfile
GALAXY_CONFIG_SQLALCHEMY_WARN_20: '1'
# TEMP: shake down SSE/notification system across full UI surface — revert before merge
GALAXY_CONFIG_OVERRIDE_ENABLE_NOTIFICATION_SYSTEM: '1'
GALAXY_CONFIG_OVERRIDE_ENABLE_SSE_HISTORY_UPDATES: '1'
GALAXY_CONFIG_OVERRIDE_ENABLE_SSE_ENTRY_POINT_UPDATES: '1'
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
@@ -18,6 +18,8 @@ export const pendingRequestsMiddleware: Middleware = {
return new Request(request, { headers });
}
const shared = getPendingAbortSignal();
// Combine with any signal the caller may have set so we don't silently
// drop their cancellation semantics.
const signal = typeof AbortSignal.any === "function" ? AbortSignal.any([request.signal, shared]) : shared;
return new Request(request, { signal });
},
+67
View File
@@ -1292,6 +1292,33 @@ export interface paths {
patch?: never;
trace?: never;
};
"/api/events/stream": {
parameters: {
query?: never;
header?: never;
path?: never;
cookie?: never;
};
/**
* Server-Sent Events stream for real-time updates.
* @description Opens a Server-Sent Events (SSE) connection that pushes real-time
* updates for notifications, history changes, and other events.
*
* On reconnect, the browser sends the ``Last-Event-ID`` header automatically.
* If the notification system is enabled, any notifications created since that
* timestamp are delivered as a catch-up ``notification_status`` event.
*
* Anonymous users receive only broadcast events.
*/
get: operations["stream_events_api_events_stream_get"];
put?: never;
post?: never;
delete?: never;
options?: never;
head?: never;
patch?: never;
trace?: never;
};
"/api/exports": {
parameters: {
query?: never;
@@ -33451,6 +33478,46 @@ export interface operations {
};
};
};
stream_events_api_events_stream_get: {
parameters: {
query?: never;
header?: {
"Last-Event-ID"?: string | null;
/** @description The user ID that will be used to effectively make this API call. Only admins and designated users can make API calls on behalf of other users. */
"run-as"?: string | null;
};
path?: never;
cookie?: never;
};
requestBody?: never;
responses: {
/** @description Successful Response */
200: {
headers: {
[name: string]: unknown;
};
content?: never;
};
/** @description Request Error */
"4XX": {
headers: {
[name: string]: unknown;
};
content: {
"application/json": components["schemas"]["MessageExceptionModel"];
};
};
/** @description Server Error */
"5XX": {
headers: {
[name: string]: unknown;
};
content: {
"application/json": components["schemas"]["MessageExceptionModel"];
};
};
};
};
index_api_exports_get: {
parameters: {
query?: {
+3 -2
View File
@@ -16,8 +16,9 @@ import { useNotificationsStore } from "@/stores/notificationsStore";
* will use a fresh signal and is not affected.
*/
export function discardActiveConnectionsBeforeAuthNavigation() {
// Stop polling watchers first so they can't kick off new fetches, then
// abort any requests still in flight via the shared AbortController.
// Order: close SSE streams first (synchronous TCP close), then stop the
// polling watchers so they can't kick off new fetches, then abort any
// requests still in flight via the shared AbortController.
useHistoryStore().stopWatchingHistory();
useEntryPointStore().stopWatchingEntryPoints();
useNotificationsStore().stopWatchingNotifications();
@@ -0,0 +1,191 @@
import { onScopeDispose, ref } from "vue";
import { withPrefix } from "@/utils/redirect";
/**
* All SSE event types the server may emit.
*/
export const SSE_EVENT_TYPES = [
"notification_update",
"broadcast_update",
"notification_status",
"history_update",
"entry_point_update",
] as const;
export type SSEEventType = (typeof SSE_EVENT_TYPES)[number];
interface SSEDebugGlobals {
__galaxy_sse_connected?: boolean;
__galaxy_sse_last_event_ts?: number;
}
function sseGlobals(): SSEDebugGlobals {
return window as unknown as SSEDebugGlobals;
}
// ---------------------------------------------------------------------------
// Module-level shared EventSource.
//
// Every call to ``useSSE`` registers its handler against this one socket so
// the tab opens a single ``/api/events/stream`` connection no matter how many
// stores listen. HTTP/1.1 caps simultaneous connections per origin at six;
// before this consolidation we burned three slots on SSE alone (history,
// notifications, entry points), which is what starved the scratchbook iframe
// flow — see the fix in ``client/src/entry/analysis/App.vue``.
// ---------------------------------------------------------------------------
type Handler = (event: MessageEvent) => void;
let sharedSource: EventSource | null = null;
const sharedConnected = ref(false);
const subscribers: Map<SSEEventType, Set<Handler>> = new Map();
// Track the per-type dispatchers we registered so ``closeSource`` removes the
// exact same listeners (``addEventListener`` matches by reference).
const dispatchers: Map<SSEEventType, Handler> = new Map();
function openSourceIfNeeded() {
if (sharedSource) {
return;
}
sharedSource = new EventSource(withPrefix("/api/events/stream"));
for (const eventType of SSE_EVENT_TYPES) {
const dispatcher: Handler = (event) => {
// Selenium tests watch ``__galaxy_sse_last_event_ts`` to prove that
// an observable state change came from an SSE push and not the
// polling fallback (where the global would never advance).
sseGlobals().__galaxy_sse_last_event_ts = Date.now();
const subs = subscribers.get(eventType);
if (!subs) {
return;
}
for (const handler of subs) {
handler(event);
}
};
dispatchers.set(eventType, dispatcher);
sharedSource.addEventListener(eventType, dispatcher);
}
sharedSource.onopen = () => {
sharedConnected.value = true;
// Global readiness flag so Selenium tests can distinguish a working
// SSE pipeline from the polling fallback.
sseGlobals().__galaxy_sse_connected = true;
};
sharedSource.onerror = () => {
// EventSource auto-reconnects natively; SSE-vs-polling is a
// config-level decision (see historyStore / notificationsStore), so
// we must not give up on transient errors here — doing so would leave
// the client with no updates at all.
sharedConnected.value = false;
sseGlobals().__galaxy_sse_connected = false;
};
// Browser EventSource teardown during a full-page navigation
// (``window.location.href = …``) is not guaranteed to happen before the
// browser issues requests for the new page — we've seen Chrome keep the
// stream alive long enough that a login/register POST reload races the
// close, and the new page then loads with a stale auth view. Force a
// synchronous ``close()`` during ``pagehide`` (fires for both reloads and
// tab-close, unlike ``beforeunload``) to close that window.
if (typeof window !== "undefined") {
window.addEventListener("pagehide", closeSource);
}
}
function closeSource() {
if (!sharedSource) {
return;
}
for (const [eventType, dispatcher] of dispatchers) {
sharedSource.removeEventListener(eventType, dispatcher);
}
dispatchers.clear();
sharedSource.close();
sharedSource = null;
sharedConnected.value = false;
sseGlobals().__galaxy_sse_connected = false;
if (typeof window !== "undefined") {
window.removeEventListener("pagehide", closeSource);
}
}
function addSubscriber(onEvent: Handler, eventTypes: readonly SSEEventType[]) {
for (const eventType of eventTypes) {
let subs = subscribers.get(eventType);
if (!subs) {
subs = new Set();
subscribers.set(eventType, subs);
}
subs.add(onEvent);
}
}
function removeSubscriber(onEvent: Handler, eventTypes: readonly SSEEventType[]): boolean {
let anyRemaining = false;
for (const eventType of eventTypes) {
const subs = subscribers.get(eventType);
if (subs) {
subs.delete(onEvent);
if (subs.size === 0) {
subscribers.delete(eventType);
}
}
}
for (const subs of subscribers.values()) {
if (subs.size > 0) {
anyRemaining = true;
break;
}
}
return anyRemaining;
}
/**
* Composable for subscribing to events on the shared SSE stream.
*
* The browser's EventSource handles reconnection automatically and sends the
* ``Last-Event-ID`` header so the server can catch up on missed events. Only
* one EventSource is opened per tab regardless of how many callers invoke
* this composable; the composable multiplexes dispatch per event type.
*
* @param onEvent - callback invoked for every matching SSE event
* @param eventTypes - subset of event types to listen to (defaults to all)
*/
export function useSSE(onEvent: Handler, eventTypes: readonly SSEEventType[] = SSE_EVENT_TYPES) {
let connected_: boolean = false;
function connect() {
if (connected_) {
return;
}
connected_ = true;
addSubscriber(onEvent, eventTypes);
openSourceIfNeeded();
}
function disconnect() {
if (!connected_) {
return;
}
connected_ = false;
const anyRemaining = removeSubscriber(onEvent, eventTypes);
if (!anyRemaining) {
closeSource();
}
}
onScopeDispose(() => {
disconnect();
});
return { connect, disconnect, connected: sharedConnected };
}
/**
* @deprecated Use `useSSE` instead. This alias exists for backward compatibility.
*/
export const useNotificationSSE = useSSE;
+23 -3
View File
@@ -48,7 +48,7 @@
</template>
<script>
import { storeToRefs } from "pinia";
import { ref, watch } from "vue";
import { computed, ref, watch } from "vue";
import { useRoute } from "vue-router/composables";
import { getGalaxyInstance } from "@/app";
@@ -105,9 +105,29 @@ export default {
const uploadModal = ref(null);
setGlobalUploadModal(uploadModal);
const embedded = useRouteQueryBool("embed");
// Treat any iframe context as embedded: scratchbook pops dataset
// displays into ``WinBox`` iframes that hit the same routes without
// an ``embed`` query param, and each one would otherwise open its own
// SSE + polling traffic, quickly saturating the HTTP/1.1 per-origin
// connection pool (e.g. ``test_scratchbook_window_persistence`` hangs
// indefinitely after two windows are open).
const inIframe = (() => {
if (typeof window === "undefined") {
return false;
}
try {
return window.top !== window.self;
} catch {
// Cross-origin access throws — that's definitely an iframe.
return true;
}
})();
const embeddedQuery = useRouteQueryBool("embed");
const embedded = computed(() => embeddedQuery.value || inIframe);
const historyStore = useHistoryStore();
historyStore.startWatchingHistory();
if (!embedded.value) {
historyStore.startWatchingHistory();
}
watch(
() => embedded.value,
@@ -0,0 +1,74 @@
/**
* Shared test helpers for the SSE-driven stores (historyStore, notificationsStore).
*
* Both stores consume the same `useSSE` composable and need:
* - a mock that captures the onEvent callback so tests can synthesize SSE messages;
* - visibility-state patching without leaking across tests (JSDOM's `document`
* is shared by every test in the same worker, so an unrestored
* `Object.defineProperty` causes silent bleed).
*
* Because ``vi.mock`` is hoisted above module-level variables, tests must
* construct the SSE-mock state via ``vi.hoisted`` and then hand it to
* ``sseMockFactory`` from inside the ``vi.mock`` factory. See the ``.test.ts``
* files in this directory for the pattern.
*/
import { vi } from "vitest";
import { type Ref, ref } from "vue";
export interface SSEMockState {
onEvent: ((event: MessageEvent) => void) | null;
connect: ReturnType<typeof vi.fn>;
disconnect: ReturnType<typeof vi.fn>;
connected?: Ref<boolean>;
}
/** Build the factory used with ``vi.mock("@/composables/useNotificationSSE", ...)``. */
export function sseMockFactory(state: SSEMockState) {
// Lazily initialize ``connected`` so existing callers that don't pass it
// still get a working ref.
if (!state.connected) {
state.connected = ref(false);
}
return {
useSSE: vi.fn((onEvent: (event: MessageEvent) => void) => {
state.onEvent = onEvent;
return { connect: state.connect, disconnect: state.disconnect, connected: state.connected };
}),
};
}
/** Synthesize an SSE message through the captured handler. */
export function emitSse(state: SSEMockState, type: string, payload: unknown): void {
if (!state.onEvent) {
throw new Error("useSSE was not called by the store under test — cannot emit an SSE event");
}
state.onEvent(new MessageEvent(type, { data: JSON.stringify(payload) }));
}
/**
* Save the current ``document.visibilityState`` descriptor and return a restorer.
* Call the restorer in ``afterEach`` to prevent patching from leaking into later tests.
*/
export function useVisibilityPatch(): {
set: (state: "visible" | "hidden") => void;
restore: () => void;
} {
const original = Object.getOwnPropertyDescriptor(document, "visibilityState");
return {
set(state: "visible" | "hidden") {
Object.defineProperty(document, "visibilityState", {
configurable: true,
get: () => state,
});
document.dispatchEvent(new Event("visibilitychange"));
},
restore() {
if (original) {
Object.defineProperty(document, "visibilityState", original);
} else {
delete (document as unknown as Record<string, unknown>).visibilityState;
}
},
};
}
+14 -1
View File
@@ -1,12 +1,25 @@
import flushPromises from "flush-promises";
import { createPinia, setActivePinia } from "pinia";
import { beforeEach, describe, expect, it } from "vitest";
import { beforeEach, describe, expect, it, vi } from "vitest";
import { HttpResponse, useServerMock } from "@/api/client/__mocks__";
import testInteractiveToolsResponse from "../components/InteractiveTools/testData/testInteractiveToolsResponse";
import { sseMockFactory } from "./_testing/sseStoreSupport";
import { useEntryPointStore } from "./entryPointStore";
// ``vi.mock`` is hoisted above module-level declarations, so the capture-state
// has to be built via ``vi.hoisted`` to be visible to the factory. Prevents
// these tests from opening a real EventSource against ``/api/events/stream``
// when ``useEntryPointStore()`` is invoked.
const sseState = vi.hoisted(() => ({
onEvent: null,
connect: vi.fn(),
disconnect: vi.fn(),
connected: null,
}));
vi.mock("@/composables/useNotificationSSE", () => sseMockFactory(sseState));
const { server, http } = useServerMock();
describe("stores/EntryPointStore", () => {
+93 -16
View File
@@ -1,10 +1,12 @@
import axios from "axios";
import isEqual from "lodash.isequal";
import { defineStore } from "pinia";
import { computed, ref } from "vue";
import { computed, ref, watch } from "vue";
import { useResourceWatcher } from "@/composables/resourceWatcher";
import { useSSE } from "@/composables/useNotificationSSE";
import { getAppRoot } from "@/onload/loadConfig";
import { useConfigStore } from "@/stores/configurationStore";
import { rethrowSimple } from "@/utils/simple-error";
const ACTIVE_POLLING_INTERVAL = 10000;
@@ -23,23 +25,8 @@ interface EntryPoint {
}
export const useEntryPointStore = defineStore("entryPointStore", () => {
const { startWatchingResource: startWatchingEntryPoints, stopWatchingResource: stopWatchingEntryPoints } =
useResourceWatcher(fetchEntryPoints, {
shortPollingInterval: ACTIVE_POLLING_INTERVAL,
enableBackgroundPolling: false, // No need to poll in the background
});
const entryPoints = ref<EntryPoint[]>([]);
const entryPointsForJob = computed(() => {
return (jobId: string) => entryPoints.value.filter((entryPoint) => entryPoint["job_id"] === jobId);
});
const entryPointsForHda = computed(() => {
return (hdaId: string) =>
entryPoints.value.filter((entryPoint) => entryPoint["output_datasets_ids"].includes(hdaId));
});
async function fetchEntryPoints() {
const url = `${getAppRoot()}api/entry_points`;
const params = { running: true };
@@ -51,6 +38,80 @@ export const useEntryPointStore = defineStore("entryPointStore", () => {
}
}
// SSE-driven path: on each entry_point_update signal, refetch the canonical
// list from REST. The event carries no data — it's a pure wake-up.
function handleEntryPointSSEEvent(_event: MessageEvent) {
fetchEntryPoints().catch((err) => console.error("Error refreshing entry points from SSE push:", err));
}
const {
connect: sseConnect,
disconnect: sseDisconnect,
connected: sseConnected,
} = useSSE(handleEntryPointSSEEvent, ["entry_point_update"]);
let stopPolling: (() => void) | null = null;
let stopConnectedWatcher: (() => void) | null = null;
let watchingInitialized = false;
// Callers opt in via ``startWatchingEntryPoints()`` (App.vue gates this on
// ``interactivetools_enable``). We then pick SSE or polling based on the
// server flag — mutually exclusive, mirroring historyStore / notificationsStore.
// ``useConfigStore`` is resolved lazily here so tests that only exercise
// the data methods don't need a ``/api/configuration`` handler registered.
function startWatchingEntryPoints() {
if (watchingInitialized) {
return;
}
watchingInitialized = true;
const configStore = useConfigStore();
const decide = () => {
if (configStore.config?.enable_sse_updates) {
// Baseline fetch + SSE. Reconnect-refetch closes the "user
// navigated away and missed events" window.
fetchEntryPoints().catch((err) => console.warn("Initial entry-point load failed", err));
sseConnect();
stopConnectedWatcher = watch(sseConnected, (isConnected, wasConnected) => {
if (isConnected && !wasConnected) {
fetchEntryPoints().catch((err) =>
console.error("Error refreshing entry points on SSE reconnect:", err),
);
}
});
} else {
const { startWatchingResource, stopWatchingResource } = useResourceWatcher(fetchEntryPoints, {
shortPollingInterval: ACTIVE_POLLING_INTERVAL,
enableBackgroundPolling: false,
});
stopPolling = stopWatchingResource;
startWatchingResource();
}
};
if (configStore.isLoaded) {
decide();
} else {
const stop = watch(
() => configStore.isLoaded,
(loaded) => {
if (loaded) {
stop();
decide();
}
},
);
}
}
const entryPointsForJob = computed(() => {
return (jobId: string) => entryPoints.value.filter((entryPoint) => entryPoint["job_id"] === jobId);
});
const entryPointsForHda = computed(() => {
return (hdaId: string) =>
entryPoints.value.filter((entryPoint) => entryPoint["output_datasets_ids"].includes(hdaId));
});
function updateEntryPoints(data: EntryPoint[]) {
let hasChanged = entryPoints.value.length !== data.length ? true : false;
if (entryPoints.value.length === 0) {
@@ -85,6 +146,22 @@ export const useEntryPointStore = defineStore("entryPointStore", () => {
}
}
// Closes the SSE stream and stops the polling watcher; paired with login
// /register flows so background traffic doesn't outlive the navigation
// and clobber the freshly authenticated session cookie.
function stopWatchingEntryPoints() {
sseDisconnect();
if (stopPolling) {
stopPolling();
stopPolling = null;
}
if (stopConnectedWatcher) {
stopConnectedWatcher();
stopConnectedWatcher = null;
}
watchingInitialized = false;
}
return {
entryPoints,
entryPointsForJob,
+194
View File
@@ -0,0 +1,194 @@
import flushPromises from "flush-promises";
import { createPinia, setActivePinia } from "pinia";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { useServerMock } from "@/api/client/__mocks__";
import { emitSse, sseMockFactory, useVisibilityPatch } from "./_testing/sseStoreSupport";
import { useHistoryStore } from "./historyStore";
// ``vi.mock`` is hoisted above module-level ``const`` declarations, so the
// capture-state has to be built via ``vi.hoisted`` to be visible to the factory.
const sseState = vi.hoisted(() => {
return {
onEvent: null as ((event: MessageEvent) => void) | null,
connect: vi.fn(),
disconnect: vi.fn(),
};
});
vi.mock("@/composables/useNotificationSSE", () => sseMockFactory(sseState));
// `watchHistory(app)` is the polling handler invoked on the short/long
// interval. We mock it so each invocation is observable without pulling in
// the history-items store, dataset store, and Galaxy app instance.
const mockWatchHistory = vi.fn().mockResolvedValue(undefined);
const mockRefreshHistoryFromPush = vi.fn().mockResolvedValue(undefined);
vi.mock("@/watch/watchHistory", () => ({
ACTIVE_POLLING_INTERVAL: 3000,
INACTIVE_POLLING_INTERVAL: 60_000,
watchHistory: (app: unknown) => mockWatchHistory(app),
refreshHistoryFromPush: (app: unknown) => mockRefreshHistoryFromPush(app),
}));
vi.mock("@/app", () => ({
getGalaxyInstance: () => ({ name: "fake-galaxy" }),
}));
const { server, http } = useServerMock();
function registerDefaultHandlers({ enableSse }: { enableSse: boolean }) {
server.use(
http.get("/api/configuration", ({ response }) => {
// eslint-disable-next-line @typescript-eslint/no-explicit-any
return response(200).json({ enable_sse_updates: enableSse } as any);
}),
);
}
async function primeStore(startFn: () => void): Promise<void> {
startFn();
// Config load is async; let the watch fire and the initial fetch complete.
await flushPromises();
await vi.runOnlyPendingTimersAsync();
await flushPromises();
}
describe("historyStore — config-driven SSE vs polling", () => {
let visibility: ReturnType<typeof useVisibilityPatch>;
beforeEach(() => {
setActivePinia(createPinia());
sseState.connect.mockClear();
sseState.disconnect.mockClear();
sseState.onEvent = null;
mockWatchHistory.mockClear();
mockRefreshHistoryFromPush.mockClear();
vi.useFakeTimers();
visibility = useVisibilityPatch();
});
afterEach(() => {
visibility.restore();
vi.useRealTimers();
});
describe("when enable_sse_updates is true (SSE scenario)", () => {
beforeEach(() => {
registerDefaultHandlers({ enableSse: true });
});
it("primes the store with one initial load, connects SSE, and does not keep polling", async () => {
const store = useHistoryStore();
await primeStore(() => store.startWatchingHistory());
expect(sseState.connect).toHaveBeenCalledTimes(1);
// One-shot initial fetch so the history panel isn't empty before
// the first SSE event arrives.
expect(mockWatchHistory).toHaveBeenCalledTimes(1);
// Advance past the short polling interval (3s) several times and
// confirm the polling handler is not invoked a second time in SSE mode.
vi.advanceTimersByTime(30_000);
await flushPromises();
expect(mockWatchHistory).toHaveBeenCalledTimes(1);
});
it("does not start polling when the tab regains visibility", async () => {
const store = useHistoryStore();
await primeStore(() => store.startWatchingHistory());
expect(mockWatchHistory).toHaveBeenCalledTimes(1);
// Simulate a tab hide/show cycle. `useResourceWatcher` registers
// a `visibilitychange` listener whose handler calls
// `startWatchingResourceIfNeeded` — in SSE mode that would
// silently resume polling. Because we never instantiated the
// watcher, no listener should exist and no poll should fire.
visibility.set("hidden");
visibility.set("visible");
await flushPromises();
vi.advanceTimersByTime(30_000);
await flushPromises();
expect(mockWatchHistory).toHaveBeenCalledTimes(1);
});
it("triggers refreshHistoryFromPush when an SSE event names the current history", async () => {
// This test asserts the store's *decision* to refresh, not the refresh
// itself — ``refreshHistoryFromPush`` is mocked so we can observe the
// dispatch. The real refresh is covered end-to-end in the Selenium
// SSE integration tests (see test/integration_selenium/test_history_sse.py).
const store = useHistoryStore();
await primeStore(() => store.startWatchingHistory());
// Drive the store to a known current-history id so the handler has
// something to match against. ``currentHistoryId`` is a computed
// that only returns the stored id when the history is present in
// ``storedHistories``, so the history has to be registered too.
// eslint-disable-next-line @typescript-eslint/no-explicit-any
store.setHistory({ id: "hist-1" } as any);
store.setCurrentHistoryId("hist-1");
mockRefreshHistoryFromPush.mockClear();
emitSse(sseState, "history_update", { history_ids: ["hist-1", "hist-2"] });
await flushPromises();
expect(mockRefreshHistoryFromPush).toHaveBeenCalledTimes(1);
});
it("ignores SSE history events that do not include the current history", async () => {
const store = useHistoryStore();
await primeStore(() => store.startWatchingHistory());
store.setCurrentHistoryId("hist-1");
mockRefreshHistoryFromPush.mockClear();
emitSse(sseState, "history_update", { history_ids: ["hist-2"] });
await flushPromises();
expect(mockRefreshHistoryFromPush).not.toHaveBeenCalled();
});
});
describe("when enable_sse_updates is false (polling scenario)", () => {
beforeEach(() => {
registerDefaultHandlers({ enableSse: false });
});
it("does not connect SSE and polls on the configured interval", async () => {
const store = useHistoryStore();
await primeStore(() => store.startWatchingHistory());
expect(sseState.connect).not.toHaveBeenCalled();
// The resource watcher invokes the handler immediately on start
// and then re-schedules after each completion. Advance past the
// short interval and confirm repeated invocations.
const initialCalls = mockWatchHistory.mock.calls.length;
expect(initialCalls).toBeGreaterThanOrEqual(1);
await vi.advanceTimersByTimeAsync(3000);
await flushPromises();
expect(mockWatchHistory.mock.calls.length).toBeGreaterThan(initialCalls);
});
it("calling startWatchingHistory again is idempotent (no second SSE, polling tick count +1 only)", async () => {
const store = useHistoryStore();
await primeStore(() => store.startWatchingHistory());
const pollsAfterFirst = mockWatchHistory.mock.calls.length;
store.startWatchingHistory();
await flushPromises();
expect(sseState.connect).not.toHaveBeenCalled();
// Calling again must not schedule a second independent polling loop.
// Advance past one interval and confirm only one handler tick fires,
// not two.
await vi.advanceTimersByTimeAsync(3000);
await flushPromises();
// Exactly one additional poll after the 3000ms advance — anything
// else means a second independent polling loop was scheduled.
const deltaAfterSecond = mockWatchHistory.mock.calls.length - pollsAfterFirst;
expect(deltaAfterSecond).toBe(1);
});
});
});
+110 -10
View File
@@ -1,5 +1,5 @@
import { defineStore } from "pinia";
import { computed, del, ref, set } from "vue";
import { computed, del, ref, set, watch } from "vue";
import {
type AnyHistory,
@@ -15,7 +15,9 @@ import type { ArchivedHistoryDetailed } from "@/api/histories.archived";
import { getGalaxyInstance } from "@/app";
import { HistoryFilters } from "@/components/History/HistoryFilters";
import { useResourceWatcher } from "@/composables/resourceWatcher";
import { useSSE } from "@/composables/useNotificationSSE";
import { useUserLocalStorage } from "@/composables/userLocalStorage";
import { useConfigStore } from "@/stores/configurationStore";
import {
createAndSelectNewHistory,
getCurrentHistoryFromServer,
@@ -30,6 +32,7 @@ import { sortByObjectProp } from "@/utils/sorting";
import {
ACTIVE_POLLING_INTERVAL,
INACTIVE_POLLING_INTERVAL,
refreshHistoryFromPush as refreshHistoryFromPushSuppliedApp,
watchHistory as watchHistorySuppliedApp,
} from "@/watch/watchHistory";
@@ -391,14 +394,94 @@ export const useHistoryStore = defineStore("historyStore", () => {
return watchHistorySuppliedApp(app);
}
const {
startWatchingResource: startWatchingHistory,
stopWatchingResource: stopWatchingHistory,
isWatchingResource: isWatchingHistory,
} = useResourceWatcher(watchHistory, {
shortPollingInterval: ACTIVE_POLLING_INTERVAL,
longPollingInterval: INACTIVE_POLLING_INTERVAL,
});
// SSE-driven history updates: when we receive a history_update event,
// immediately trigger a refresh of the current history
const SSE_HISTORY_EVENT_TYPES = ["history_update"] as const;
const { connect: sseHistoryConnect, disconnect: sseHistoryDisconnect } = useSSE(
handleHistorySSEEvent,
SSE_HISTORY_EVENT_TYPES,
);
let stopHistoryPolling: (() => void) | null = null;
let stopIsWatchingWatcher: (() => void) | null = null;
function handleHistorySSEEvent(event: MessageEvent) {
try {
const data = JSON.parse(event.data);
const changedHistoryIds: string[] = data.history_ids ?? [];
if (currentHistoryId.value && changedHistoryIds.includes(currentHistoryId.value)) {
// SSE is itself the signal that the history changed — force the
// refresh so the update_time short-circuit in watchHistoryOnce
// can't suppress the contents fetch.
const app = getGalaxyInstance();
refreshHistoryFromPushSuppliedApp(app).catch((err) =>
console.error("Error refreshing history from SSE push:", err),
);
}
} catch (e) {
console.error("Error handling history SSE event:", e);
}
}
// Choose between SSE and polling based on the server config flag
// `enable_sse_updates`. SSE success at the socket level is not a
// reliable proxy: the `/api/events/stream` endpoint accepts connections
// even when the HistoryAuditMonitor is disabled, so relying on the
// EventSource `connected` state would silently stop polling without any
// events ever arriving.
//
// `useResourceWatcher` is instantiated lazily because it registers a
// `visibilitychange` listener that calls `startWatchingResourceIfNeeded`
// every time the tab regains focus — in SSE mode that would re-start
// polling we explicitly don't want.
const isWatchingHistory = ref(false);
let watchingInitialized = false;
function startWatchingHistoryWithSSE() {
if (watchingInitialized) {
return;
}
watchingInitialized = true;
const configStore = useConfigStore();
const decide = () => {
if (configStore.config?.enable_sse_updates) {
// SSE delivers incremental updates only; the store still needs
// a baseline fetch so the history panel isn't empty until the
// first change arrives.
watchHistory().catch((err) => console.warn("Initial history load failed", err));
sseHistoryConnect();
} else {
// The resource watcher fires its handler once immediately and
// then re-schedules on the polling interval, which covers the
// initial load as well as ongoing updates.
const { startWatchingResource, stopWatchingResource, isWatchingResource } = useResourceWatcher(
watchHistory,
{
shortPollingInterval: ACTIVE_POLLING_INTERVAL,
longPollingInterval: INACTIVE_POLLING_INTERVAL,
},
);
stopHistoryPolling = stopWatchingResource;
stopIsWatchingWatcher = watch(isWatchingResource, (v) => (isWatchingHistory.value = v), {
immediate: true,
});
startWatchingResource();
}
};
if (configStore.isLoaded) {
decide();
} else {
const stop = watch(
() => configStore.isLoaded,
(loaded) => {
if (loaded) {
stop();
decide();
}
},
);
}
}
async function loadHistoryById(historyId: string) {
if (!isLoadingHistory.has(historyId)) {
@@ -497,6 +580,23 @@ export const useHistoryStore = defineStore("historyStore", () => {
return contentStats;
}
// Closes SSE and stops polling so the watcher can't emit a trailing
// anonymous-cookie request that would overwrite the authenticated
// ``galaxysession`` cookie set by the login/register response.
function stopWatchingHistory() {
sseHistoryDisconnect();
if (stopHistoryPolling) {
stopHistoryPolling();
stopHistoryPolling = null;
}
if (stopIsWatchingWatcher) {
stopIsWatchingWatcher();
stopIsWatchingWatcher = null;
}
isWatchingHistory.value = false;
watchingInitialized = false;
}
return {
histories,
changingCurrentHistory,
@@ -525,7 +625,7 @@ export const useHistoryStore = defineStore("historyStore", () => {
restoreHistory,
restoreHistories,
handleTotalCountChange,
startWatchingHistory,
startWatchingHistory: startWatchingHistoryWithSSE,
stopWatchingHistory,
isWatchingHistory,
loadCurrentHistory,
@@ -0,0 +1,185 @@
import flushPromises from "flush-promises";
import { createPinia, setActivePinia } from "pinia";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { useServerMock } from "@/api/client/__mocks__";
import type { UserNotification } from "@/api/notifications";
import { emitSse, sseMockFactory, useVisibilityPatch } from "./_testing/sseStoreSupport";
import { useNotificationsStore } from "./notificationsStore";
// ``vi.mock`` is hoisted above module-level ``const`` declarations, so the
// capture-state has to be built via ``vi.hoisted`` to be visible to the factory.
const sseState = vi.hoisted(() => {
return {
onEvent: null as ((event: MessageEvent) => void) | null,
connect: vi.fn(),
disconnect: vi.fn(),
};
});
vi.mock("@/composables/useNotificationSSE", () => sseMockFactory(sseState));
// Realistic fixture: a single unread notification, as returned by
// GET /api/notifications. Shape mirrors UserNotification.
function makeNotificationFixture(overrides: Partial<UserNotification> = {}): UserNotification {
return {
id: "notif-1",
source: "galaxy_test",
category: "message",
variant: "info",
create_time: "2026-01-01T00:00:00",
update_time: "2026-01-01T00:00:00",
publication_time: "2026-01-01T00:00:00",
expiration_time: null,
seen_time: null,
deleted: false,
content: { category: "message", subject: "hello", message: "welcome" },
...overrides,
} as UserNotification;
}
const SCENARIO_NOTIFICATION = makeNotificationFixture();
const SCENARIO_STATUS_SINCE = {
total_unread_count: 1,
notifications: [SCENARIO_NOTIFICATION],
broadcasts: [],
};
const { server, http } = useServerMock();
const statusSpy = vi.fn();
function registerDefaultHandlers({ enableSseUpdates }: { enableSseUpdates: boolean }) {
server.use(
http.get("/api/configuration", ({ response }) => {
// eslint-disable-next-line @typescript-eslint/no-explicit-any
return response(200).json({
enable_notification_system: true,
enable_sse_updates: enableSseUpdates,
} as any);
}),
http.get("/api/notifications", ({ response }) => {
return response(200).json([SCENARIO_NOTIFICATION]);
}),
http.get("/api/notifications/broadcast", ({ response }) => {
return response(200).json([]);
}),
http.get("/api/notifications/status", ({ response }) => {
statusSpy();
// eslint-disable-next-line @typescript-eslint/no-explicit-any
return response(200).json(SCENARIO_STATUS_SINCE as any);
}),
);
}
/** Config load + initial fetch + store-decision watch needs a couple of ticks. */
async function primeStore(startFn: () => Promise<void> | void): Promise<void> {
// Let the config-store fetch resolve before the store's `watch` runs.
await vi.runOnlyPendingTimersAsync();
await startFn();
// Two flush cycles: one for the config watch, one for the resulting fetch.
await flushPromises();
await vi.runOnlyPendingTimersAsync();
await flushPromises();
}
describe("notificationsStore — config-driven SSE vs polling", () => {
let visibility: ReturnType<typeof useVisibilityPatch>;
beforeEach(() => {
setActivePinia(createPinia());
sseState.connect.mockClear();
sseState.disconnect.mockClear();
sseState.onEvent = null;
statusSpy.mockClear();
vi.useFakeTimers();
visibility = useVisibilityPatch();
});
afterEach(() => {
visibility.restore();
vi.useRealTimers();
});
describe("when enable_sse_updates is true (SSE scenario)", () => {
beforeEach(() => {
registerDefaultHandlers({ enableSseUpdates: true });
});
it("connects SSE and does not poll the status endpoint", async () => {
const store = useNotificationsStore();
await primeStore(() => store.startWatchingNotifications());
expect(sseState.connect).toHaveBeenCalledTimes(1);
// Advance well past the polling interval (30s) and confirm
// the status endpoint is never polled while SSE is the active channel.
vi.advanceTimersByTime(120_000);
await flushPromises();
expect(statusSpy).not.toHaveBeenCalled();
});
it("does not start polling when the tab regains visibility", async () => {
const store = useNotificationsStore();
await primeStore(() => store.startWatchingNotifications());
visibility.set("hidden");
visibility.set("visible");
await flushPromises();
vi.advanceTimersByTime(120_000);
await flushPromises();
expect(statusSpy).not.toHaveBeenCalled();
});
it("ingests notification_update events into the store state", async () => {
const store = useNotificationsStore();
await primeStore(() => store.startWatchingNotifications());
const pushed = makeNotificationFixture({
id: "notif-2",
content: { category: "message", subject: "pushed via sse", message: "hi" },
});
emitSse(sseState, "notification_update", pushed);
await flushPromises();
expect(store.notifications.map((n) => n.id)).toContain("notif-2");
expect(store.totalUnreadCount).toBeGreaterThan(0);
});
it("ingests notification_status catch-up events on reconnect", async () => {
const store = useNotificationsStore();
await primeStore(() => store.startWatchingNotifications());
emitSse(sseState, "notification_status", {
total_unread_count: 42,
notifications: [makeNotificationFixture({ id: "notif-catchup" })],
broadcasts: [],
});
await flushPromises();
expect(store.totalUnreadCount).toBe(42);
expect(store.notifications.map((n) => n.id)).toContain("notif-catchup");
});
});
describe("when enable_sse_updates is false (polling scenario)", () => {
beforeEach(() => {
registerDefaultHandlers({ enableSseUpdates: false });
});
it("does not connect SSE and polls the status endpoint on the configured interval", async () => {
const store = useNotificationsStore();
await primeStore(() => store.startWatchingNotifications());
expect(sseState.connect).not.toHaveBeenCalled();
// Advance past the short polling interval (30s) and confirm
// the status endpoint is hit by the resource watcher.
vi.advanceTimersByTime(30_000);
await flushPromises();
expect(statusSpy).toHaveBeenCalled();
});
});
});
+129 -7
View File
@@ -1,9 +1,11 @@
import { defineStore } from "pinia";
import { computed, ref } from "vue";
import { computed, ref, watch } from "vue";
import { GalaxyApi } from "@/api";
import type { NotificationChanges, UserNotification, UserNotificationsBatchUpdateRequest } from "@/api/notifications";
import { useResourceWatcher } from "@/composables/resourceWatcher";
import { useSSE } from "@/composables/useNotificationSSE";
import { useConfigStore } from "@/stores/configurationStore";
import { rethrowSimple } from "@/utils/simple-error";
import { mergeObjectListsById } from "@/utils/utils";
@@ -13,11 +15,6 @@ const ACTIVE_POLLING_INTERVAL = 30000; // 30 seconds
const INACTIVE_POLLING_INTERVAL = ACTIVE_POLLING_INTERVAL * 20; // 10 minutes
export const useNotificationsStore = defineStore("notificationsStore", () => {
const { startWatchingResource: startWatchingNotifications, stopWatchingResource: stopWatchingNotifications } =
useResourceWatcher(getNotificationStatus, {
shortPollingInterval: ACTIVE_POLLING_INTERVAL,
longPollingInterval: INACTIVE_POLLING_INTERVAL,
});
const broadcastsStore = useBroadcastsStore();
const totalUnreadCount = ref<number>(0);
@@ -28,6 +25,45 @@ export const useNotificationsStore = defineStore("notificationsStore", () => {
const unreadNotifications = computed(() => notifications.value.filter((n) => !n.seen_time));
// --- SSE setup (listen only for notification event types) ---
const NOTIFICATION_EVENT_TYPES = ["notification_update", "broadcast_update", "notification_status"] as const;
const { connect: sseConnect, disconnect: sseDisconnect } = useSSE(handleSSEEvent, NOTIFICATION_EVENT_TYPES);
let stopPolling: (() => void) | null = null;
function handleSSEEvent(event: MessageEvent) {
try {
const data = JSON.parse(event.data);
switch (event.type) {
case "notification_update":
notifications.value = mergeObjectListsById(
notifications.value,
[data as UserNotification],
"create_time",
"desc",
);
updateUnreadCount();
break;
case "broadcast_update":
broadcastsStore.updateBroadcasts([data]);
break;
case "notification_status":
// Full catch-up on reconnect (same shape as GET /api/notifications/status)
totalUnreadCount.value = data.total_unread_count;
notifications.value = mergeObjectListsById(
notifications.value,
data.notifications as UserNotification[],
"create_time",
"desc",
);
broadcastsStore.updateBroadcasts(data.broadcasts);
break;
}
lastNotificationUpdate.value = new Date();
} catch (e) {
console.error("Error handling SSE event:", e);
}
}
async function loadNotifications() {
const { data, error } = await GalaxyApi().GET("/api/notifications");
@@ -76,6 +112,75 @@ export const useNotificationsStore = defineStore("notificationsStore", () => {
}
}
// Choose between SSE and polling based on the server config flag
// `enable_sse_updates`. The `/api/events/stream` endpoint accepts
// connections regardless of the flag, so we cannot rely on EventSource
// connectivity to decide — config is the source of truth. Callers gate
// on `enable_notification_system` before reaching here, so we only have
// to pick the transport.
//
// `useResourceWatcher` is instantiated lazily because it registers a
// `visibilitychange` listener that calls `startWatchingResourceIfNeeded`
// every time the tab regains focus — in SSE mode that would re-start
// polling we explicitly don't want.
let watchingInitialized = false;
function ensureWatchingWithConfig() {
if (watchingInitialized) {
return;
}
watchingInitialized = true;
const configStore = useConfigStore();
const decide = () => {
if (configStore.config?.enable_sse_updates) {
sseConnect();
} else {
const { startWatchingResource: startPolling, stopWatchingResource } = useResourceWatcher(
getNotificationStatus,
{
shortPollingInterval: ACTIVE_POLLING_INTERVAL,
longPollingInterval: INACTIVE_POLLING_INTERVAL,
},
);
stopPolling = stopWatchingResource;
startPolling();
}
};
if (configStore.isLoaded) {
decide();
} else {
const stop = watch(
() => configStore.isLoaded,
(loaded) => {
if (loaded) {
stop();
decide();
}
},
);
}
}
async function startWatchingNotifications() {
// Always do an initial load first
if (!lastNotificationUpdate.value) {
try {
loadingNotifications.value = true;
await broadcastsStore.loadBroadcasts();
await loadNotifications();
updateUnreadCount();
lastNotificationUpdate.value = new Date();
} catch (e) {
console.error(e);
} finally {
loadingNotifications.value = false;
}
}
ensureWatchingWithConfig();
}
async function updateBatchNotification(request: UserNotificationsBatchUpdateRequest) {
const { error } = await GalaxyApi().PUT("/api/notifications", {
body: request,
@@ -88,7 +193,11 @@ export const useNotificationsStore = defineStore("notificationsStore", () => {
if (request.changes.deleted) {
notifications.value = notifications.value.filter((n) => !request.notification_ids.includes(n.id));
}
startWatchingNotifications();
// When polling is the active transport, trigger a poll to refresh
// state after a local mutation. SSE delivers the update on its own.
if (!useConfigStore().config?.enable_sse_updates) {
startWatchingNotifications();
}
}
async function updateNotification(notification: UserNotification, changes: NotificationChanges) {
@@ -99,6 +208,19 @@ export const useNotificationsStore = defineStore("notificationsStore", () => {
totalUnreadCount.value = notifications.value.filter((n) => !n.seen_time).length;
}
// Closes the SSE stream and stops the polling watcher so nothing running
// in the background can outlive a full-page navigation (login/register).
// A late-arriving response from an anonymous-cookie request would otherwise
// overwrite the just-issued authenticated ``galaxysession`` cookie.
function stopWatchingNotifications() {
sseDisconnect();
if (stopPolling) {
stopPolling();
stopPolling = null;
}
watchingInitialized = false;
}
return {
notifications,
totalUnreadCount,
+46 -34
View File
@@ -39,13 +39,29 @@ export async function watchHistory(app) {
}
export async function watchHistoryOnce(app) {
return _fetchHistoryAndChangedItems(app, { force: false });
}
/**
* Forces a fresh history + changed-items fetch, ignoring the `lastUpdateTime`
* short-circuit. Use this when an out-of-band signal (SSE `history_update`)
* already told us the history changed — we shouldn't re-gate on update_time.
*/
export async function refreshHistoryFromPush(app) {
return _fetchHistoryAndChangedItems(app, { force: true });
}
async function _fetchHistoryAndChangedItems(app, { force }) {
const historyStore = useHistoryStore();
const historyItemsStore = useHistoryItemsStore();
const datasetStore = useDatasetStore();
const collectionElementsStore = useCollectionElementsStore();
// get current history
const checkForUpdate = new Date();
// Always pass the `since` cursor so the server can short-circuit cheaply
// when nothing has changed; SSE-driven `force` only bypasses the
// client-side update_time equality gate, not the server-side cursor —
// we don't want to refetch items the client already has.
const history = await historyStore.loadCurrentHistory(lastUpdateTime);
const { lastCheckedTime } = storeToRefs(historyItemsStore);
lastCheckedTime.value = checkForUpdate;
@@ -53,39 +69,35 @@ export async function watchHistoryOnce(app) {
return;
}
// continue if the history update time has changed
if (!lastUpdateTime || lastUpdateTime < history.update_time) {
const historyId = history.id;
lastUpdateTime = history.update_time;
historyItemsStore.setLastUpdateTime();
// execute request to obtain recently changed items
const params = {
v: "dev",
limit: limit,
q: "update_time-ge",
qv: lastRequestDate.toISOString(),
};
// request detailed info only for the expanded datasets
const detailedIds = getCurrentlyExpandedHistoryContentIds();
if (detailedIds.length) {
params["details"] = detailedIds.join(",");
}
const url = `/api/histories/${historyId}/contents`;
lastRequestDate = new Date();
const payload = await urlData({ url, params });
// show warning that not all changes have been obtained
if (payload && payload.length == limit) {
console.debug(`Reached limit of monitored changes (limit=${limit}).`);
}
// pass changed items to attached stores
historyStore.setHistory(history);
datasetStore.saveDatasets(payload);
historyItemsStore.saveHistoryItems(historyId, payload);
collectionElementsStore.saveCollections(payload);
// trigger changes in legacy handler
if (app) {
app.user.loadFromApi(app.user.id || "current");
}
if (!force && lastUpdateTime && lastUpdateTime >= history.update_time) {
return;
}
const historyId = history.id;
lastUpdateTime = history.update_time;
historyItemsStore.setLastUpdateTime();
const params = {
v: "dev",
limit: limit,
q: "update_time-ge",
qv: lastRequestDate.toISOString(),
};
const detailedIds = getCurrentlyExpandedHistoryContentIds();
if (detailedIds.length) {
params["details"] = detailedIds.join(",");
}
const url = `/api/histories/${historyId}/contents`;
lastRequestDate = new Date();
const payload = await urlData({ url, params });
if (payload && payload.length == limit) {
console.debug(`Reached limit of monitored changes (limit=${limit}).`);
}
historyStore.setHistory(history);
datasetStore.saveDatasets(payload);
historyItemsStore.saveHistoryItems(historyId, payload);
collectionElementsStore.saveCollections(payload);
if (app) {
app.user.loadFromApi(app.user.id || "current");
}
}
+43 -28
View File
@@ -99,16 +99,33 @@ export function galaxyDevServerPlugin() {
const originalWrite = res.write.bind(res);
const originalEnd = res.end.bind(res);
// Buffer to collect response body
// Buffer to collect response body. We only buffer when the
// upstream response is HTML; everything else (JSON, binary,
// and crucially `text/event-stream`) must pass straight
// through, because streaming responses never call
// `res.end()` and would otherwise stall indefinitely.
const chunks = [];
let isHtml = false;
// Tri-state: null = undecided (first write hasn't landed yet),
// true = stream it through untransformed,
// false = buffer for HTML rewrite on end().
let passthrough = null;
function decidePassthrough() {
if (passthrough !== null) {
return;
}
const contentType = res.getHeader("content-type");
passthrough = !contentType || !contentType.toString().includes("text/html");
}
// Override write to collect chunks
res.write = function (chunk, encoding, callback) {
decidePassthrough();
if (passthrough) {
return originalWrite(chunk, encoding, callback);
}
if (chunk) {
chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk, encoding));
}
// Don't write yet - we'll write in end()
if (typeof encoding === "function") {
encoding(); // encoding is actually the callback
} else if (typeof callback === "function") {
@@ -119,41 +136,39 @@ export function galaxyDevServerPlugin() {
// Override end to transform and send response
res.end = function (chunk, encoding, callback) {
decidePassthrough();
if (passthrough) {
return originalEnd(chunk, encoding, callback);
}
if (chunk) {
chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk, encoding));
}
// Check content type
const contentType = res.getHeader("content-type");
isHtml = contentType && contentType.toString().includes("text/html");
// Combine all chunks
let body = Buffer.concat(chunks);
// Transform HTML responses that contain Galaxy bundles
if (isHtml) {
// Decompress gzip responses (common from remote Galaxy servers)
const contentEncoding = res.getHeader("content-encoding");
if (contentEncoding === "gzip") {
try {
body = gunzipSync(body);
} catch (e) {
// If decompression fails, continue with original body
console.warn("[galaxy-dev-server] Failed to decompress gzip response:", e.message);
}
// Decompress gzip responses (common from remote Galaxy servers)
const contentEncoding = res.getHeader("content-encoding");
if (contentEncoding === "gzip") {
try {
body = gunzipSync(body);
} catch (e) {
// If decompression fails, continue with original body
console.warn("[galaxy-dev-server] Failed to decompress gzip response:", e.message);
}
}
let htmlString = body.toString("utf-8");
if (htmlString.includes("bundled.js") || htmlString.includes("/static/dist/")) {
htmlString = transformGalaxyHtml(htmlString);
body = Buffer.from(htmlString, "utf-8");
let htmlString = body.toString("utf-8");
if (htmlString.includes("bundled.js") || htmlString.includes("/static/dist/")) {
htmlString = transformGalaxyHtml(htmlString);
body = Buffer.from(htmlString, "utf-8");
// Update content-length header
res.setHeader("content-length", body.length);
// Update content-length header
res.setHeader("content-length", body.length);
// Remove content-encoding since we've decompressed it
res.removeHeader("content-encoding");
}
// Remove content-encoding since we've decompressed it
res.removeHeader("content-encoding");
}
// Send the response
+67
View File
@@ -281,6 +281,23 @@
:Type: int
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
``kombu_sqla_transport_cleanup_interval``
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
:Description:
Time (in seconds) between attempts to delete fully-consumed rows
from the Kombu SQLAlchemy transport tables (``kombu_message``).
Only relevant when ``amqp_internal_connection`` uses a
``sqlalchemy+*`` scheme (the default with an on-disk
control.sqlite); the SQLAlchemy transport has no built-in TTL, so
without this task the tables grow unbounded. The task no-ops on
non-SQLAlchemy brokers (RabbitMQ/Redis honor per-message
expiration natively). Set to 0 to disable the cleanup task.
:Default: ``900``
:Type: int
~~~~~~~~~~~~~
``file_path``
~~~~~~~~~~~~~
@@ -3408,6 +3425,18 @@
:Type: bool
~~~~~~~~~~~~~~~~~~~~~~~~~~
``queue_metrics_interval``
~~~~~~~~~~~~~~~~~~~~~~~~~~
:Description:
How often (in seconds) the Celery beat task emits queue-depth,
SSE-connection, and WorkerProcess gauges. Only active when
statsd_host is set. Set to 0 to disable.
:Default: ``15``
:Type: int
~~~~~~~~~~~~~~~~~~~~~~
``library_import_dir``
~~~~~~~~~~~~~~~~~~~~~~
@@ -5804,6 +5833,41 @@
:Type: str
~~~~~~~~~~~~~~~~~~~~~~
``enable_sse_updates``
~~~~~~~~~~~~~~~~~~~~~~
:Description:
Enables real-time updates via Server-Sent Events (SSE), replacing
the history (3 s), entry-point (10 s) and notification (30 s)
polling loops with push events delivered over a single
``/api/events/stream`` connection per browser tab. A background
monitor watches for history changes (via PostgreSQL LISTEN/NOTIFY,
or audit-table polling as a fallback for SQLite); entry-point
changes are dispatched directly from the code paths that mutate
them; in-app notifications and broadcasts are pushed when
``enable_notification_system`` is also true. When disabled,
polling remains the source of updates for all three. See the admin
guide "Server-Sent Events for real-time updates" for the full
architecture, monitoring guidance and proxy configuration.
:Default: ``false``
:Type: bool
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
``history_audit_monitor_poll_interval``
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
:Description:
The interval in seconds between history audit table polls when
using the polling fallback (SQLite or when PostgreSQL
LISTEN/NOTIFY is unavailable). Only used when enable_sse_updates
is true. Lower values mean faster updates but more database
queries. Recommended range: 1-5 seconds.
:Default: ``2``
:Type: int
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
``enable_notification_system``
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -5815,6 +5879,9 @@
finished, etc.
The system allows notification scheduling and expiration, and
users can opt-out of specific notification categories or channels.
Delivery is push-based via Server-Sent Events when
``enable_sse_updates`` is also true, and falls back to 30-second
polling against ``/api/notifications/status`` otherwise.
Admins can schedule and broadcast notifications that will be
visible to all users, including special server-wide announcements
such as scheduled maintenance, high load warnings, and event
+1
View File
@@ -14,6 +14,7 @@ Galaxy Deployment & Administration
nginx
apache
scaling
sse_updates
cluster
jobs
job_metrics
+11
View File
@@ -233,6 +233,17 @@ previous section:
## Advanced Configuration Topics
### Server-Sent Events (real-time updates)
Galaxy can push history, entry-point and notification updates to the browser
via a long-lived Server-Sent Events stream at ``/api/events/stream``. nginx
will buffer that response by default, which breaks the stream — either rely
on the ``X-Accel-Buffering: no`` header Galaxy already sets, or add an
explicit ``location /api/events/stream`` block that disables buffering and
raises the read/send timeouts. The full configuration block, monitoring
guidance, and the architecture overview live in
[Server-Sent Events for real-time updates](sse_updates.md#configuring-nginx).
### Sending Files With Nginx
Galaxy sends files (e.g. dataset downloads) by opening the file and streaming it in chunks through the proxy server.
+8
View File
@@ -190,6 +190,14 @@ Some of these options deserve explanation:
4-12 workers should be able to handle hundreds if not thousands of requests per second.
* `extra_args`: You can specify additional arguments to pass to gunicorn here.
```{note}
If you have enabled [Server-Sent Events for real-time updates](sse_updates.md), every connected browser tab holds an
open connection to one Gunicorn worker for the lifetime of the tab. Gravity configures Gunicorn with the
``uvicorn.workers.UvicornWorker`` async worker by default, so a single worker process can hold thousands of idle SSE
connections without blocking — the limit you actually need to watch is memory headroom and the per-process
file-descriptor ulimit, not the worker count.
```
Note that the performance option values given above are just examples and should be tuned per your specific needs.
However, as given, they are a good place to start.
+280
View File
@@ -0,0 +1,280 @@
# Server-Sent Events (SSE) for real-time updates
Galaxy can push history changes, interactive-tool entry-point changes, and
in-app notifications to connected browsers via [Server-Sent Events][sse-mdn]
instead of polling. This replaces the legacy 3-second history poll and
10-second entry-point poll with a single long-lived HTTP connection per
browser tab, dramatically reducing API load on busy servers and giving users
sub-second update latency.
This document describes the architecture, the configuration knobs, the
metrics admins should watch, and how to configure NGINX so the long-lived
event connection is not buffered or prematurely terminated.
[sse-mdn]: https://developer.mozilla.org/en-US/docs/Web/API/Server-sent_events
## How it works
A high-level view of the moving parts on the server:
```
┌────────────────┐ LISTEN/NOTIFY ┌──────────────────────┐
│ Postgres │ ─────────────────► │ HistoryAuditMonitor │
│ history_audit │ (or audit poll │ (one elected process) │
│ │ on SQLite) └──────────┬───────────┘
└────────────────┘ │ Kombu
▼
┌─────────────────────────────┐
│ galaxy_queue_worker │
│ control task fan-out: │
│ • history_update │
│ • entry_point_update │
│ • notification_update │
│ • broadcast_update │
└──────────────┬──────────────┘
│ in-process
▼
┌─────────────────────────────────────┐
│ Each gunicorn worker: │
│ SSEConnectionManager → asyncio │
│ queue per connected browser tab │
└──────────────┬──────────────────────┘
│ HTTP chunked
▼
┌─────────────────────────────────────┐
│ Browser EventSource │
│ (single stream, multiplexed events)│
└─────────────────────────────────────┘
```
Concretely:
1. **One stream per browser, many event types.** The browser opens a single
`EventSource` against `/api/events/stream`. The same connection carries
`history_update`, `entry_point_update`, `notification_update`,
`broadcast_update`, and `notification_status` events.
2. **Per-process registries.** Each Gunicorn worker keeps an
`SSEConnectionManager` that holds an `asyncio.Queue` per connected tab,
indexed by user id (and Galaxy session id, so anonymous users still
receive their own history's updates).
3. **Producers.**
- History updates come from a `HistoryAuditMonitor` that watches
`history_audit` via PostgreSQL `LISTEN/NOTIFY` (instant) or by polling
the audit table on SQLite. Only one process in the cluster is the
producer, picked by `DatabaseHeartbeat` leader election. If a
standalone `galaxy-sse-monitor` process is running it always wins;
otherwise one webapp picks it up.
- Entry-point updates are dispatched directly from the code paths that
mutate interactive-tool entry points — there is no separate watcher.
- Notifications dispatch SSE events from `NotificationManager` whenever
a notification or broadcast is created.
4. **Cross-process fan-out.** Producers don't know which worker holds the
recipient's connection, so all events go through a Kombu control task
broadcast on the internal AMQP bus. Every worker receives every event
and locally drops the ones for users it doesn't currently hold a
connection for.
5. **Reconnect catch-up.** When a browser reconnects after a network blip,
it sends `Last-Event-ID`. The server replays an aggregated
`notification_status` covering everything since that timestamp. History
updates come with an `update_time` cursor in the payload, so the client
can request the delta itself.
### Standalone monitor (recommended for production)
The `galaxy-sse-monitor` console script (installed by the `galaxy-app`
package) runs the `HistoryAuditMonitor` outside the webapp processes. This
is the recommended layout for production because:
- The webapp processes never compete with each other for the audit-monitor
role on cold starts.
- Restarting the webapp tier doesn't briefly stall history updates while
another worker is elected.
- The monitor needs only DB + AMQP access, so it can be sized
independently and runs with a much smaller resident-set than a webapp.
A typical Gravity supervisor entry looks like the existing
`galaxy-celery-worker` block — point at the same `galaxy.yml` and run
`galaxy-sse-monitor` on its own. With the daemon present, the
`HistoryAuditMonitor` registered on the webapp side stays idle (the
heartbeat election picks the daemon) but is still wired up so it can take
over if the daemon goes away.
If `enable_sse_updates` is `false`, `galaxy-sse-monitor` will start, log a
warning, and idle — it does no work and produces no events.
## Configuration
There is a single admin-facing flag for SSE-driven updates:
```yaml
galaxy:
enable_sse_updates: true
```
This controls **all three** SSE-driven paths (history, entry-point,
notifications). When `false`:
- `HistoryAuditMonitor` is not registered, so the cluster does no
`LISTEN/NOTIFY` or audit-table polling for history changes.
- The browser falls back to its existing 3-second history poll and
10-second entry-point poll.
- Notifications fall back to the existing 30-second polling against
`/api/notifications/status`.
`enable_notification_system` is independent: it gates whether the
notification system is available at all (notification creation, delivery,
preferences, broadcasts). With `enable_notification_system: true`:
- `enable_sse_updates: true` → notifications arrive via SSE.
- `enable_sse_updates: false` → notifications are polled.
With `enable_notification_system: false` the entire notification system is
off — there is nothing to push or poll.
The polling-fallback knob for the history audit monitor stays available:
```yaml
galaxy:
history_audit_monitor_poll_interval: 2 # seconds, SQLite / no-LISTEN only
```
This only matters when running on SQLite or in setups where PostgreSQL
`LISTEN/NOTIFY` is unavailable.
## What to monitor
When statsd is configured (via `statsd_host` and friends), the SSE
plumbing emits the following metrics. Capture these on the same dashboard
you use for Gunicorn worker health:
| Metric | Type | Source | Meaning |
| ------------------------------------------ | ------- | --------------------- | ------------------------------------------------------------ |
| `galaxy.sse.connections.dropped` | counter | `SSEConnectionManager` | A per-connection asyncio queue filled up; an event was lost. |
| `galaxy.sse.dispatch.count` (tag: `task`) | counter | `SSEEventDispatcher` | Control-task fan-outs by event kind. |
| `galaxy.sse.dispatch.latency_ms` (tag: `task`) | timing | `SSEEventDispatcher` | Wall time spent enqueueing the control task. |
| `galaxy.sse.dispatch.skipped_no_qw` | counter | `SSEEventDispatcher` | Producer tried to dispatch with no queue worker bound — events would have been dropped. |
You can also expose a "currently connected SSE clients" gauge if you wire
one up: each `SSEConnectionManager` instance publishes
`total_broadcast_connections` (all connections, including anonymous) and
`total_per_user_connections` (connections bound to a specific user). These
are per-worker numbers; sum across workers for a cluster total.
Alerting recommendations:
- **`galaxy.sse.connections.dropped` > 0 sustained** indicates a slow or
stuck client whose queue filled up. Occasional drops on a network blip
are normal; a steady rate is a bug or a misconfigured proxy holding
events back too long.
- **`galaxy.sse.dispatch.skipped_no_qw` > 0** means events are being lost
because the producer process can't reach the AMQP bus. Check the
`amqp_internal_connection` config and the AMQP broker health.
- **`galaxy.sse.dispatch.latency_ms` p95 climbing** points at an AMQP
bottleneck (broker load, network) — events will land late.
In addition, watch the standard worker metrics. SSE connections are
long-lived (tens of minutes is common), but Galaxy runs Gunicorn with
`uvicorn.workers.UvicornWorker` (configured by Gravity by default), so
each worker process is async and can hold thousands of idle SSE
connections without blocking other requests. The practical concern on
busy servers is therefore memory and file-descriptor headroom, not
worker exhaustion: budget a few KB per connection plus one fd per
connection per worker, and raise `ulimit -n` accordingly.
## Configuring NGINX
The SSE endpoint is served at `/api/events/stream`. It is a normal
HTTP/1.1 chunked response, so it works through NGINX without any special
modules — but you must turn off response buffering and raise the
read/send timeouts, otherwise events will arrive in batched bursts (or
not at all until the connection times out).
Galaxy already sets `X-Accel-Buffering: no` on the response, which
disables NGINX's response buffering for that one endpoint without
affecting buffering on the rest of Galaxy. That alone is enough on most
setups. The block below adds the read/send timeouts and HTTP/1.1
upgrade-friendly headers explicitly so the connection survives long
idle periods between events:
```nginx
# Long-lived Server-Sent Events stream.
# Galaxy sends ``X-Accel-Buffering: no`` on the response, which
# disables nginx response buffering just for this endpoint.
# The keepalive comment fires every 30s so the read timeout
# only needs to be a comfortable margin above that.
location /api/events/stream {
proxy_pass http://unix:/srv/galaxy/var/gunicorn.sock;
proxy_set_header Host $http_host;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_http_version 1.1;
proxy_set_header Connection "";
# Disable buffering and gzip explicitly. ``X-Accel-Buffering``
# already does this, but pinning it here also covers setups
# where a sub-filter strips upstream headers.
proxy_buffering off;
proxy_cache off;
gzip off;
# Keepalives fire every 30s; allow generous slack on top.
proxy_read_timeout 1h;
proxy_send_timeout 1h;
}
```
Place this `location` block **above** the catch-all `location /` block in
your existing Galaxy `server {}` (see the [NGINX proxy guide](nginx.md)).
NGINX matches longest prefix first, so the order doesn't matter for
correctness, but keeping all the override blocks together at the top of
the server block makes the special-case handling easy to find.
If you serve Galaxy at a URL prefix (`/galaxy`), prefix the location too:
```nginx
location /galaxy/api/events/stream {
proxy_pass http://unix:/srv/galaxy/var/gunicorn.sock:/galaxy;
# ...same body as above
}
```
### Other proxies
If you front Galaxy with something other than NGINX, the same rules
apply: disable response buffering for `/api/events/stream`, allow the
connection to stay open for at least the `keepalive` interval (30 s by
default) plus a healthy margin, and pass the request through HTTP/1.1
without forcing `Connection: close`.
- **Apache `mod_proxy_http`**: add `ProxyPass` with
`flushpackets=on flushwait=5` and bump `ProxyTimeout` to at least a
few minutes. Avoid `mod_deflate` on this endpoint.
- **HAProxy**: the connection is a plain HTTP/1.1 chunked response and
needs no special handling beyond `timeout server` and
`timeout tunnel` raised above 30 s.
- **Cloudflare and other CDNs**: many CDNs buffer HTTP/1.1 chunked
responses by default. Either bypass the CDN for `/api/events/stream`
or follow the CDN's documented pattern for SSE streaming.
## Verifying the deployment
After enabling `enable_sse_updates`, three quick checks confirm the
stream is healthy end-to-end:
1. From the browser DevTools Network tab, open Galaxy and look for a
`GET /api/events/stream` request that stays in the **pending** state
with a constantly incrementing transferred-bytes count. The response
`Content-Type` is `text/event-stream`.
2. Trigger a history change (run a tool, rename a dataset). The Network
tab should show a `history_update` event in the EventStream view of
that connection within a second or two, and the history panel
refreshes without a polling round-trip.
3. On the server side, `galaxy.sse.dispatch.count` should be ticking up
for each event kind your users exercise. If you wired up the
connection gauges, they should reflect roughly one connection per
open browser tab.
If the connection opens but no events arrive, the most common causes
are: a proxy buffering responses (revisit the NGINX section), or a
producer that can't reach AMQP (see `galaxy.sse.dispatch.skipped_no_qw`).
+109 -23
View File
@@ -65,6 +65,7 @@ from galaxy.managers.file_source_instances import (
from galaxy.managers.folders import FolderManager
from galaxy.managers.hdas import HDAManager
from galaxy.managers.histories import HistoryManager
from galaxy.managers.history_audit_monitor import HistoryAuditMonitor
from galaxy.managers.interactivetool import InteractiveToolManager
from galaxy.managers.jobs import (
JobManager as JobQueryManager,
@@ -77,6 +78,8 @@ from galaxy.managers.notification import NotificationManager
from galaxy.managers.object_store_instances import UserObjectStoreResolverImpl
from galaxy.managers.roles import RoleManager
from galaxy.managers.session import GalaxySessionManager
from galaxy.managers.sse import SSEConnectionManager
from galaxy.managers.sse_dispatch import SSEEventDispatcher
from galaxy.managers.tasks import (
AsyncTasksManager,
CeleryAsyncTasksManager,
@@ -150,8 +153,12 @@ from galaxy.structured_app import (
)
from galaxy.tool_shed.cache import ToolShedRepositoryCache
from galaxy.tool_shed.galaxy_install.client import InstallationTarget
from galaxy.tool_shed.galaxy_install.installed_repository_manager import InstalledRepositoryManager
from galaxy.tool_shed.galaxy_install.update_repository_manager import UpdateRepositoryManager
from galaxy.tool_shed.galaxy_install.installed_repository_manager import (
InstalledRepositoryManager,
)
from galaxy.tool_shed.galaxy_install.update_repository_manager import (
UpdateRepositoryManager,
)
from galaxy.tool_util.data import ToolDataTableManager as BaseToolDataTableManager
from galaxy.tool_util.deps import containers
from galaxy.tool_util.deps.dependencies import AppInfo
@@ -250,7 +257,11 @@ class SentryClientMixin:
# "cannot find 'file_name' while searching for 'species_chromosomes.file_name'"]
# If we don't do this issues are never properly grouped since by default the calling stack is inspected,
# and that is always unique in cheetah as it is dynamically generated.
event["fingerprint"] = [str(exc_value), str(exc_value.tool_version), str(exc_value.__cause__)]
event["fingerprint"] = [
str(exc_value),
str(exc_value.tool_version),
str(exc_value.__cause__),
]
event.setdefault("tags", {}).update(
{
"tool_is_latest": exc_value.is_latest,
@@ -307,7 +318,10 @@ class MinimalGalaxyApplication(BasicSharedApp, HaltableContainer, SentryClientMi
config_file = kwargs.get("global_conf", {}).get("__file__", None)
if config_file:
log.debug('Using "galaxy.ini" config file: %s', config_file)
self._configure_models(check_migrate_databases=self.config.check_migrate_databases, config_file=config_file)
self._configure_models(
check_migrate_databases=self.config.check_migrate_databases,
config_file=config_file,
)
# Security helper
self._configure_security()
self._register_singleton(IdEncodingHelper, self.security)
@@ -411,7 +425,11 @@ class MinimalGalaxyApplication(BasicSharedApp, HaltableContainer, SentryClientMi
index_help = getattr(self.config, "index_tool_help", True)
self.toolbox_search = self._register_singleton(
ToolBoxSearch,
ToolBoxSearch(self.toolbox, index_dir=self.config.tool_search_index_dir, index_help=index_help),
ToolBoxSearch(
self.toolbox,
index_dir=self.config.tool_search_index_dir,
index_help=index_help,
),
)
@property
@@ -496,8 +514,9 @@ class MinimalGalaxyApplication(BasicSharedApp, HaltableContainer, SentryClientMi
templates = ConfiguredObjectStoreTemplates.from_app_config(self.config, vault_configured=vault_configured)
self.object_store_templates = self._register_singleton(ConfiguredObjectStoreTemplates, templates)
user_object_store_resolver = self._register_abstract_singleton(
UserObjectStoreResolver, UserObjectStoreResolverImpl # type: ignore[type-abstract]
) # Ignored because of https://github.com/python/mypy/issues/4717
UserObjectStoreResolver, # type: ignore[type-abstract] # https://github.com/python/mypy/issues/4717
UserObjectStoreResolverImpl,
)
kwds["user_object_store_resolver"] = user_object_store_resolver
self.object_store = build_object_store_from_config(self.config, **kwds)
@@ -600,7 +619,13 @@ class GalaxyManagerApplication(MinimalManagerApp, MinimalGalaxyApplication):
model: GalaxyModelMapping
def __init__(self, configure_logging=True, use_converters=True, use_display_applications=True, **kwargs):
def __init__(
self,
configure_logging=True,
use_converters=True,
use_display_applications=True,
**kwargs,
):
super().__init__(**kwargs)
self._register_singleton(MinimalManagerApp, self) # type: ignore[type-abstract]
self.execution_timer_factory = self._register_singleton(
@@ -617,7 +642,8 @@ class GalaxyManagerApplication(MinimalManagerApp, MinimalGalaxyApplication):
# Initialize job metrics manager, needs to be in place before
# config so per-destination modifications can be made.
self.job_metrics = self._register_singleton(
JobMetrics, JobMetrics(self.config.job_metrics_config_file, self.config.job_metrics, app=self)
JobMetrics,
JobMetrics(self.config.job_metrics_config_file, self.config.job_metrics, app=self),
)
# Initialize the job management configuration
self.job_config = self._register_singleton(jobs.JobConfiguration)
@@ -655,11 +681,30 @@ class GalaxyManagerApplication(MinimalManagerApp, MinimalGalaxyApplication):
self.role_manager = self._register_singleton(RoleManager)
self.job_manager = self._register_singleton(JobManager)
# AMQP connection + a publisher-ready queue worker. Celery worker processes
# inherit this via GalaxyManagerApplication so they can fan out SSE events
# to web workers (no consumer thread is started here — see bind_publisher).
self.amqp_internal_connection_obj = galaxy.queues.connection_from_config(self.config)
if self.amqp_internal_connection_obj is not None:
self.queue_worker = self._register_singleton(GalaxyQueueWorker, GalaxyQueueWorker(self))
self.queue_worker.bind_publisher()
# SSE dispatcher must be registered before NotificationManager so Lagom
# can auto-inject the Optional[SSEEventDispatcher] constructor arg.
self._register_singleton(
SSEEventDispatcher,
SSEEventDispatcher(
queue_worker=getattr(self, "queue_worker", None),
application_stack=self.application_stack,
statsd_client=self.execution_timer_factory.galaxy_statsd_client,
),
)
self.notification_manager = self._register_singleton(NotificationManager)
self.interactivetool_manager = InteractiveToolManager(self)
self.task_manager = self._register_abstract_singleton(
AsyncTasksManager, CeleryAsyncTasksManager # type: ignore[type-abstract] # https://github.com/python/mypy/issues/4717
AsyncTasksManager, # type: ignore[type-abstract] # https://github.com/python/mypy/issues/4717
CeleryAsyncTasksManager,
)
# ConfiguredFileSources
@@ -671,10 +716,12 @@ class GalaxyManagerApplication(MinimalManagerApp, MinimalGalaxyApplication):
self._register_singleton(FileSourcePluginLoader, file_source_plugin_loader)
self.file_source_templates = self._register_singleton(ConfiguredFileSourceTemplates, templates)
self._register_singleton(
UserDefinedFileSourcesConfig, UserDefinedFileSourcesConfig.from_app_config(self.config)
UserDefinedFileSourcesConfig,
UserDefinedFileSourcesConfig.from_app_config(self.config),
)
user_defined_file_sources = self._register_abstract_singleton(
UserDefinedFileSources, UserDefinedFileSourcesImpl # type: ignore[type-abstract] # https://github.com/python/mypy/issues/4717
UserDefinedFileSources, # type: ignore[type-abstract] # https://github.com/python/mypy/issues/4717
UserDefinedFileSourcesImpl,
)
configured_file_source_conf: ConfiguredFileSourcesConf = ConfiguredFileSourcesConf.from_app_config(self.config)
file_sources = ConfiguredFileSources(
@@ -690,7 +737,8 @@ class GalaxyManagerApplication(MinimalManagerApp, MinimalGalaxyApplication):
# Load security policy.
self.security_agent = self.model.security_agent
self.host_security_agent = galaxy.model.security.HostAgent(
self.security_agent.sa_session, permitted_actions=self.security_agent.permitted_actions
self.security_agent.sa_session,
permitted_actions=self.security_agent.permitted_actions,
)
# We need the datatype registry for running certain tasks that modify HDAs, and to build the registry we need
@@ -791,6 +839,7 @@ class UniverseApplication(StructuredApp, GalaxyManagerApplication, InstallationT
("queue worker", self._shutdown_queue_worker),
("file watcher", self._shutdown_watcher),
("database heartbeat", self._shutdown_database_heartbeat),
("history audit monitor", self._shutdown_history_audit_monitor),
("workflow scheduler", self._shutdown_scheduling_manager),
("object store", self._shutdown_object_store),
("job manager", self._shutdown_job_manager),
@@ -805,21 +854,33 @@ class UniverseApplication(StructuredApp, GalaxyManagerApplication, InstallationT
# A lot of postfork initialization depends on the server name, ensure it is set immediately after forking before other postfork functions
self.application_stack.register_postfork_function(self.application_stack.set_postfork_server_name, self)
self.config.reload_sanitize_allowlist(explicit="sanitize_allowlist_file" in kwargs)
self.amqp_internal_connection_obj = galaxy.queues.connection_from_config(self.config)
# queue_worker *can* be initialized with a queue, but here we don't
# want to and we'll allow postfork to bind and start it.
self.queue_worker = self._register_singleton(GalaxyQueueWorker, GalaxyQueueWorker(self))
# amqp_internal_connection_obj and queue_worker are built in GalaxyManagerApplication
# (so Celery workers also get a publisher); here we only register the consumer path,
# which is started later via the application_stack postfork hook.
# SSE connection manager for real-time notification push.
# Consumed via ``depends(SSEConnectionManager)`` / ``app[SSEConnectionManager]``,
# so no module-level attribute is needed — keep the container wiring only.
self._register_singleton(
SSEConnectionManager,
SSEConnectionManager(
statsd_client=self.execution_timer_factory.galaxy_statsd_client,
),
)
# AI agent registry and service
agent_registry = build_agent_registry(self.config)
self._register_singleton(AgentRegistry, agent_registry)
self._register_singleton(AgentService, AgentService(self.config, JobQueryManager(self), agent_registry))
self._register_singleton(
AgentService,
AgentService(self.config, JobQueryManager(self), agent_registry),
)
self.dependency_resolvers_view = self._register_singleton(
DependencyResolversView, DependencyResolversView(self)
)
self.test_data_resolver = self._register_singleton(
TestDataResolver, TestDataResolver(file_dirs=self.config.tool_test_data_directories)
TestDataResolver,
TestDataResolver(file_dirs=self.config.tool_test_data_directories),
)
self.api_keys_manager = self._register_singleton(ApiKeyManager)
@@ -876,7 +937,9 @@ class UniverseApplication(StructuredApp, GalaxyManagerApplication, InstallationT
# Start the heartbeat process if configured and available
if self.config.use_heartbeat:
self.heartbeat = heartbeat.Heartbeat(
self.config, period=self.config.heartbeat_interval, fname=self.config.heartbeat_log
self.config,
period=self.config.heartbeat_interval,
fname=self.config.heartbeat_log,
)
self.heartbeat.daemon = True
self.application_stack.register_postfork_function(self.heartbeat.start)
@@ -886,7 +949,9 @@ class UniverseApplication(StructuredApp, GalaxyManagerApplication, InstallationT
from galaxy.authnz import managers
self.authnz_manager = managers.AuthnzManager(
self, self.config.oidc_config_file, self.config.oidc_backends_config_file
self,
self.config.oidc_config_file,
self.config.oidc_backends_config_file,
)
# If there is only a single external authentication provider in use
@@ -923,7 +988,12 @@ class UniverseApplication(StructuredApp, GalaxyManagerApplication, InstallationT
self.workflow_completion_hook_registry,
)
self.application_stack.register_postfork_function(self.workflow_completion_monitor.start)
self.haltables.append(("WorkflowCompletionMonitor", self.workflow_completion_monitor.shutdown_monitor))
self.haltables.append(
(
"WorkflowCompletionMonitor",
self.workflow_completion_monitor.shutdown_monitor,
)
)
# Start the job manager
self.application_stack.register_postfork_function(self.job_manager.start)
@@ -942,12 +1012,23 @@ class UniverseApplication(StructuredApp, GalaxyManagerApplication, InstallationT
self.database_heartbeat.add_change_callback(self.watchers.change_state)
self.application_stack.register_postfork_function(self.database_heartbeat.start)
# History audit monitor for SSE-based history updates. The monitor only
# runs on the single process elected via DatabaseHeartbeat's
# is_history_audit_monitor role — a standalone ``galaxy-sse-monitor``
# daemon wins that election when present, otherwise one webapp picks it
# up. start/stop are driven by heartbeat role transitions rather than
# postfork, so the monitor cleanly migrates when the leader dies.
if self.config.enable_sse_updates:
monitor = self._register_singleton(HistoryAuditMonitor)
self.database_heartbeat.add_audit_monitor_change_callback(monitor.on_role_change)
# Start web stack message handling
self.application_stack.register_postfork_function(self.application_stack.start)
self.application_stack.register_postfork_function(self.queue_worker.bind_and_start)
# Reload toolbox to pick up changes to toolbox made after master was ready
self.application_stack.register_postfork_function(
lambda: reload_toolbox(self, save_integrated_tool_panel=False), post_fork_only=True
lambda: reload_toolbox(self, save_integrated_tool_panel=False),
post_fork_only=True,
)
# Delay toolbox index until after startup
self.application_stack.register_postfork_function(
@@ -975,6 +1056,11 @@ class UniverseApplication(StructuredApp, GalaxyManagerApplication, InstallationT
def _shutdown_database_heartbeat(self):
self.database_heartbeat.shutdown()
def _shutdown_history_audit_monitor(self):
if not self.config.enable_sse_updates:
return
self[HistoryAuditMonitor].shutdown()
def _shutdown_scheduling_manager(self):
self.workflow_scheduling_manager.shutdown()
+2 -1
View File
@@ -119,6 +119,7 @@ class MockApp(di.Container, GalaxyDataTestApp):
history_manager: HistoryManager
job_metrics: JobMetrics
vault: Optional[Vault] = None
execution_timer_factory: Any
stop: bool
is_webapp: bool = True
@@ -159,7 +160,7 @@ class MockApp(di.Container, GalaxyDataTestApp):
self.application_stack = ApplicationStack()
self.auth_manager = AuthManager(self.config)
self.user_manager = UserManager(cast(BasicSharedApp, self))
self.execution_timer_factory = Bunch(get_timer=StructuredExecutionTimer)
self.execution_timer_factory = Bunch(get_timer=StructuredExecutionTimer, galaxy_statsd_client=None)
self.interactivetool_manager = Bunch(create_interactivetool=lambda *args, **kwargs: None)
self.is_job_handler = False
self.biotools_metadata_source = None
+1 -2
View File
@@ -798,7 +798,6 @@ def _send_oidc_profile_update_notification(trans, user, updates: list[str]) -> N
NotificationVariant,
PersonalNotificationCategory,
)
from galaxy.webapps.galaxy.services.notifications import NotificationService
labels: dict[str, str] = {
"email": "email address",
@@ -819,7 +818,7 @@ def _send_oidc_profile_update_notification(trans, user, updates: list[str]) -> N
),
galaxy_url=None,
)
NotificationService(trans.app.notification_manager).send_notification_internal(request, force_sync=True)
trans.app.notification_manager.send_notification_internal(request, force_sync=True)
except Exception as exc:
log.warning("OIDC profile update notification failed for user %s: %s", user.id, exc)
+4
View File
@@ -251,6 +251,10 @@ def setup_periodic_tasks(config, celery_app):
beat_schedule: dict[str, dict[str, Any]] = {}
schedule_task("prune_history_audit_table", config.history_audit_table_prune_interval)
schedule_task("cleanup_short_term_storage", config.short_term_storage_cleanup_interval)
schedule_task("prune_kombu_sqla_transport", config.kombu_sqla_transport_cleanup_interval)
if config.statsd_host:
schedule_task("emit_queue_metrics_task", config.queue_metrics_interval)
if config.enable_notification_system:
schedule_task("cleanup_expired_notifications", config.expired_notifications_cleanup_interval)
+25
View File
@@ -0,0 +1,25 @@
from celery.result import AsyncResult
from galaxy.schema.schema import AsyncTaskResultSummary
def async_task_summary(async_result: AsyncResult) -> AsyncTaskResultSummary:
name = None
try:
name = async_result.name
except AttributeError:
# if backend is disabled, we won't have this
pass
queue = None
try:
queue = async_result.queue
except AttributeError:
# if backend is disabled, we won't have this
pass
return AsyncTaskResultSummary(
id=str(async_result.id),
ignored=async_result.ignored,
name=name,
queue=queue,
)
+64 -1
View File
@@ -9,13 +9,17 @@ from typing import (
Any,
Optional,
)
from urllib.parse import urlparse
from lagom.exceptions import UnresolvableType
from sqlalchemy import (
and_,
create_engine,
delete,
exists,
false,
select,
text,
update,
)
@@ -40,6 +44,8 @@ from galaxy.managers.lddas import LDDAManager
from galaxy.managers.markdown_util import generate_branded_pdf
from galaxy.managers.model_stores import ModelStoreManager
from galaxy.managers.notification import NotificationManager
from galaxy.managers.queue_metrics import emit_queue_metrics
from galaxy.managers.sse import SSEConnectionManager
from galaxy.managers.tool_data import ToolDataImportManager
from galaxy.managers.workflow_completion import WorkflowCompletionManager
from galaxy.metadata.set_metadata import set_metadata_portable
@@ -76,7 +82,10 @@ from galaxy.security.vault import (
Vault,
)
from galaxy.short_term_storage import ShortTermStorageMonitor
from galaxy.structured_app import MinimalManagerApp
from galaxy.structured_app import (
MinimalManagerApp,
StructuredApp,
)
from galaxy.tools import create_tool_from_representation
from galaxy.tools.data_fetch import do_fetch
from galaxy.util import galaxy_directory
@@ -591,6 +600,38 @@ def prune_history_audit_table(sa_session: galaxy_scoped_session):
model.HistoryAudit.prune(sa_session)
@galaxy_task(action="cleaning up Kombu SQLAlchemy transport")
def prune_kombu_sqla_transport(config: GalaxyAppConfiguration):
"""Delete fully-consumed rows from the Kombu SQLAlchemy transport tables.
Kombu's SQLAlchemy transport marks consumed messages with ``visible=0`` but
never deletes them — without this task the control-queue tables grow
without bound on the default on-disk sqlite broker. On AMQP / Redis the
broker has native TTL, so this task is a no-op.
"""
broker_url = config.amqp_internal_connection
if not broker_url:
log.debug("kombu cleanup: no broker URL configured, skipping")
return
scheme = urlparse(broker_url).scheme
if not scheme.startswith("sqlalchemy"):
log.debug("kombu cleanup: broker scheme %s is not sqlalchemy, skipping", scheme)
return
# Kombu's SQLA transport URL is ``sqlalchemy+<dialect>://...``. Strip the
# ``sqlalchemy+`` prefix to get an engine URL we can hand to SQLAlchemy.
sa_url = broker_url[len("sqlalchemy+") :] if broker_url.startswith("sqlalchemy+") else broker_url
engine = create_engine(sa_url)
try:
with engine.begin() as conn:
result = conn.execute(text("DELETE FROM kombu_message WHERE visible = 0"))
log.info("kombu cleanup: deleted %s consumed messages", result.rowcount)
except Exception:
log.exception("kombu cleanup: failed to prune kombu_message")
finally:
engine.dispose()
@galaxy_task(action="clean up short term storage")
def cleanup_short_term_storage(storage_monitor: ShortTermStorageMonitor):
"""Cleanup short term storage."""
@@ -628,6 +669,28 @@ def dispatch_pending_notifications(notification_manager: NotificationManager):
log.info(f"Successfully dispatched {count} notifications.")
@galaxy_task(action="emit queue and SSE observability metrics")
def emit_queue_metrics_task(app: StructuredApp):
"""Sample control-queue depth, SSE connection count, and worker rows → statsd.
Resolves the narrow collaborators ``emit_queue_metrics`` needs from the app
container and passes them in — keeps the emitter module free of
``StructuredApp`` service-locator lookups.
"""
try:
sse_manager: Optional[SSEConnectionManager] = app[SSEConnectionManager]
except UnresolvableType:
sse_manager = None
emit_queue_metrics(
statsd_client=app.execution_timer_factory.galaxy_statsd_client,
connection=app.amqp_internal_connection_obj,
application_stack=app.application_stack,
model=app.model,
sse_manager=sse_manager,
)
@galaxy_task(action="clean up job working directories")
def cleanup_jwds(sa_session: galaxy_scoped_session, object_store: BaseObjectStore, config: GalaxyAppConfiguration):
"""Cleanup job working directories for failed jobs that are older than X days"""
@@ -512,6 +512,16 @@ galaxy:
# history_audit database table. Set to 0 to disable pruning.
#history_audit_table_prune_interval: 3600
# Time (in seconds) between attempts to delete fully-consumed rows
# from the Kombu SQLAlchemy transport tables (``kombu_message``). Only
# relevant when ``amqp_internal_connection`` uses a ``sqlalchemy+*``
# scheme (the default with an on-disk control.sqlite); the SQLAlchemy
# transport has no built-in TTL, so without this task the tables grow
# unbounded. The task no-ops on non-SQLAlchemy brokers (RabbitMQ/Redis
# honor per-message expiration natively). Set to 0 to disable the
# cleanup task.
#kombu_sqla_transport_cleanup_interval: 900
# Where dataset files are stored. It must be accessible at the same
# path on any cluster nodes that will run Galaxy jobs, unless using
# Pulsar. The default value has been changed from 'files' to 'objects'
@@ -1962,6 +1972,11 @@ galaxy:
# really. Do not set this in production environments.
#statsd_mock_calls: false
# How often (in seconds) the Celery beat task emits queue-depth,
# SSE-connection, and WorkerProcess gauges. Only active when
# statsd_host is set. Set to 0 to disable.
#queue_metrics_interval: 15
# Add an option to the library upload form which allows administrators
# to upload a directory of files.
#library_import_dir: null
@@ -3129,12 +3144,36 @@ galaxy:
# Message to display on the export citations tool page
#citations_export_message_html: When writing up your analysis, remember to include all references that should be cited in order to completely describe your work. Also, please remember to <a href="https://galaxyproject.org/citing-galaxy">cite Galaxy</a>.
# Enables real-time updates via Server-Sent Events (SSE), replacing
# the history (3 s), entry-point (10 s) and notification (30 s)
# polling loops with push events delivered over a single
# ``/api/events/stream`` connection per browser tab. A background
# monitor watches for history changes (via PostgreSQL LISTEN/NOTIFY,
# or audit-table polling as a fallback for SQLite); entry-point
# changes are dispatched directly from the code paths that mutate
# them; in-app notifications and broadcasts are pushed when
# ``enable_notification_system`` is also true. When disabled, polling
# remains the source of updates for all three. See the admin guide
# "Server-Sent Events for real-time updates" for the full
# architecture, monitoring guidance and proxy configuration.
#enable_sse_updates: false
# The interval in seconds between history audit table polls when using
# the polling fallback (SQLite or when PostgreSQL LISTEN/NOTIFY is
# unavailable). Only used when enable_sse_updates is true. Lower
# values mean faster updates but more database queries. Recommended
# range: 1-5 seconds.
#history_audit_monitor_poll_interval: 2
# Enables the Notification System integrated in Galaxy.
# Users can receive automatic notifications when a certain resource is
# shared with them or when some long running operations have finished,
# etc.
# The system allows notification scheduling and expiration, and users
# can opt-out of specific notification categories or channels.
# Delivery is push-based via Server-Sent Events when
# ``enable_sse_updates`` is also true, and falls back to 30-second
# polling against ``/api/notifications/status`` otherwise.
# Admins can schedule and broadcast notifications that will be visible
# to all users, including special server-wide announcements such as
# scheduled maintenance, high load warnings, and event announcements,
@@ -222,6 +222,18 @@ mapping:
Time (in seconds) between attempts to remove old rows from the history_audit database table.
Set to 0 to disable pruning.
kombu_sqla_transport_cleanup_interval:
type: int
default: 900
required: false
desc: |
Time (in seconds) between attempts to delete fully-consumed rows from the Kombu SQLAlchemy
transport tables (``kombu_message``). Only relevant when ``amqp_internal_connection`` uses a
``sqlalchemy+*`` scheme (the default with an on-disk control.sqlite); the SQLAlchemy transport
has no built-in TTL, so without this task the tables grow unbounded. The task no-ops on
non-SQLAlchemy brokers (RabbitMQ/Redis honor per-message expiration natively). Set to 0 to
disable the cleanup task.
file_path:
type: str
default: objects
@@ -2512,6 +2524,14 @@ mapping:
Mock out statsd client calls - only used by testing infrastructure really.
Do not set this in production environments.
queue_metrics_interval:
type: int
default: 15
required: false
desc: |
How often (in seconds) the Celery beat task emits queue-depth, SSE-connection,
and WorkerProcess gauges. Only active when statsd_host is set. Set to 0 to disable.
library_import_dir:
type: str
required: false
@@ -4293,6 +4313,33 @@ mapping:
desc: |
Message to display on the export citations tool page
enable_sse_updates:
type: bool
default: false
required: false
desc: |
Enables real-time updates via Server-Sent Events (SSE), replacing the
history (3 s), entry-point (10 s) and notification (30 s) polling loops
with push events delivered over a single ``/api/events/stream``
connection per browser tab. A background monitor watches for history
changes (via PostgreSQL LISTEN/NOTIFY, or audit-table polling as a
fallback for SQLite); entry-point changes are dispatched directly from
the code paths that mutate them; in-app notifications and broadcasts
are pushed when ``enable_notification_system`` is also true.
When disabled, polling remains the source of updates for all three.
See the admin guide "Server-Sent Events for real-time updates" for the
full architecture, monitoring guidance and proxy configuration.
history_audit_monitor_poll_interval:
type: int
default: 2
required: false
desc: |
The interval in seconds between history audit table polls when using the polling
fallback (SQLite or when PostgreSQL LISTEN/NOTIFY is unavailable). Only used when
enable_sse_updates is true. Lower values mean faster updates but more
database queries. Recommended range: 1-5 seconds.
enable_notification_system:
type: bool
default: false
@@ -4304,6 +4351,10 @@ mapping:
The system allows notification scheduling and expiration, and users can opt-out of specific notification categories or channels.
Delivery is push-based via Server-Sent Events when ``enable_sse_updates``
is also true, and falls back to 30-second polling against
``/api/notifications/status`` otherwise.
Admins can schedule and broadcast notifications that will be visible to all users, including special server-wide announcements such as scheduled maintenance, high load warnings, and event announcements, to name a few examples.
enable_mcp_server:
@@ -18,6 +18,7 @@ tomli==2.4.1 ; python_full_version < '3.11'
types-awscrt==0.31.3
types-bleach==6.3.0.20260408
types-boto==2.49.18.20241019
types-cachetools==6.2.0.20260408
types-contextvars==2.4.7.3
types-dataclasses==0.6.6
types-docutils==0.22.3.20260408
+1
View File
@@ -229,6 +229,7 @@ class ConfigSerializer(base.ModelSerializer):
"tool_training_recommendations_link": _use_config,
"tool_training_recommendations_api_url": _use_config,
"enable_notification_system": _use_config,
"enable_sse_updates": _use_config,
"instance_resource_url": _use_config,
"instance_access_url": _use_config,
"organization_name": _use_config,
@@ -0,0 +1,318 @@
"""Monitor for history audit table changes.
Detects history changes via PostgreSQL LISTEN/NOTIFY (instant) or by polling
the history_audit table (SQLite fallback). Dispatches SSE events to connected
users via Kombu control tasks.
Only active when ``enable_sse_updates`` is True in the Galaxy config.
"""
import logging
import select
import threading
import time
from collections import (
defaultdict,
OrderedDict,
)
from collections.abc import Iterator
from datetime import (
datetime,
timedelta,
)
from typing import (
Any,
Optional,
)
from sqlalchemy import select as sa_select
from sqlalchemy.engine import Engine
from galaxy.config import GalaxyAppConfiguration
from galaxy.managers.sse_dispatch import SSEEventDispatcher
from galaxy.model import (
GalaxySessionToHistoryAssociation,
History,
HistoryAudit,
)
from galaxy.model.mapping import GalaxyModelMapping
log = logging.getLogger(__name__)
CHANNEL_NAME = "galaxy_history_update"
OWNER_CACHE_MAX = 10_000
DEBOUNCE_SECONDS = 0.2
class _PgListenAdapter:
"""Thin DBAPI-level adapter for PostgreSQL LISTEN/NOTIFY.
Hides the receiving-API differences between psycopg2 (``conn.poll()`` +
``conn.notifies`` list, driven by ``select.select``) and psycopg3
(``conn.notifies(timeout=...)`` generator). The SA URL is used to inherit
DSN / SSL / auth config, but the connection itself is opened directly with
the DBAPI driver so it stays outside the SA pool — LISTEN connections must
live for the lifetime of the monitor and never be returned to the pool.
"""
# Typed once at the class level so both driver branches can assign without
# re-annotating the same attribute.
_conn: Any
driver: str
def __init__(self, engine: Engine) -> None:
# Strip the SA ``+driver`` suffix so the raw DBAPI libraries accept the URL.
dsn = engine.url.set(drivername="postgresql").render_as_string(hide_password=False)
driver = engine.dialect.driver
if driver == "psycopg":
import psycopg # conditional: psycopg3 driver
self._conn = psycopg.connect(dsn, autocommit=True)
self.driver = "psycopg3"
else:
import psycopg2 # conditional: psycopg2 driver
self._conn = psycopg2.connect(dsn)
self._conn.autocommit = True # same effect as set_isolation_level(AUTOCOMMIT)
self.driver = "psycopg2"
def listen(self, channel: str) -> None:
with self._conn.cursor() as cursor:
cursor.execute(f"LISTEN {channel};")
def close(self) -> None:
try:
self._conn.close()
except Exception:
log.debug("Error closing LISTEN connection", exc_info=True)
def poll(self, timeout: float) -> Iterator[str]:
"""Block up to ``timeout`` seconds and yield notification payloads.
Returns an empty iterator on timeout so callers can uniformly treat
"nothing received in this tick" regardless of driver.
"""
if self.driver == "psycopg3":
# psycopg3: notifies() is a blocking generator bounded by ``timeout``.
yield from (n.payload for n in self._conn.notifies(timeout=timeout))
return
# psycopg2: block on the socket via select(), then drain notifies list.
if select.select([self._conn], [], [], timeout) == ([], [], []):
return
self._conn.poll()
while self._conn.notifies:
yield self._conn.notifies.pop(0).payload
class HistoryAuditMonitor:
"""Background thread that monitors history_audit for changes and dispatches SSE events.
On PostgreSQL: uses LISTEN/NOTIFY for instant notification.
On SQLite: polls history_audit table at a configurable interval.
"""
def __init__(
self,
config: GalaxyAppConfiguration,
model: GalaxyModelMapping,
sse_dispatcher: SSEEventDispatcher,
) -> None:
self._config = config
self._model = model
self._dispatcher = sse_dispatcher
self.poll_interval: int = config.history_audit_monitor_poll_interval
self._is_postgres: bool = "postgres" in model.engine.name
self._exit = threading.Event()
self._thread: Optional[threading.Thread] = None
self._active = False
# Bounded LRU cache: history_id -> (user_id, session_ids), refreshed on miss.
# For registered-owned histories: (user_id, ()); for anonymous histories:
# (None, (session_id, ...)) — a history can be associated with multiple
# sessions via GalaxySessionToHistoryAssociation.
self._history_owner_cache: OrderedDict[int, tuple[Optional[int], tuple[int, ...]]] = OrderedDict()
def start(self) -> None:
if self._active:
return
self._active = True
self._exit.clear() # allow restart after a previous shutdown
target = self._listen_postgres if self._is_postgres else self._poll_audit_table
self._thread = threading.Thread(
target=target,
name="history_audit_monitor",
daemon=True,
)
self._thread.start()
log.info(
"HistoryAuditMonitor started (mode=%s, interval=%ds)",
"pg_listen" if self._is_postgres else "poll",
self.poll_interval,
)
def shutdown(self) -> None:
if not self._active:
return
self._active = False
self._exit.set()
if self._thread:
self._thread.join(timeout=5)
self._thread = None
log.info("HistoryAuditMonitor stopped")
def on_role_change(self, is_leader: bool) -> None:
"""Heartbeat callback: start/stop the monitor as this process's election state changes."""
if is_leader:
self.start()
else:
self.shutdown()
# --- PostgreSQL LISTEN/NOTIFY mode ---
def _listen_postgres(self) -> None:
"""LISTEN for history update notifications.
Works against both psycopg2 and psycopg3 — whichever driver the SA
engine was built with. Falls back to the SQLite polling path if the
DBAPI driver can't be imported or the initial LISTEN fails.
"""
try:
adapter = _PgListenAdapter(self._model.engine)
adapter.listen(CHANNEL_NAME)
except Exception:
log.warning(
"Failed to establish PostgreSQL LISTEN connection, falling back to polling",
exc_info=True,
)
self._poll_audit_table()
return
log.debug("LISTEN %s established (driver=%s)", CHANNEL_NAME, adapter.driver)
pending: dict[int, float] = {} # history_id -> first_seen_time
try:
while not self._exit.is_set():
received_any = False
for payload in adapter.poll(self.poll_interval):
received_any = True
try:
history_id = int(payload)
except (ValueError, TypeError):
continue
pending.setdefault(history_id, time.monotonic())
if not received_any:
# Timeout — flush anything that's been pending since last tick
if pending:
self._dispatch_history_updates(set(pending.keys()))
pending.clear()
continue
# Debounce: dispatch events that have been pending long enough
now = time.monotonic()
ready = {hid for hid, ts in pending.items() if now - ts >= DEBOUNCE_SECONDS}
if ready:
self._dispatch_history_updates(ready)
for hid in ready:
del pending[hid]
except Exception:
log.exception("HistoryAuditMonitor LISTEN loop error")
finally:
adapter.close()
# --- SQLite polling fallback ---
def _poll_audit_table(self) -> None:
"""Poll history_audit for recent changes."""
last_check = datetime.utcnow() - timedelta(seconds=self.poll_interval)
while not self._exit.is_set():
try:
check_time = datetime.utcnow()
stmt = (
sa_select(HistoryAudit.history_id)
.where(HistoryAudit.update_time > last_check)
.group_by(HistoryAudit.history_id)
)
with self._model.new_session() as session:
changed_ids = set(session.scalars(stmt).all())
if changed_ids:
self._dispatch_history_updates(changed_ids)
last_check = check_time
except Exception:
log.exception("HistoryAuditMonitor poll error")
self._exit.wait(self.poll_interval)
# --- Common dispatch logic ---
def _dispatch_history_updates(self, history_ids: set[int]) -> None:
"""Map history_ids to user_ids / session_ids and send Kombu control task.
Raw integer history IDs are sent across the control queue; encoding is
deferred to the ``history_update`` task handler on the receiving side,
keeping this manager free of presentation concerns.
"""
# Resolve owners for unknown history_ids
unknown = history_ids - self._history_owner_cache.keys()
if unknown:
self._refresh_owner_cache(unknown)
user_updates: dict[str, list[int]] = defaultdict(list)
session_updates: dict[str, list[int]] = defaultdict(list)
for history_id in history_ids:
entry = self._history_owner_cache.get(history_id)
if entry is None:
continue
user_id, session_ids = entry
if user_id is not None:
user_updates[str(user_id)].append(history_id)
else:
for session_id in session_ids:
session_updates[str(session_id)].append(history_id)
if not user_updates and not session_updates:
return
self._dispatcher.history_update(
user_updates=dict(user_updates),
session_updates=dict(session_updates) if session_updates else None,
)
def _refresh_owner_cache(self, history_ids: set[int]) -> None:
"""Look up ownership for given history_ids and update the bounded cache.
Registered-owned histories resolve with just ``History.user_id``. For
histories where ``user_id IS NULL`` we additionally fetch associated
``galaxy_session.id`` values from ``GalaxySessionToHistoryAssociation``
so the anonymous SSE dispatch path can target the right browser.
"""
try:
with self._model.new_session() as session:
stmt = sa_select(History.id, History.user_id).where(History.id.in_(history_ids))
anon_history_ids: set[int] = set()
for row in session.execute(stmt):
hid, uid = row[0], row[1]
self._history_owner_cache[hid] = (uid, ())
self._history_owner_cache.move_to_end(hid)
if uid is None:
anon_history_ids.add(hid)
if anon_history_ids:
assoc_stmt = sa_select(
GalaxySessionToHistoryAssociation.history_id,
GalaxySessionToHistoryAssociation.session_id,
).where(GalaxySessionToHistoryAssociation.history_id.in_(anon_history_ids))
sessions_by_history: dict[int, list[int]] = defaultdict(list)
for row in session.execute(assoc_stmt):
hid, sid = row[0], row[1]
if sid is not None:
sessions_by_history[hid].append(sid)
for hid, sids in sessions_by_history.items():
self._history_owner_cache[hid] = (None, tuple(sids))
while len(self._history_owner_cache) > OWNER_CACHE_MAX:
self._history_owner_cache.popitem(last=False)
except Exception:
log.debug("Failed to refresh history owner cache", exc_info=True)
+24 -1
View File
@@ -6,6 +6,7 @@ from collections.abc import (
)
from typing import (
Any,
Optional,
TYPE_CHECKING,
Union,
)
@@ -28,6 +29,7 @@ from sqlalchemy import (
)
from galaxy import exceptions
from galaxy.managers.sse_dispatch import SSEEventDispatcher
from galaxy.model import (
InteractiveToolEntryPoint,
Job,
@@ -147,7 +149,11 @@ class InteractiveToolManager:
Manager for dealing with InteractiveTools
"""
def __init__(self, app: "MinimalManagerApp") -> None:
def __init__(
self,
app: "MinimalManagerApp",
dispatcher: Optional[SSEEventDispatcher] = None,
) -> None:
self.app = app
self.security = app.security
self.sa_session = app.model.context
@@ -157,6 +163,12 @@ class InteractiveToolManager:
app.config.interactivetoolsproxy_map or app.config.interactivetools_map,
self.encoder.encode_id,
)
# Lagom can't auto-inject ``SSEEventDispatcher`` here because the
# ``app: "MinimalManagerApp"`` hint is only a forward reference
# (TYPE_CHECKING import), so ``get_type_hints`` on this signature
# fails. Resolve through the container explicitly — ``resolve_or_none``
# returns ``None`` for mocks/test apps that never registered one.
self.dispatcher = dispatcher if dispatcher is not None else app.resolve_or_none(SSEEventDispatcher)
def create_entry_points(
self, job: Job, tool: "Tool", entry_points=Union[Iterable[dict[str, Any]], None], flush: bool = True
@@ -198,6 +210,17 @@ class InteractiveToolManager:
configured.append(ep)
if configured:
self.sa_session.commit()
# Fan out an SSE push so the user's browser can refresh the entry
# point list immediately instead of waiting for the 10 s poll.
# Anonymous jobs fall back to polling — ``push_to_user`` keys on
# user_id, and anonymous clients sit in the broadcast-only set.
if self.dispatcher is not None and job.user_id is not None:
try:
self.dispatcher.entry_point_update(user_id=job.user_id)
except Exception:
# The DB commit is authoritative; the SSE event is best
# effort. Never let a dispatch failure poison the caller.
log.exception("Failed to dispatch entry_point_update SSE event for job %s", job.id)
return dict(not_configured=not_configured, configured=configured)
def save_entry_point(self, entry_point: InteractiveToolEntryPoint) -> None:
+79 -11
View File
@@ -5,6 +5,7 @@ from typing import (
cast,
NamedTuple,
Optional,
Union,
)
from urllib.parse import urlparse
@@ -28,6 +29,7 @@ from sqlalchemy.sql import Select
from typing_extensions import Protocol
from galaxy import util
from galaxy.celery.helpers import async_task_summary
from galaxy.config import (
GalaxyAppConfiguration,
templates,
@@ -37,6 +39,7 @@ from galaxy.exceptions import (
ObjectNotFound,
)
from galaxy.managers.markdown_util import to_html
from galaxy.managers.sse_dispatch import SSEEventDispatcher
from galaxy.model import (
GroupRoleAssociation,
Notification,
@@ -49,6 +52,7 @@ from galaxy.model.scoped_session import galaxy_scoped_session
from galaxy.schema.notifications import (
AnyNotificationContent,
BroadcastNotificationCreateRequest,
BroadcastNotificationResponse,
MandatoryNotificationCategory,
MessageNotificationContent,
NewSharedItemNotificationContent,
@@ -56,14 +60,17 @@ from galaxy.schema.notifications import (
NotificationCategorySettings,
NotificationChannelSettings,
NotificationCreateData,
NotificationCreatedResponse,
NotificationCreateRequest,
NotificationRecipients,
NotificationResponse,
NotificationVariant,
PersonalNotificationCategory,
UpdateUserNotificationPreferencesRequest,
UserNotificationPreferences,
UserNotificationUpdateRequest,
)
from galaxy.schema.schema import AsyncTaskResultSummary
log = logging.getLogger(__name__)
@@ -94,9 +101,15 @@ class NotificationChannelPlugin(Protocol):
class NotificationManager:
"""Manager class to interact with the database models related with Notifications."""
def __init__(self, sa_session: galaxy_scoped_session, config: GalaxyAppConfiguration):
def __init__(
self,
sa_session: galaxy_scoped_session,
config: GalaxyAppConfiguration,
sse_dispatcher: Optional[SSEEventDispatcher] = None,
):
self.sa_session = sa_session
self.config = config
self.sse_dispatcher = sse_dispatcher
self.recipient_resolver = NotificationRecipientResolver(strategy=DefaultStrategy(sa_session))
self.user_notification_columns: list[InstrumentedAttribute] = [
Notification.id,
@@ -164,8 +177,36 @@ class NotificationManager:
notifications_sent = self._create_associations(notification, recipient_users)
self.sa_session.commit()
# Push SSE events to connected users via control queue
user_ids = [user.id for user in recipient_users]
self._notify_users_via_sse(user_ids, notification)
return notification, notifications_sent
def send_notification_internal(
self, request: NotificationCreateRequest, force_sync: bool = False
) -> Union[NotificationCreatedResponse, AsyncTaskResultSummary]:
"""Sends a notification to a list of recipients (users, groups or roles).
If `force_sync` is set to `True`, the notification recipients will be processed synchronously instead of
in a background task.
Note: This function is meant for internal use from other callers that don't need to check sender permissions.
"""
if self.can_send_notifications_async and not force_sync:
# Local import: galaxy.celery.tasks imports NotificationManager at module load,
# so importing it at module level here would be a circular dependency.
from galaxy.celery.tasks import send_notification_to_recipients_async
result = send_notification_to_recipients_async.delay(request)
return async_task_summary(result)
notification, recipient_user_count = self.send_notification_to_recipients(request)
return NotificationCreatedResponse(
total_notifications_sent=recipient_user_count,
notification=NotificationResponse.model_validate(notification),
)
def _create_associations(self, notification: Notification, users: list[User]) -> int:
success_count = 0
for user in users:
@@ -179,6 +220,26 @@ class NotificationManager:
continue
return success_count
def _notify_users_via_sse(self, user_ids: list[int], notification: Notification) -> None:
"""Broadcast a control task to all workers to push SSE events to connected users."""
if not self.sse_dispatcher or not user_ids:
return
try:
payload = NotificationResponse.model_validate(notification).model_dump_json()
self.sse_dispatcher.notify_users(user_ids, payload)
except Exception:
log.warning("Failed to send SSE notification event", exc_info=True)
def _notify_broadcast_via_sse(self, notification: Notification) -> None:
"""Broadcast a control task to all workers to push SSE broadcast events."""
if not self.sse_dispatcher:
return
try:
payload = BroadcastNotificationResponse.model_validate(notification).model_dump_json()
self.sse_dispatcher.notify_broadcast(payload)
except Exception:
log.warning("Failed to send SSE broadcast event", exc_info=True)
def dispatch_pending_notifications_via_channels(self) -> int:
"""
Dispatches all pending notifications to the users depending on the configured channels.
@@ -273,6 +334,7 @@ class NotificationManager:
notification = self._create_notification_model(request)
self.sa_session.add(notification)
self.sa_session.commit()
self._notify_broadcast_via_sse(notification)
return notification
def get_user_notification(self, user: User, notification_id: int, active_only: Optional[bool] = True):
@@ -353,7 +415,10 @@ class NotificationManager:
return result
def update_user_notifications(
self, user: User, notification_ids: set[int], request: UserNotificationUpdateRequest
self,
user: User,
notification_ids: set[int],
request: UserNotificationUpdateRequest,
) -> int:
"""Updates a batch of notifications associated with the user using the requested values."""
updated_row_count = 0
@@ -447,7 +512,8 @@ class NotificationManager:
UserNotificationAssociation.notification_id.in_(expired_notifications_stmt)
)
result = cast(
CursorResult, self.sa_session.execute(delete_stmt, execution_options={"synchronize_session": False})
CursorResult,
self.sa_session.execute(delete_stmt, execution_options={"synchronize_session": False}),
)
deleted_associations_count = result.rowcount
@@ -474,7 +540,10 @@ class NotificationManager:
return notification
def _user_notifications_query(
self, user: User, since: Optional[datetime] = None, active_only: Optional[bool] = True
self,
user: User,
since: Optional[datetime] = None,
active_only: Optional[bool] = True,
):
stmt = (
select(*self.user_notification_columns)
@@ -552,7 +621,7 @@ class DefaultStrategy(NotificationRecipientResolverStrategy):
user_ids_from_roles_stmt = self._get_all_user_ids_from_roles_query(all_role_ids)
union_stmt = union(user_ids_from_groups_stmt, user_ids_from_roles_stmt)
user_ids_from_groups_and_roles = {id for id, in self.sa_session.execute(union_stmt)}
user_ids_from_groups_and_roles = {id for (id,) in self.sa_session.execute(union_stmt)}
unique_user_ids.update(user_ids_from_groups_and_roles)
stmt = select(User).where(User.id.in_(unique_user_ids))
@@ -591,7 +660,7 @@ class DefaultStrategy(NotificationRecipientResolverStrategy):
.where(GroupRoleAssociation.role_id.in_(role_ids))
.distinct()
)
group_ids_from_roles = {id for id, in self.sa_session.execute(stmt) if id is not None}
group_ids_from_roles = {id for (id,) in self.sa_session.execute(stmt) if id is not None}
new_group_ids = group_ids_from_roles - processed_group_ids
# Get role IDs associated with any of the given group IDs
@@ -601,7 +670,7 @@ class DefaultStrategy(NotificationRecipientResolverStrategy):
.where(GroupRoleAssociation.group_id.in_(group_ids))
.distinct()
)
role_ids_from_groups = {id for id, in self.sa_session.execute(stmt) if id is not None}
role_ids_from_groups = {id for (id,) in self.sa_session.execute(stmt) if id is not None}
new_role_ids = role_ids_from_groups - processed_role_ids
# Stop if there are no new group or role IDs to process
@@ -713,7 +782,6 @@ class EmailNotificationTemplateBuilder(Protocol):
class MessageEmailNotificationTemplateBuilder(EmailNotificationTemplateBuilder):
markdown_to = {
TemplateFormats.HTML: to_html,
TemplateFormats.TXT: lambda x: x, # TODO: strip markdown?
@@ -730,9 +798,10 @@ class MessageEmailNotificationTemplateBuilder(EmailNotificationTemplateBuilder):
class NewSharedItemEmailNotificationTemplateBuilder(EmailNotificationTemplateBuilder):
def get_content(self, template_format: TemplateFormats) -> AnyNotificationContent:
content = NewSharedItemNotificationContent.model_construct(**self.notification.content) # type: ignore[arg-type]
content = NewSharedItemNotificationContent.model_construct(
**self.notification.content
) # type: ignore[arg-type]
return content
def get_subject(self) -> str:
@@ -741,7 +810,6 @@ class NewSharedItemEmailNotificationTemplateBuilder(EmailNotificationTemplateBui
class EmailNotificationChannelPlugin(NotificationChannelPlugin):
# Register the supported email templates here
email_templates_by_category: dict[PersonalNotificationCategory, type[EmailNotificationTemplateBuilder]] = {
PersonalNotificationCategory.message: MessageEmailNotificationTemplateBuilder,
+164
View File
@@ -0,0 +1,164 @@
"""Periodic gauge emitter for control-queue depth and SSE-connection counts.
Scheduled by Celery beat (see ``galaxy.celery.__init__.setup_periodic_tasks``)
at a fixed cadence. Opens a short-lived kombu connection, iterates the control
queues returned by ``all_control_queues_for_declare`` and samples each queue's
message-count via a passive declare. Also samples in-memory connection counts
from ``SSEConnectionManager`` and the active-``WorkerProcess`` count from the
database.
All instrumentation no-ops when ``statsd_client`` is ``None`` — i.e. statsd
isn't configured.
The sub-emitters take narrow, typed collaborators (a kombu connection, an
application stack, a model mapping, the statsd client, an SSE manager) rather
than the whole ``StructuredApp``. The Celery task in
``galaxy.celery.tasks.emit_queue_metrics_task`` is the composition root that
resolves those narrow deps from the app and passes them in.
"""
import datetime
import logging
from collections import defaultdict
from collections.abc import Callable
from typing import (
Optional,
TYPE_CHECKING,
)
from sqlalchemy import (
func,
select,
)
from galaxy.managers.sse import SSEConnectionManager
from galaxy.model import WorkerProcess
from galaxy.model.mapping import GalaxyModelMapping
from galaxy.model.orm.now import now
from galaxy.queues import (
all_control_queues_for_declare,
DEFAULT_ACTIVE_PROCESS_WINDOW_SECONDS,
)
from galaxy.web_stack import ApplicationStack
if TYPE_CHECKING:
from kombu import Connection
from galaxy.web.statsd_client import VanillaGalaxyStatsdClient
log = logging.getLogger(__name__)
def emit_sse_connection_gauges(
statsd_client: "VanillaGalaxyStatsdClient",
sse_manager: SSEConnectionManager,
) -> None:
"""Emit ``galaxy.sse.connections.active`` gauges by connection kind."""
statsd_client.timing(
"galaxy.sse.connections.active",
sse_manager.total_broadcast_connections,
tags={"kind": "broadcast"},
)
statsd_client.timing(
"galaxy.sse.connections.active",
sse_manager.total_per_user_connections,
tags={"kind": "per_user"},
)
def emit_control_queue_depth(
statsd_client: "VanillaGalaxyStatsdClient",
connection: "Optional[Connection]",
application_stack: ApplicationStack,
) -> None:
"""Emit ``galaxy.control_queue.depth`` per active webapp/handler queue.
A per-queue passive declare can fail on transports that don't implement it
(e.g. the sqlalchemy kombu transport) or for queues that don't yet exist on
the broker. Those are expected and quiet — logged at DEBUG, no metric, move
on. Errors at the broker-connection layer propagate up so the caller can
surface them.
"""
if connection is None:
return
queues = all_control_queues_for_declare(application_stack)
if not queues:
return
with connection.clone() as conn:
channel = conn.channel()
try:
for queue in queues:
try:
declared = queue.bind(channel).queue_declare(passive=True)
except Exception:
log.debug(
"queue_metrics: passive declare failed for %s",
queue.name,
exc_info=True,
)
continue
statsd_client.timing(
"galaxy.control_queue.depth",
declared.message_count,
tags={"queue_name": queue.name},
)
finally:
channel.close()
def emit_worker_process_gauge(
statsd_client: "VanillaGalaxyStatsdClient",
model: GalaxyModelMapping,
) -> None:
"""Emit ``galaxy.worker_process.active`` gauge grouped by ``app_type``."""
cutoff = now() - datetime.timedelta(seconds=DEFAULT_ACTIVE_PROCESS_WINDOW_SECONDS)
stmt = (
select(WorkerProcess.app_type, func.count(WorkerProcess.id))
.where(WorkerProcess.update_time > cutoff)
.group_by(WorkerProcess.app_type)
)
counts: dict[str, int] = defaultdict(int)
with model.new_session() as session:
for app_type, count in session.execute(stmt):
counts[app_type or "unknown"] = int(count)
for app_type, count in counts.items():
statsd_client.timing(
"galaxy.worker_process.active",
count,
tags={"app_type": app_type},
)
def _run(name: str, statsd_client: "VanillaGalaxyStatsdClient", fn: Callable[[], None]) -> None:
"""Run a sub-emitter, isolating its failures.
A broken sub-emitter logs once at WARNING and increments
``galaxy.queue_metrics.error`` (tagged by emitter name) so the failure is
observable in metrics without the Celery-beat wrapper logging on every
tick. The other sub-emitters continue to run on this tick.
"""
try:
fn()
except Exception:
log.warning("queue_metrics: %s emitter failed", name, exc_info=True)
statsd_client.incr("galaxy.queue_metrics.error", tags={"emitter": name})
def emit_queue_metrics(
statsd_client: "Optional[VanillaGalaxyStatsdClient]",
connection: "Optional[Connection]",
application_stack: ApplicationStack,
model: GalaxyModelMapping,
sse_manager: Optional[SSEConnectionManager],
) -> None:
"""Periodic entry-point — no-ops when statsd isn't configured."""
if statsd_client is None:
return
if sse_manager is not None:
_run("sse_connections", statsd_client, lambda: emit_sse_connection_gauges(statsd_client, sse_manager))
_run(
"control_queue_depth",
statsd_client,
lambda: emit_control_queue_depth(statsd_client, connection, application_stack),
)
_run("worker_process", statsd_client, lambda: emit_worker_process_gauge(statsd_client, model))
+242
View File
@@ -0,0 +1,242 @@
"""Server-Sent Events (SSE) connection manager for real-time notifications.
Manages per-worker in-memory mapping of user IDs to asyncio.Queue instances,
enabling push of events from any thread (e.g. Kombu control queue worker)
to async SSE endpoint handlers running in the uvicorn event loop.
"""
import asyncio
import logging
from collections import defaultdict
from collections.abc import (
AsyncIterator,
Awaitable,
Callable,
)
from dataclasses import dataclass
from datetime import datetime
from typing import (
Optional,
)
from galaxy.model.orm.now import now
from galaxy.web.statsd_client import VanillaGalaxyStatsdClient
log = logging.getLogger(__name__)
def make_event_id() -> str:
"""Return an SSE ``id`` string for Last-Event-ID replay.
Uses ``galaxy.model.orm.now`` so the timestamp format matches the rest of
Galaxy's database-backed timestamps (timezone-naive UTC). Kept in one place
so producers and the parse path cannot drift.
"""
return now().isoformat()
def parse_event_id(event_id: str) -> Optional[datetime]:
"""Inverse of :func:`make_event_id`. Returns ``None`` if unparseable."""
try:
return datetime.fromisoformat(event_id)
except (ValueError, TypeError):
return None
#: Async callable returning True when the client has disconnected. The SSE
#: stream loop polls this each iteration so managers don't depend on starlette.
IsDisconnected = Callable[[], Awaitable[bool]]
@dataclass
class SSEEvent:
"""An event to be sent to an SSE client."""
event: str # e.g. "notification_update", "broadcast_update", "notification_status"
data: str # JSON payload
id: Optional[str] = None # ISO timestamp, used by EventSource as Last-Event-ID on reconnect
def to_wire(self) -> str:
"""Serialize this event to the SSE wire format (``event:…\\ndata:…\\n[id:…\\n]\\n``)."""
frame = f"event: {self.event}\ndata: {self.data}\n"
if self.id:
frame += f"id: {self.id}\n"
return frame + "\n"
class SSEConnectionManager:
"""Per-worker manager for SSE connections.
Maps user_ids to sets of asyncio.Queue instances. Each SSE connection
gets its own queue. The manager is thread-safe for push operations
via ``loop.call_soon_threadsafe``.
Lifecycle:
- Instantiated once per Galaxy worker process (on app object).
- ``connect()`` is called from the SSE async endpoint (event loop thread).
- ``disconnect()`` is called from the SSE endpoint's ``finally`` block.
- ``push_to_user()`` / ``push_broadcast()`` are called from ANY thread
(typically the Kombu daemon thread via control task handlers).
"""
def __init__(self, statsd_client: Optional[VanillaGalaxyStatsdClient] = None) -> None:
self._connections: dict[int, set[asyncio.Queue]] = defaultdict(set)
self._session_connections: dict[int, set[asyncio.Queue]] = defaultdict(set)
self._broadcast_connections: set[asyncio.Queue] = set()
self._loop: Optional[asyncio.AbstractEventLoop] = None
self._statsd_client = statsd_client
def _ensure_loop(self) -> None:
"""Capture the running asyncio event loop. Must be called from async context."""
if self._loop is None or self._loop.is_closed():
self._loop = asyncio.get_running_loop()
# -- Called from ASYNC context (uvicorn event loop thread) --
def connect(self, user_id: Optional[int], galaxy_session_id: Optional[int] = None) -> asyncio.Queue:
"""Register a new SSE connection. Returns a queue to await events from.
Called from the SSE endpoint handler (async context). A ``ready`` event is
enqueued immediately so that clients (and tests) can synchronize on the
server-side subscription rather than the underlying socket open event.
``galaxy_session_id`` is the dispatch key for events that target a
specific browser session (e.g. history updates for anonymous users,
whose ``user_id`` is ``None``).
"""
self._ensure_loop()
queue: asyncio.Queue = asyncio.Queue(maxsize=64)
if user_id is not None:
self._connections[user_id].add(queue)
if galaxy_session_id is not None:
self._session_connections[galaxy_session_id].add(queue)
self._broadcast_connections.add(queue)
queue.put_nowait(SSEEvent(event="ready", data=""))
log.debug(
"SSE connection opened for user_id=%s session_id=%s (total=%d)",
user_id,
galaxy_session_id,
len(self._broadcast_connections),
)
return queue
def disconnect(
self,
user_id: Optional[int],
queue: asyncio.Queue,
galaxy_session_id: Optional[int] = None,
) -> None:
"""Unregister an SSE connection.
Called from the SSE endpoint's ``finally`` block (async context).
"""
if user_id is not None:
self._connections[user_id].discard(queue)
if not self._connections[user_id]:
del self._connections[user_id]
if galaxy_session_id is not None:
self._session_connections[galaxy_session_id].discard(queue)
if not self._session_connections[galaxy_session_id]:
del self._session_connections[galaxy_session_id]
self._broadcast_connections.discard(queue)
log.debug(
"SSE connection closed for user_id=%s session_id=%s (total=%d)",
user_id,
galaxy_session_id,
len(self._broadcast_connections),
)
# -- Called from ANY thread (Kombu thread or async) --
def push_to_user(self, user_id: int, event: SSEEvent) -> None:
"""Thread-safe. Push an event to all SSE connections for a specific user."""
for queue in list(self._connections.get(user_id, [])):
self._safe_put(queue, event)
def push_to_session(self, galaxy_session_id: int, event: SSEEvent) -> None:
"""Thread-safe. Push an event to all SSE connections for a specific galaxy_session.
Used to route per-browser events (e.g. history updates for anonymous
histories) when there is no registered ``user_id`` to key on.
"""
for queue in list(self._session_connections.get(galaxy_session_id, [])):
self._safe_put(queue, event)
def push_broadcast(self, event: SSEEvent) -> None:
"""Thread-safe. Push an event to ALL connected SSE clients."""
for queue in list(self._broadcast_connections):
self._safe_put(queue, event)
def _safe_put(self, queue: asyncio.Queue, event: SSEEvent) -> None:
"""Cross the thread boundary safely using ``call_soon_threadsafe``."""
if self._loop is None or self._loop.is_closed():
return
try:
self._loop.call_soon_threadsafe(self._do_put, queue, event)
except RuntimeError:
# Event loop is closed or shutting down
pass
def _do_put(self, queue: asyncio.Queue, event: SSEEvent) -> None:
"""Runs ON the event loop thread. Safe to touch asyncio.Queue here."""
try:
queue.put_nowait(event)
except asyncio.QueueFull:
log.warning("SSE queue full, dropping event: %s", event.event)
if self._statsd_client is not None:
self._statsd_client.incr("galaxy.sse.connections.dropped")
@property
def connected_user_ids(self) -> set[int]:
return set(self._connections.keys())
@property
def total_connections(self) -> int:
return len(self._broadcast_connections)
@property
def total_broadcast_connections(self) -> int:
"""Number of all active SSE connections (includes anonymous).
Every connection is added to ``_broadcast_connections``; this is the
most accurate "SSE clients currently connected" gauge.
"""
return len(self._broadcast_connections)
@property
def total_per_user_connections(self) -> int:
"""Number of active SSE connections bound to a specific user_id."""
return sum(len(queues) for queues in self._connections.values())
# -- High-level streaming helper --
async def stream(
self,
is_disconnected: IsDisconnected,
user_id: Optional[int],
catch_up: Optional[SSEEvent] = None,
keepalive: float = 30.0,
galaxy_session_id: Optional[int] = None,
) -> AsyncIterator[str]:
"""Yield SSE-framed strings for one connected client.
Handles ``connect``, optional catch-up event priming, the main event
loop with a keepalive comment on timeout, disconnect detection, and
``disconnect`` in ``finally``. The ``is_disconnected`` callable is
what the service passes in (typically ``request.is_disconnected`` from
starlette) so the manager stays framework-agnostic.
"""
queue = self.connect(user_id, galaxy_session_id)
if catch_up is not None:
await queue.put(catch_up)
try:
while True:
if await is_disconnected():
break
try:
event: SSEEvent = await asyncio.wait_for(queue.get(), timeout=keepalive)
yield event.to_wire()
except asyncio.TimeoutError:
yield ": keepalive\n\n"
finally:
self.disconnect(user_id, queue, galaxy_session_id)
+166
View File
@@ -0,0 +1,166 @@
"""Producer-side SSE helper: fans events out across Galaxy processes.
Kept in its own module (separate from ``galaxy.managers.sse``) because the
dispatcher depends on ``galaxy.queue_worker`` and ``galaxy.queues``, while
``queue_worker`` in turn depends on the connection types in ``sse``. Splitting
the producer (``SSEEventDispatcher``) from the connection state
(``SSEConnectionManager`` / ``SSEEvent``) breaks that cycle without requiring
inline imports in the hot path.
"""
import logging
import threading
import time
from collections.abc import Callable
from typing import (
Any,
Optional,
)
from cachetools import TTLCache
from kombu import Queue
from galaxy.managers.sse import make_event_id
from galaxy.queue_worker import (
ControlTask,
GalaxyQueueWorker,
)
from galaxy.queues import all_control_queues_for_declare
from galaxy.web.statsd_client import VanillaGalaxyStatsdClient
from galaxy.web_stack import ApplicationStack
log = logging.getLogger(__name__)
class SSEEventDispatcher:
"""Fans out SSE events across all Galaxy worker processes via the control queue.
Dependencies are injected individually so the dispatcher can be unit-tested
without a full ``app``. ``queue_worker`` is ``Optional`` because unit-test
mock apps and Galaxy configurations without AMQP don't construct one — the
dispatcher silently no-ops in that case.
``statsd_client`` is optional — if ``None`` (statsd not configured), all
instrumentation becomes a cheap attribute-lookup no-op.
"""
# TTL for the active-worker declare-queue cache. The WorkerProcess heartbeat
# writes every 60 s and ``all_control_queues_for_declare`` filters on a 120 s
# window, so a 30 s cache cannot produce a result that wasn't also valid in
# the non-cached call. Surfaced as a class constant so tests can monkey-patch.
_DECLARE_QUEUES_TTL_SECONDS = 30
def __init__(
self,
queue_worker: Optional[GalaxyQueueWorker],
application_stack: ApplicationStack,
statsd_client: Optional[VanillaGalaxyStatsdClient] = None,
clock: Callable[[], float] = time.monotonic,
# Factory return is typed ``Any`` so ``ControlTask`` itself and test-only
# duck-typed doubles (FakeControlTask/BoomControlTask/NoopControlTask)
# all satisfy the signature under mypy.
control_task_factory: Callable[[GalaxyQueueWorker], Any] = ControlTask,
queues_provider: Optional[Callable[[], list[Queue]]] = None,
) -> None:
self._queue_worker = queue_worker
self._application_stack = application_stack
self._statsd_client = statsd_client
self._clock = clock
self._control_task_factory = control_task_factory
# Default provider closes over application_stack so tests can pass a
# plain ``lambda: [...]`` without needing a stack.
self._queues_provider = queues_provider or (
lambda: all_control_queues_for_declare(application_stack, webapp_only=True)
)
self._declare_queues_cache: TTLCache = TTLCache(maxsize=1, ttl=self._DECLARE_QUEUES_TTL_SECONDS, timer=clock)
self._declare_queues_lock = threading.RLock()
def _get_declare_queues(self) -> list[Queue]:
# Empty results (startup before DatabaseHeartbeat registers this process,
# or a transient DB error swallowed by the provider) must not be pinned
# for the full TTL — they'd silently drop every SSE event until the next
# expiry. Only cache non-empty results.
with self._declare_queues_lock:
try:
return self._declare_queues_cache["webapp"]
except KeyError:
queues = self._queues_provider()
if queues:
self._declare_queues_cache["webapp"] = queues
return queues
def _send(self, task: str, kwargs: dict[str, Any]) -> None:
if self._queue_worker is None:
# AMQP not configured at all (e.g. unit-test mock app). Skip silently.
log.debug("SSE dispatch skipped: no queue_worker configured (task=%s)", task)
if self._statsd_client is not None:
self._statsd_client.incr("galaxy.sse.dispatch.skipped_no_qw")
return
if self._statsd_client is not None:
self._statsd_client.incr("galaxy.sse.dispatch.count", tags={"task": task})
# Only fan out to webapp processes — job handlers and workflow schedulers
# don't have browser SSE connections to push to.
declare_queues = self._get_declare_queues()
control_task = self._control_task_factory(self._queue_worker)
start_time = time.perf_counter() if self._statsd_client is not None else 0.0
try:
control_task.send_task(
payload={"task": task, "kwargs": kwargs},
routing_key="control.*",
expiration=10,
declare_queues=declare_queues,
)
finally:
if self._statsd_client is not None:
dt_ms = int((time.perf_counter() - start_time) * 1000)
self._statsd_client.timing("galaxy.sse.dispatch.latency_ms", dt_ms, tags={"task": task})
def notify_users(self, user_ids: list[int], payload: str, event_id: Optional[str] = None) -> None:
self._send(
"notify_users",
{
"user_ids": user_ids,
"payload": payload,
"event_id": event_id or make_event_id(),
},
)
def notify_broadcast(self, payload: str, event_id: Optional[str] = None) -> None:
self._send(
"notify_broadcast",
{
"payload": payload,
"event_id": event_id or make_event_id(),
},
)
def history_update(
self,
user_updates: dict[str, list[int]],
event_id: Optional[str] = None,
session_updates: Optional[dict[str, list[int]]] = None,
) -> None:
kwargs: dict[str, Any] = {
"user_updates": user_updates,
"event_id": event_id or make_event_id(),
}
if session_updates:
# Only include when non-empty: anonymous histories are uncommon on
# most deployments, and an empty dict is wasted wire payload.
kwargs["session_updates"] = session_updates
self._send("history_update", kwargs)
def entry_point_update(self, user_id: int, event_id: Optional[str] = None) -> None:
"""Fan out a wake-up ``entry_point_update`` event for one user.
The client always refetches the canonical entry-point list on receipt,
so no IDs are sent — keeping the payload small and the dispatch path
free of per-event encoding work.
"""
self._send(
"entry_point_update",
{
"user_id": user_id,
"event_id": event_id or make_event_id(),
},
)
+52 -5
View File
@@ -17,6 +17,8 @@ from galaxy.model.orm.now import now
log = logging.getLogger(__name__)
WEBAPP = "webapp" # WorkerProcess.app_type for web apps.
SSE_MONITOR = "sse_monitor" # WorkerProcess.app_type for the standalone SSE monitor process.
SSE_MONITOR_SERVER_PREFIX = "sse_monitor." # server_name prefix used by the standalone SSE monitor process.
class DatabaseHeartbeat:
@@ -27,7 +29,9 @@ class DatabaseHeartbeat:
self.hostname = socket.gethostname()
self._engine = application_stack.app.model.engine
self._is_config_watcher = False
self._is_history_audit_monitor = False
self._observers = []
self._audit_monitor_observers = []
self.exit = threading.Event()
self.thread = None
self.active = False
@@ -72,6 +76,9 @@ class DatabaseHeartbeat:
def add_change_callback(self, callback):
self._observers.append(callback)
def add_audit_monitor_change_callback(self, callback):
self._audit_monitor_observers.append(callback)
@property
def is_config_watcher(self):
return self._is_config_watcher
@@ -83,6 +90,28 @@ class DatabaseHeartbeat:
for callback in self._observers:
callback(self._is_config_watcher)
@property
def is_history_audit_monitor(self):
return self._is_history_audit_monitor
@is_history_audit_monitor.setter
def is_history_audit_monitor(self, value):
self._is_history_audit_monitor = value
log.debug(
"%s %s history audit monitor",
self.server_name,
"is" if self._is_history_audit_monitor else "is not",
)
for callback in self._audit_monitor_observers:
callback(self._is_history_audit_monitor)
def _app_type(self):
if self.application_stack.app.is_webapp:
return WEBAPP
if self.server_name.startswith(SSE_MONITOR_SERVER_PREFIX):
return SSE_MONITOR
return None
def update_watcher_designation(self):
expression = self._worker_process_identifying_clause()
stmt = select(WorkerProcess).with_for_update(of=WorkerProcess).where(expression)
@@ -90,19 +119,37 @@ class DatabaseHeartbeat:
worker_process = session.scalars(stmt).first()
if not worker_process:
worker_process = WorkerProcess(server_name=self.server_name, hostname=self.hostname)
if self.application_stack.app.is_webapp:
worker_process.app_type = WEBAPP
app_type = self._app_type()
if app_type is not None:
worker_process.app_type = app_type
worker_process.update_time = now()
worker_process.pid = self.pid
session.add(worker_process)
active = list(self.get_active_processes(self.heartbeat_interval + 1))
# We only want a single process watching the various config files on the file system.
# We just pick the max server name for simplicity
webapp_servers = [
p.server_name for p in self.get_active_processes(self.heartbeat_interval + 1) if p.app_type == WEBAPP
]
webapp_servers = [p.server_name for p in active if p.app_type == WEBAPP]
is_config_watcher = bool(webapp_servers) and self.server_name == max(webapp_servers)
if is_config_watcher != self.is_config_watcher:
self.is_config_watcher = is_config_watcher
# The history-audit monitor is a single elected process too, but preference
# goes to a standalone sse_monitor daemon when one is running so the
# monitor's postgres LISTEN isn't blocked by webapp GIL pauses. If no
# dedicated process is registered we fall back to a webapp (same
# max-server_name tiebreaker as config_watcher).
audit_leader = self._elect_audit_leader(active, webapp_servers)
is_history_audit_monitor = audit_leader is not None and self.server_name == audit_leader
if is_history_audit_monitor != self.is_history_audit_monitor:
self.is_history_audit_monitor = is_history_audit_monitor
@staticmethod
def _elect_audit_leader(active, webapp_servers):
monitor_servers = [p.server_name for p in active if p.app_type == SSE_MONITOR]
if monitor_servers:
return min(monitor_servers)
if webapp_servers:
return max(webapp_servers)
return None
def send_database_heartbeat(self):
if self.active:
@@ -0,0 +1,62 @@
"""Add pg_notify to history audit triggers
Revision ID: b8d5e2f9a1c7
Revises: f5e9e4bca542
Create Date: 2026-04-17 14:30:00.000000
The SSE-based history update pipeline (see `managers/history_audit_monitor.py`)
depends on a PostgreSQL LISTEN on the `galaxy_history_update` channel. For
existing installations, trigger functions installed by earlier migrations (most
recently `c716ee82337b_replace_triggers`) do not emit the corresponding
`pg_notify`, so the monitor wakes up only from the poll-timeout fallback and
per-history events are never dispatched in real time. This revision replaces
both audit trigger functions with versions that emit `pg_notify` for each
affected history id, matching `model/triggers/update_audit_table.py` used on
fresh installs.
The STATEMENT-vs-ROW decision must match the trigger DEFINITION installed by
`c716ee82337b` so the function body references the right context (`new_table`
vs `NEW`); both use `version > 10` (and treat offline mode as STATEMENT).
SQLite installations use the poll-only path and require no change.
"""
from alembic import op
from galaxy.model.migrations.util import (
_is_sqlite,
transaction,
)
from galaxy.model.triggers.update_audit_table import (
build_trigger_fn,
fn_prefix,
use_statement_trigger,
)
revision = "b8d5e2f9a1c7"
down_revision = "f5e9e4bca542"
branch_labels = None
depends_on = None
def upgrade():
if _is_sqlite():
return
with transaction():
_install_functions(with_notify=True)
def downgrade():
if _is_sqlite():
return
with transaction():
_install_functions(with_notify=False)
def _install_functions(with_notify: bool) -> None:
version_info = op.get_bind().engine.dialect.server_version_info
# Offline mode (no live connection) matches c716ee82337b: assume STATEMENT.
statement = version_info is None or use_statement_trigger(version_info[0])
for id_field in ("history_id", "id"):
fn_name = f"{fn_prefix}_{id_field}"
op.execute(build_trigger_fn(fn_name, id_field, use_statement=statement, with_notify=with_notify))
+69 -44
View File
@@ -3,6 +3,9 @@ from sqlalchemy import DDL
# function name prefix
fn_prefix = "fn_audit_history_by"
# channel used by pg_notify so HistoryAuditMonitor can LISTEN for updates
NOTIFY_CHANNEL = "galaxy_history_update"
# map between source table and associated incoming id field
trigger_config = {
"history_dataset_association": "history_id",
@@ -11,6 +14,68 @@ trigger_config = {
}
def use_statement_trigger(version: int) -> bool:
"""Return True when the postgres version supports the STATEMENT variant.
Fresh installs and the pg_notify migration share this predicate to ensure
the trigger function body (STATEMENT references new_table, ROW references NEW)
matches the trigger definition installed at that version.
"""
return version > 10
def build_trigger_fn(function_name: str, id_field: str, *, use_statement: bool, with_notify: bool = True) -> str:
"""Build the plpgsql CREATE OR REPLACE FUNCTION body for an audit trigger.
Shared between runtime install (update_audit_table.install) and alembic
migrations so the two cannot drift.
"""
if use_statement:
notify_block = (
f"""
FOR _history_id IN SELECT DISTINCT {id_field} FROM new_table WHERE {id_field} IS NOT NULL
LOOP
PERFORM pg_notify('{NOTIFY_CHANNEL}', _history_id::text);
END LOOP;
"""
if with_notify
else ""
)
declare_block = "DECLARE _history_id integer;" if with_notify else ""
return f"""
CREATE OR REPLACE FUNCTION {function_name}()
RETURNS TRIGGER
LANGUAGE 'plpgsql'
AS $BODY$
{declare_block}
BEGIN
INSERT INTO history_audit (history_id, update_time)
SELECT DISTINCT {id_field}, clock_timestamp() AT TIME ZONE 'UTC'
FROM new_table
WHERE {id_field} IS NOT NULL
ON CONFLICT DO NOTHING;
{notify_block}
RETURN NULL;
END;
$BODY$
"""
notify_stmt = f"PERFORM pg_notify('{NOTIFY_CHANNEL}', NEW.{id_field}::text);" if with_notify else ""
return f"""
CREATE OR REPLACE FUNCTION {function_name}()
RETURNS TRIGGER
LANGUAGE 'plpgsql'
AS $BODY$
BEGIN
INSERT INTO history_audit (history_id, update_time)
VALUES (NEW.{id_field}, clock_timestamp() AT TIME ZONE 'UTC')
ON CONFLICT DO NOTHING;
{notify_stmt}
RETURN NULL;
END;
$BODY$
"""
def install(engine):
"""Install history audit table triggers"""
sql = _postgres_install(engine) if "postgres" in engine.name else _sqlite_install()
@@ -41,47 +106,6 @@ def _postgres_install(engine):
sql = []
# PostgreSQL trigger function template
# need to make separate functions purely because the incoming history_id field name will be
# different for different source tables. There may be a fancier way to dynamically choose
# between incoming fields, but having 2 triggers fns seems straightforward
def statement_trigger_fn(id_field):
fn = f"{fn_prefix}_{id_field}"
return f"""
CREATE OR REPLACE FUNCTION {fn}()
RETURNS TRIGGER
LANGUAGE 'plpgsql'
AS $BODY$
BEGIN
INSERT INTO history_audit (history_id, update_time)
SELECT DISTINCT {id_field}, clock_timestamp() AT TIME ZONE 'UTC'
FROM new_table
WHERE {id_field} IS NOT NULL
ON CONFLICT DO NOTHING;
RETURN NULL;
END;
$BODY$
"""
def row_trigger_fn(id_field):
fn = f"{fn_prefix}_{id_field}"
return f"""
CREATE OR REPLACE FUNCTION {fn}()
RETURNS TRIGGER
LANGUAGE 'plpgsql'
AS $BODY$
BEGIN
INSERT INTO history_audit (history_id, update_time)
VALUES (NEW.{id_field}, clock_timestamp() AT TIME ZONE 'UTC')
ON CONFLICT DO NOTHING;
RETURN NULL;
END;
$BODY$
"""
def trigger_def(source_table: str, id_field: str, operation: str, version: int, when: str = "AFTER") -> str:
fn = f"{fn_prefix}_{id_field}"
# PostgreSQL supports many triggers per operation/table so the label can
@@ -93,7 +117,7 @@ def _postgres_install(engine):
# The use of the keyword PROCEDURE here is historical and deprecated (https://www.postgresql.org/docs/11/sql-createtrigger.html).
function_keyword = "FUNCTION" if version >= 11 else "PROCEDURE"
create_or_replace = "CREATE OR REPLACE" if version >= 14 else "CREATE"
if version >= 10 and when == "AFTER":
if use_statement_trigger(version) and when == "AFTER":
return f"""
{create_or_replace} TRIGGER {trigger_name}
AFTER {operation}
@@ -114,10 +138,11 @@ def _postgres_install(engine):
# pick row or statement triggers depending on postgres version
version = engine.dialect.server_version_info[0]
trigger_fn = statement_trigger_fn if version >= 10 else row_trigger_fn
statement = use_statement_trigger(version)
for id_field in ["history_id", "id"]:
sql.append(trigger_fn(id_field))
fn_name = f"{fn_prefix}_{id_field}"
sql.append(build_trigger_fn(fn_name, id_field, use_statement=statement, with_notify=True))
for source_table, id_field in trigger_config.items():
for operation in ["UPDATE", "INSERT"]:
@@ -9,6 +9,7 @@ galaxy-data.
import os
import shutil
import tempfile
from types import SimpleNamespace
from typing import Optional
from galaxy import (
@@ -105,6 +106,11 @@ class GalaxyDataTestApp:
model.setup_global_object_store_for_models(self.object_store)
self.security_agent = self.model.security_agent
self.tag_handler = GalaxyTagHandler(self.model.session)
# statsd/observability plumbing — stubbed out so paths that read
# ``app.execution_timer_factory.galaxy_statsd_client`` (e.g. the
# queue-worker instrumentation) degrade to a no-op if ever exercised
# against this mock instead of raising ``AttributeError``.
self.execution_timer_factory = SimpleNamespace(galaxy_statsd_client=None)
self.init_datatypes()
def init_datatypes(self):
+212 -13
View File
@@ -4,6 +4,7 @@ reloading the toolbox, etc., across multiple processes.
"""
import importlib
import json
import logging
import math
import socket
@@ -12,8 +13,11 @@ import threading
import time
from inspect import ismodule
from typing import (
Any,
cast,
Optional,
TYPE_CHECKING,
TypedDict,
)
from kombu import (
@@ -27,6 +31,10 @@ from kombu.pools import producers
import galaxy.queues
from galaxy import util
from galaxy.config import reload_config_options
from galaxy.managers.sse import (
SSEConnectionManager,
SSEEvent,
)
from galaxy.model import User
from galaxy.tools import ToolBox
from galaxy.tools.data_manager.manager import DataManagers
@@ -43,7 +51,49 @@ if TYPE_CHECKING:
)
def send_local_control_task(app: "StructuredApp", task: str, get_response: bool = False, kwargs: Optional[dict] = None):
class NotifyUsersPayload(TypedDict, total=False):
"""Wire contract for the ``notify_users`` control-task kwargs."""
user_ids: list[int]
payload: str
event_id: Optional[str]
class NotifyBroadcastPayload(TypedDict, total=False):
"""Wire contract for the ``notify_broadcast`` control-task kwargs."""
payload: str
event_id: Optional[str]
class HistoryUpdatePayload(TypedDict, total=False):
"""Wire contract for the ``history_update`` control-task kwargs.
``user_updates`` maps stringified user IDs to lists of (unencoded) history IDs.
``session_updates`` is the parallel route for anonymous-owned histories,
keyed by stringified ``galaxy_session.id`` (the dispatch key never leaves
the server — browsers never see it). Stringified because AMQP JSON
serialization coerces dict keys to strings.
"""
user_updates: dict[str, list[int]]
session_updates: dict[str, list[int]]
event_id: Optional[str]
class EntryPointUpdatePayload(TypedDict, total=False):
"""Wire contract for the ``entry_point_update`` control-task kwargs."""
user_id: int
event_id: Optional[str]
def send_local_control_task(
app: "StructuredApp",
task: str,
get_response: bool = False,
kwargs: Optional[dict] = None,
) -> Any:
"""
This sends a message to the process-local control worker, which is useful
for one-time asynchronous tasks like recalculating user disk usage.
@@ -57,7 +107,16 @@ def send_local_control_task(app: "StructuredApp", task: str, get_response: bool
return control_task.send_task(payload, routing_key, local=True, get_response=get_response)
def send_control_task(app, task, noop_self=False, get_response=False, routing_key="control.*", kwargs=None):
def send_control_task(
app: "StructuredApp",
task: str,
noop_self: bool = False,
get_response: bool = False,
routing_key: str = "control.*",
kwargs: Optional[dict] = None,
expiration: Optional[int] = None,
declare_queues: Optional[list[Queue]] = None,
) -> Any:
"""
This sends a control task out to all processes, useful for things like
reloading a data table, which needs to happen individually in all
@@ -65,6 +124,9 @@ def send_control_task(app, task, noop_self=False, get_response=False, routing_ke
Set noop_self to True to not run task for current process.
Set get_response to True to wait for and return the task results
as a list.
Set expiration to a number of seconds for message TTL.
Pass ``declare_queues`` to override the default active-processes list —
e.g. the SSE dispatcher uses this to restrict fan-out to webapp processes.
"""
if kwargs is None:
kwargs = {}
@@ -73,7 +135,13 @@ def send_control_task(app, task, noop_self=False, get_response=False, routing_ke
if noop_self:
payload["noop"] = app.config.server_name
control_task = ControlTask(app.queue_worker)
return control_task.send_task(payload=payload, routing_key=routing_key, get_response=get_response)
return control_task.send_task(
payload=payload,
routing_key=routing_key,
get_response=get_response,
expiration=expiration,
declare_queues=declare_queues,
)
class ControlTask:
@@ -107,10 +175,19 @@ class ControlTask:
if message.properties["correlation_id"] == self.correlation_id:
self.response = message.payload["result"]
def send_task(self, payload, routing_key, local=False, get_response=False, timeout=10):
def send_task(
self,
payload: dict,
routing_key: str,
local: bool = False,
get_response: bool = False,
timeout: int = 10,
expiration: Optional[int] = None,
declare_queues: Optional[list[Queue]] = None,
):
if local:
declare_queues = self.control_queues
else:
elif declare_queues is None:
declare_queues = self.declare_queues
reply_to = None
callback_queue = []
@@ -129,14 +206,24 @@ class ControlTask:
correlation_id=self.correlation_id,
retry=True,
headers={"epoch": time.time()},
expiration=expiration,
)
if get_response:
with Consumer(self.connection, on_message=self.on_response, queues=callback_queue, no_ack=True):
with Consumer(
self.connection,
on_message=self.on_response,
queues=callback_queue,
no_ack=True,
):
while self.response is self._response:
self.connection.drain_events(timeout=timeout)
return self.response
except TimeoutError:
log.exception("Error waiting for task: '%s' sent with routing key '%s'", payload, routing_key)
log.exception(
"Error waiting for task: '%s' sent with routing key '%s'",
payload,
routing_key,
)
except Exception:
log.exception("Error queueing async task: '%s'. for %s", payload, routing_key)
@@ -189,7 +276,10 @@ def _get_new_toolbox(app: "UniverseApplication", save_integrated_tool_panel: boo
tool_configs = app.config.tool_configs
new_toolbox = ToolBox(
tool_configs, app.config.tool_path, app, save_integrated_tool_panel=save_integrated_tool_panel
tool_configs,
app.config.tool_path,
app,
save_integrated_tool_panel=save_integrated_tool_panel,
)
new_toolbox.data_manager_tools = app.toolbox.data_manager_tools
app.datatypes_registry.load_datatype_converters(new_toolbox, use_cached=True)
@@ -309,6 +399,71 @@ def admin_job_lock(app, **kwargs):
log.info(f"Administrative Job Lock is now set to {job_lock}. Jobs will {'not' if job_lock else 'now'} dispatch.")
def notify_users(app: "MinimalManagerApp", **kwargs) -> None:
"""Push SSE events to connected users on this worker process."""
payload = cast(NotifyUsersPayload, kwargs)
sse_manager = app[SSEConnectionManager]
event = SSEEvent(
event="notification_update",
data=payload.get("payload", "{}"),
id=payload.get("event_id"),
)
for user_id in payload.get("user_ids", []):
sse_manager.push_to_user(user_id, event)
def notify_broadcast(app: "MinimalManagerApp", **kwargs) -> None:
"""Push SSE broadcast events to all connected clients on this worker process."""
payload = cast(NotifyBroadcastPayload, kwargs)
sse_manager = app[SSEConnectionManager]
event = SSEEvent(
event="broadcast_update",
data=payload.get("payload", "{}"),
id=payload.get("event_id"),
)
sse_manager.push_broadcast(event)
def history_update(app: "MinimalManagerApp", **kwargs) -> None:
"""Push SSE history update events to connected users on this worker process.
Encodes integer history IDs here (not in the monitor) so the manager layer
stays free of presentation/security concerns. Handles both user-keyed
routing (registered users) and galaxy_session-keyed routing (anonymous
histories, which have ``user_id IS NULL``).
"""
payload = cast(HistoryUpdatePayload, kwargs)
sse_manager = app[SSEConnectionManager]
event_id = payload.get("event_id")
encode = app.security.encode_id
for user_id_str, history_ids in payload.get("user_updates", {}).items():
user_id = int(user_id_str)
encoded_ids = [encode(hid) for hid in history_ids]
data = json.dumps({"history_ids": encoded_ids})
event = SSEEvent(event="history_update", data=data, id=event_id)
sse_manager.push_to_user(user_id, event)
for session_id_str, history_ids in payload.get("session_updates", {}).items():
session_id = int(session_id_str)
encoded_ids = [encode(hid) for hid in history_ids]
data = json.dumps({"history_ids": encoded_ids})
event = SSEEvent(event="history_update", data=data, id=event_id)
sse_manager.push_to_session(session_id, event)
def entry_point_update(app: "MinimalManagerApp", **kwargs) -> None:
"""Push a wake-up SSE event to a single connected user.
The payload is empty by design: the client refetches ``/api/entry_points``
(the canonical source) on receipt, so there's nothing to narrow or merge.
Dropping the IDs from the payload also avoids per-event ``encode_id`` work
at 1000+ events/s.
"""
payload = cast(EntryPointUpdatePayload, kwargs)
sse_manager = app[SSEConnectionManager]
event = SSEEvent(event="entry_point_update", data="{}", id=payload.get("event_id"))
sse_manager.push_to_user(int(payload["user_id"]), event)
control_message_to_task = {
"create_panel_section": create_panel_section,
"reload_tool": reload_tool,
@@ -324,6 +479,10 @@ control_message_to_task = {
"reconfigure_watcher": reconfigure_watcher,
"reload_tour": reload_tour,
"reload_core_config": reload_core_config,
"notify_users": notify_users,
"notify_broadcast": notify_broadcast,
"history_update": history_update,
"entry_point_update": entry_point_update,
}
@@ -354,7 +513,14 @@ class GalaxyQueueWorker(ConsumerProducerMixin, threading.Thread):
self.control_queues = []
self.epoch = 0
def send_control_task(self, task, noop_self=False, get_response=False, routing_key="control.*", kwargs=None):
def send_control_task(
self,
task,
noop_self=False,
get_response=False,
routing_key="control.*",
kwargs=None,
):
return send_control_task(
app=self.app,
task=task,
@@ -369,14 +535,32 @@ class GalaxyQueueWorker(ConsumerProducerMixin, threading.Thread):
@property
def declare_queues(self):
# dynamically produce queues, allows addressing all known processes at a given time
# Dynamically produce queues, allows addressing all known processes at a given time.
return galaxy.queues.all_control_queues_for_declare(self.app.application_stack)
def bind_publisher(self):
"""Set up the queues needed to PUBLISH control tasks (no consumer thread).
Safe to call from any process that needs to produce control messages — notably
Celery workers, which want to fan out SSE events to web workers but must not
start a consumer themselves.
Always (re)binds. A prefork call in ``GalaxyManagerApplication.__init__`` binds
using the parent's ``config.server_name``; under gunicorn with ``--preload``
the child's ``set_postfork_server_name`` mutates ``server_name`` to e.g.
``main.1`` after fork. ``bind_and_start`` calls back into this so the
consumer's queues match what post-fork producers declare.
"""
self.exchange_queue, self.direct_queue = galaxy.queues.control_queues_from_config(self.app.config)
self.control_queues = [self.exchange_queue, self.direct_queue]
def bind_and_start(self):
# This is post-forking, so we got the correct sever name
log.info("Binding and starting galaxy control worker for %s", self.app.config.server_name)
self.exchange_queue, self.direct_queue = galaxy.queues.control_queues_from_config(self.app.config)
self.control_queues = [self.exchange_queue, self.direct_queue]
log.info(
"Binding and starting galaxy control worker for %s",
self.app.config.server_name,
)
self.bind_publisher()
self.epoch = time.time()
self.start()
@@ -388,8 +572,14 @@ class GalaxyQueueWorker(ConsumerProducerMixin, threading.Thread):
def process_task(self, body, message):
result = "NO_RESULT"
task_name = body.get("task")
statsd_client = self.app.execution_timer_factory.galaxy_statsd_client
if statsd_client is not None and task_name is not None:
statsd_client.incr("galaxy.control_queue.task.count", tags={"task": task_name})
if body["task"] in self.task_mapping:
if body.get("noop", None) != self.app.config.server_name:
outcome = "ok"
handler_start = time.perf_counter() if statsd_client is not None else 0.0
try:
f = self.task_mapping[body["task"]]
if message.headers.get("epoch", math.inf) > self.epoch:
@@ -410,7 +600,16 @@ class GalaxyQueueWorker(ConsumerProducerMixin, threading.Thread):
result = "NO_OP"
except Exception:
# this shouldn't ever throw an exception, but...
outcome = "error"
log.exception("Error running control task type: %s", body["task"])
finally:
if statsd_client is not None and task_name is not None:
dt_ms = int((time.perf_counter() - handler_start) * 1000)
statsd_client.timing(
"galaxy.control_queue.task.latency_ms",
dt_ms,
tags={"task": task_name, "outcome": outcome},
)
else:
result = "NO_OP"
else:
+45 -7
View File
@@ -4,29 +4,67 @@ All message queues used by Galaxy
"""
import datetime
import logging
import socket
from typing import Optional
from typing import (
Optional,
TYPE_CHECKING,
)
from kombu import (
Connection,
Exchange,
Queue,
)
from sqlalchemy import select
from galaxy.model import WorkerProcess
from galaxy.model.orm.now import now
if TYPE_CHECKING:
from galaxy.web_stack import ApplicationStack
log = logging.getLogger(__name__)
ALL_CONTROL = "control.*"
galaxy_exchange = Exchange("galaxy_core_exchange", type="topic")
DEFAULT_ACTIVE_PROCESS_WINDOW_SECONDS = 120
# Matches WorkerProcess.app_type set by DatabaseHeartbeat for webapp processes.
WEBAPP_APP_TYPE = "webapp"
def all_control_queues_for_declare(application_stack):
def all_control_queues_for_declare(application_stack: "ApplicationStack", webapp_only: bool = False) -> list[Queue]:
"""
For in-memory routing (used by sqlalchemy-based transports), we need to be able to
build the entire routing table in producers.
Queries ``WorkerProcess`` directly rather than going through
``DatabaseHeartbeat`` so this works from Celery workers too — they have a
``model`` but no heartbeat thread. Without this, a notification created in
a Celery task publishes a ``notify_users`` control task with an empty
``declare`` list, so on the sqlalchemy+sqlite kombu transport the message
never lands in a web worker's queue.
When ``webapp_only`` is True, only returns queues for processes that have
registered themselves with ``app_type='webapp'``. This is what the SSE
dispatcher wants: job handlers and workflow schedulers have no browser
connections, so routing SSE events to them is wasted work.
"""
# Get all active processes and construct queues for each process
process_names = (
f"{p.server_name}@{p.hostname}" for p in application_stack.app.database_heartbeat.get_active_processes()
)
return [Queue(f"control.{server_name}", galaxy_exchange, routing_key="control.*") for server_name in process_names]
app = application_stack.app
try:
stmt = select(WorkerProcess).where(
WorkerProcess.update_time > now() - datetime.timedelta(seconds=DEFAULT_ACTIVE_PROCESS_WINDOW_SECONDS)
)
if webapp_only:
stmt = stmt.where(WorkerProcess.app_type == WEBAPP_APP_TYPE)
with app.model.new_session() as session:
processes = session.scalars(stmt).all()
except Exception:
log.debug("Failed to look up active processes for control-queue declare", exc_info=True)
return []
return [Queue(f"control.{p.server_name}@{p.hostname}", galaxy_exchange, routing_key="control.*") for p in processes]
def control_queues_from_config(config):
+19 -2
View File
@@ -112,6 +112,7 @@ See Also
"""
import abc
import logging
from contextlib import contextmanager
from typing import (
Any,
@@ -157,6 +158,8 @@ from .playwright_element import PlaywrightElement
from .wait_methods_mixin import WaitMethodsMixin
from .web_element_protocol import WebElementProtocol
logger = logging.getLogger(__name__)
UNSPECIFIED_TIMEOUT = object()
@@ -1153,9 +1156,23 @@ class HasPlaywrightDriver(TimeoutMessageMixin, WaitMethodsMixin, Generic[WaitTyp
This closes all windows/tabs and releases all system resources.
The driver cannot be used after calling this method.
``browser.close()`` can raise — in CI we've seen it time out or hit
target-detached errors — and if the exception escapes before
``playwright.stop()`` runs, the per-instance asyncio loop is left
registered as "running" on the main thread. Every subsequent test's
``sync_playwright().__enter__`` then refuses to start with "Playwright
Sync API inside the asyncio loop", cascading the whole shard into
errors. Always tear down the Playwright instance even if the browser
close failed.
"""
self.close()
self._playwright_resources.playwright.stop()
try:
self.close()
finally:
try:
self._playwright_resources.playwright.stop()
except Exception:
logger.exception("Error stopping Playwright instance during quit()")
__all__ = (
+30
View File
@@ -0,0 +1,30 @@
"""Standalone SSE monitor process.
Runs the :class:`~galaxy.managers.history_audit_monitor.HistoryAuditMonitor`
(and, in principle, any future SSE event producer) in its own OS process so a
webapp's GIL pauses or fork-lifecycle events can never block history-update
dispatch. Webapps still consume SSE events normally; only the *producer* moves.
The process registers itself via :class:`~galaxy.model.database_heartbeat.DatabaseHeartbeat`
with a server_name beginning with ``sse_monitor.``. The heartbeat's
``is_history_audit_monitor`` election prefers any such process, so the monitor
automatically migrates here when this daemon is running. If it stops, a webapp
is re-elected within one heartbeat interval (~60s).
Starting the daemon
-------------------
Installed::
GALAXY_CONFIG_FILE=/etc/galaxy/galaxy.yml galaxy-sse-monitor
From a source checkout::
GALAXY_CONFIG_FILE=config/galaxy.yml python -m galaxy.sse_monitor
For production deployments, run under systemd / supervisord alongside the
webapp. A native ``gravity`` (``galaxyctl``) service type for this daemon is
tracked as a follow-up against the ``gravity`` package — until that lands,
either start the process via your process manager of choice, or omit it and
let a webapp fall back to the monitor role.
"""
+99
View File
@@ -0,0 +1,99 @@
"""Entry point for the standalone ``galaxy-sse-monitor`` daemon.
Loads a minimal :class:`GalaxyManagerApplication` (same shape the Celery
workers use), forces the server_name to ``sse_monitor.<host>.<pid>`` so
DatabaseHeartbeat's election picks this process as the history-audit monitor,
and blocks on SIGINT/SIGTERM.
"""
import logging
import os
import signal
import socket
import sys
import threading
from galaxy.celery import get_app_properties
from galaxy.model.database_heartbeat import DatabaseHeartbeat
log = logging.getLogger("galaxy.sse_monitor")
def _build_server_name() -> str:
return f"sse_monitor.{socket.gethostname()}.{os.getpid()}"
def _build_app(server_name: str):
kwargs = get_app_properties() or {}
if not kwargs:
raise RuntimeError(
"GALAXY_CONFIG_FILE (or GALAXY_ROOT_DIR with an on-disk Galaxy config) is required "
"to start galaxy-sse-monitor"
)
kwargs = dict(kwargs)
kwargs["check_migrate_databases"] = False
kwargs["use_display_applications"] = False
kwargs["use_converters"] = False
kwargs["server_name"] = server_name
import galaxy.app
return galaxy.app.GalaxyManagerApplication(configure_logging=True, **kwargs)
def main() -> int:
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s %(message)s")
server_name = _build_server_name()
log.info("Starting galaxy-sse-monitor as %s", server_name)
app = _build_app(server_name)
# GalaxyManagerApplication doesn't wire a DatabaseHeartbeat (that lives on
# UniverseApplication, which pulls in the webapp stack we don't need). We
# spin up our own and register the audit-monitor callback so election
# transitions start/stop the producer cleanly.
heartbeat = DatabaseHeartbeat(application_stack=app.application_stack)
monitor = None
if app.config.enable_sse_updates:
from galaxy.managers.history_audit_monitor import HistoryAuditMonitor
monitor = app[HistoryAuditMonitor]
heartbeat.add_audit_monitor_change_callback(monitor.on_role_change)
else:
log.warning("enable_sse_updates is False — galaxy-sse-monitor will idle with no producers")
heartbeat.start()
shutdown = threading.Event()
def _handle_signal(signum, _frame):
log.info("Received signal %s, shutting down galaxy-sse-monitor", signum)
shutdown.set()
signal.signal(signal.SIGINT, _handle_signal)
signal.signal(signal.SIGTERM, _handle_signal)
try:
shutdown.wait()
finally:
if monitor is not None:
try:
monitor.shutdown()
except Exception:
log.exception("Error shutting down HistoryAuditMonitor")
try:
heartbeat.shutdown()
except Exception:
log.exception("Error shutting down database heartbeat")
try:
app.shutdown()
except Exception:
log.exception("Error shutting down GalaxyManagerApplication")
return 0
if __name__ == "__main__":
sys.exit(main())
+5 -1
View File
@@ -41,6 +41,7 @@ from galaxy.webhooks import WebhooksRegistry
from galaxy.workflow.trs_proxy import TrsProxy
if TYPE_CHECKING:
from galaxy.app import ExecutionTimerFactory
from galaxy.config_watchers import ConfigWatchers
from galaxy.jobs import JobConfiguration
from galaxy.jobs.manager import JobManager
@@ -57,7 +58,9 @@ if TYPE_CHECKING:
WorkflowsManager,
)
from galaxy.tool_shed.galaxy_install.client import DataManagersInterface
from galaxy.tool_shed.galaxy_install.installed_repository_manager import InstalledRepositoryManager
from galaxy.tool_shed.galaxy_install.installed_repository_manager import (
InstalledRepositoryManager,
)
from galaxy.tool_util.data import ToolDataTableManager
from galaxy.tools import ToolBox
from galaxy.tools.cache import ToolCache
@@ -174,6 +177,7 @@ class StructuredApp(MinimalManagerApp):
vault: Vault
webhooks_registry: WebhooksRegistry
queue_worker: Any # 'galaxy.queue_worker.GalaxyQueueWorker'
execution_timer_factory: "ExecutionTimerFactory"
data_provider_registry: Any # 'galaxy.visualization.data_providers.registry.DataProviderRegistry'
tool_cache: "ToolCache"
tool_shed_repository_cache: Optional[ToolShedRepositoryCache]
+62
View File
@@ -0,0 +1,62 @@
"""
API endpoint for Server-Sent Events (SSE) stream.
Provides a unified event stream for all real-time push events (notifications,
history updates, etc.) independent of the notification system configuration.
"""
import logging
from typing import Optional
from fastapi import (
Header,
Request,
)
from starlette.responses import StreamingResponse
from galaxy.managers.context import ProvidesUserContext
from galaxy.webapps.galaxy.services.events import EventsService
from . import (
depends,
DependsOnTrans,
Router,
)
log = logging.getLogger(__name__)
router = Router(tags=["events"])
@router.cbv
class FastAPIEvents:
service: EventsService = depends(EventsService)
@router.get(
"/api/events/stream",
summary="Server-Sent Events stream for real-time updates.",
response_class=StreamingResponse,
)
async def stream_events(
self,
request: Request,
trans: ProvidesUserContext = DependsOnTrans,
last_event_id: Optional[str] = Header(None, alias="Last-Event-ID"),
) -> StreamingResponse:
"""Opens a Server-Sent Events (SSE) connection that pushes real-time
updates for notifications, history changes, and other events.
On reconnect, the browser sends the ``Last-Event-ID`` header automatically.
If the notification system is enabled, any notifications created since that
timestamp are delivered as a catch-up ``notification_status`` event.
Anonymous users receive only broadcast events.
"""
return StreamingResponse(
self.service.open_stream(trans, last_event_id, request.is_disconnected),
media_type="text/event-stream",
headers={
"Cache-Control": "no-cache",
"Connection": "keep-alive",
"X-Accel-Buffering": "no",
},
)
+1 -1
View File
@@ -11,6 +11,7 @@ from pydantic import (
Field,
)
from galaxy.celery.helpers import async_task_summary
from galaxy.celery.tasks import import_data_bundle
from galaxy.managers.context import ProvidesUserContext
from galaxy.managers.tool_data import ToolDataManager
@@ -25,7 +26,6 @@ from galaxy.tool_util.data._schema import (
ToolDataItem,
)
from galaxy.webapps.base.api import GalaxyFileResponse
from galaxy.webapps.galaxy.services.base import async_task_summary
from . import (
depends,
DependsOnTrans,
@@ -7,8 +7,6 @@ from typing import (
Optional,
)
from celery.result import AsyncResult
from galaxy.exceptions import (
AuthenticationRequired,
ConfigDoesNotAllowException,
@@ -32,7 +30,6 @@ from galaxy.model.store import (
)
from galaxy.schema.fields import EncodedDatabaseIdField
from galaxy.schema.schema import (
AsyncTaskResultSummary,
ToolRequestDetailedModel,
ToolRequestModel,
)
@@ -185,28 +182,6 @@ class ConsumesModelStores:
)
def async_task_summary(async_result: AsyncResult) -> AsyncTaskResultSummary:
name = None
try:
name = async_result.name
except AttributeError:
# if backend is disabled, we won't have this
pass
queue = None
try:
queue = async_result.queue
except AttributeError:
# if backend is disabled, we won't have this
pass
return AsyncTaskResultSummary(
id=str(async_result.id),
ignored=async_result.ignored,
name=name,
queue=queue,
)
def _encode_tool_request(tool_request: ToolRequest, security: IdEncodingHelper) -> dict[str, Any]:
"""Encode request IDs using strongly-typed parameter walking."""
tool_source_model = tool_request.tool_source
@@ -24,6 +24,7 @@ from galaxy import (
util,
web,
)
from galaxy.celery.helpers import async_task_summary
from galaxy.celery.tasks import compute_dataset_hash
from galaxy.datatypes.binary import Binary
from galaxy.datatypes.dataproviders.exceptions import NoProviderAvailable
@@ -88,10 +89,7 @@ from galaxy.visualization.data_providers.genome import (
)
from galaxy.visualization.data_providers.registry import DataProviderRegistry
from galaxy.webapps.base.controller import UsesVisualizationMixin
from galaxy.webapps.galaxy.services.base import (
async_task_summary,
ServiceBase,
)
from galaxy.webapps.galaxy.services.base import ServiceBase
log = logging.getLogger(__name__)
@@ -0,0 +1,44 @@
"""Service layer for the unified SSE events endpoint.
Unlike :class:`NotificationService.open_stream`, this service does **not**
require the notification system to be enabled — ``/api/events/stream`` also
serves history updates and other event types independent of the notification
configuration. When notifications are disabled the catch-up event is simply
skipped; the stream still delivers other push events.
"""
from collections.abc import AsyncIterator
from typing import (
Optional,
)
from galaxy.managers.context import ProvidesUserContext
from galaxy.managers.sse import (
IsDisconnected,
SSEConnectionManager,
)
from galaxy.webapps.galaxy.services.base import ServiceBase
from galaxy.webapps.galaxy.services.notifications import NotificationService
class EventsService(ServiceBase):
def __init__(self, sse_manager: SSEConnectionManager, notifications: NotificationService) -> None:
self.sse_manager = sse_manager
self.notifications = notifications
def open_stream(
self,
user_context: ProvidesUserContext,
last_event_id: Optional[str],
is_disconnected: IsDisconnected,
) -> AsyncIterator[str]:
"""Open an SSE events stream.
Anonymous users still register under their ``galaxy_session.id`` so the
server can route per-session events (e.g. ``history_update`` for
anonymous-owned histories) even when ``user_id`` is ``None``.
"""
user_id = user_context.user.id if not user_context.anonymous else None
session_id = user_context.galaxy_session.id if user_context.galaxy_session else None
catch_up = self.notifications.build_status_catchup(user_context, last_event_id)
return self.sse_manager.stream(is_disconnected, user_id, catch_up=catch_up, galaxy_session_id=session_id)
@@ -24,6 +24,7 @@ from galaxy import (
exceptions as glx_exceptions,
model,
)
from galaxy.celery.helpers import async_task_summary
from galaxy.celery.tasks import (
import_model_store,
prepare_history_download,
@@ -87,7 +88,6 @@ from galaxy.security.idencoding import IdEncodingHelper
from galaxy.short_term_storage import ShortTermStorageAllocator
from galaxy.util import restore_text
from galaxy.webapps.galaxy.services.base import (
async_task_summary,
ConsumesModelStores,
model_store_storage_target,
ServesExportStores,
@@ -21,6 +21,7 @@ from typing_extensions import (
)
from galaxy import exceptions
from galaxy.celery.helpers import async_task_summary
from galaxy.celery.tasks import (
change_datatype,
materialize as materialize_task,
@@ -118,7 +119,6 @@ from galaxy.security.idencoding import IdEncodingHelper
from galaxy.short_term_storage import ShortTermStorageAllocator
from galaxy.util.zipstream import ZipstreamWrapper
from galaxy.webapps.galaxy.services.base import (
async_task_summary,
ConsumesModelStores,
ensure_celery_tasks_enabled,
model_store_storage_target,
@@ -6,6 +6,7 @@ from typing import (
from pydantic import Field
from galaxy.celery.helpers import async_task_summary
from galaxy.celery.tasks import (
prepare_invocation_download,
write_invocation_to,
@@ -61,7 +62,6 @@ from galaxy.schema.tasks import (
from galaxy.security.idencoding import IdEncodingHelper
from galaxy.short_term_storage import ShortTermStorageAllocator
from galaxy.webapps.galaxy.services.base import (
async_task_summary,
ConsumesModelStores,
ensure_celery_tasks_enabled,
model_store_storage_target,
+1 -1
View File
@@ -16,6 +16,7 @@ from galaxy import (
exceptions,
model,
)
from galaxy.celery.helpers import async_task_summary
from galaxy.celery.tasks import queue_jobs
from galaxy.managers import hdas
from galaxy.managers.base import security_check
@@ -60,7 +61,6 @@ from galaxy.tool_util.parameters import (
ToolParameterBundleModel,
)
from galaxy.webapps.galaxy.services.base import (
async_task_summary,
ServiceBase,
)
from .tools import validate_tool_for_running
@@ -5,7 +5,6 @@ from typing import (
Union,
)
from galaxy.celery.tasks import send_notification_to_recipients_async
from galaxy.exceptions import (
AdminRequiredException,
AuthenticationRequired,
@@ -14,6 +13,12 @@ from galaxy.exceptions import (
)
from galaxy.managers.context import ProvidesUserContext
from galaxy.managers.notification import NotificationManager
from galaxy.managers.sse import (
make_event_id,
parse_event_id,
SSEConnectionManager,
SSEEvent,
)
from galaxy.model import User
from galaxy.schema.fields import Security
from galaxy.schema.notifications import (
@@ -35,15 +40,27 @@ from galaxy.schema.notifications import (
UserNotificationUpdateRequest,
)
from galaxy.schema.schema import AsyncTaskResultSummary
from galaxy.webapps.galaxy.services.base import (
async_task_summary,
ServiceBase,
)
from galaxy.webapps.galaxy.services.base import ServiceBase
class NotificationService(ServiceBase):
def __init__(self, notification_manager: NotificationManager):
def __init__(self, notification_manager: NotificationManager, sse_manager: SSEConnectionManager):
self.notification_manager = notification_manager
self.sse_manager = sse_manager
@property
def notifications_enabled(self) -> bool:
return self.notification_manager.notifications_enabled
def send_internal_notification(
self, request: NotificationCreateRequest, force_sync: bool = False
) -> Union[NotificationCreatedResponse, AsyncTaskResultSummary]:
"""Send a system-emitted notification on behalf of internal callers (e.g. share flows).
Unlike :meth:`send_notification`, this skips admin/permission checks because the
caller has already resolved the recipient set and is not acting on user input.
"""
return self.notification_manager.send_notification_internal(request, force_sync=force_sync)
def send_notification(
self, sender_context: ProvidesUserContext, payload: NotificationCreateRequestBody
@@ -62,28 +79,7 @@ class NotificationService(ServiceBase):
recipients=payload.recipients,
galaxy_url=galaxy_url,
)
return self.send_notification_internal(request)
def send_notification_internal(
self, request: NotificationCreateRequest, force_sync: bool = False
) -> Union[NotificationCreatedResponse, AsyncTaskResultSummary]:
"""Sends a notification to a list of recipients (users, groups or roles).
If `force_sync` is set to `True`, the notification recipients will be processed synchronously instead of
in a background task.
Note: This function is meant for internal use from other services that don't need to check sender permissions.
"""
if self.notification_manager.can_send_notifications_async and not force_sync:
result = send_notification_to_recipients_async.delay(request)
summary = async_task_summary(result)
return summary
notification, recipient_user_count = self.notification_manager.send_notification_to_recipients(request)
return NotificationCreatedResponse(
total_notifications_sent=recipient_user_count,
notification=NotificationResponse.model_validate(notification),
)
return self.notification_manager.send_notification_internal(request)
def broadcast(
self, sender_context: ProvidesUserContext, payload: BroadcastNotificationCreateRequest
@@ -99,6 +95,27 @@ class NotificationService(ServiceBase):
total_notifications_sent=1, notification=NotificationResponse.model_validate(notification)
)
def build_status_catchup(
self, user_context: ProvidesUserContext, last_event_id: Optional[str]
) -> Optional[SSEEvent]:
"""Build a ``notification_status`` SSE event covering everything since ``last_event_id``.
Returns ``None`` when catch-up isn't possible (no ``Last-Event-ID``,
unparseable timestamp, or notifications disabled) so callers can simply
pass the result to ``SSEConnectionManager.stream`` without extra guards.
"""
if not last_event_id or not self.notification_manager.notifications_enabled:
return None
since = parse_event_id(last_event_id)
if since is None:
return None
catchup = self.get_notifications_status(user_context, since)
return SSEEvent(
event="notification_status",
data=catchup.model_dump_json(),
id=make_event_id(),
)
def get_notifications_status(self, user_context: ProvidesUserContext, since: datetime) -> NotificationStatusSummary:
"""Returns the status of (unread or updated) notifications received by the user **since** a particular date and time.
+1 -1
View File
@@ -4,6 +4,7 @@ from typing import (
)
from galaxy import exceptions
from galaxy.celery.helpers import async_task_summary
from galaxy.celery.tasks import prepare_pdf_download
from galaxy.managers import base
from galaxy.managers.markdown_util import (
@@ -34,7 +35,6 @@ from galaxy.security.idencoding import IdEncodingHelper
from galaxy.short_term_storage import ShortTermStorageAllocator
from galaxy.webapps.galaxy.api.common import PageIdPathParam
from galaxy.webapps.galaxy.services.base import (
async_task_summary,
ensure_celery_tasks_enabled,
ServiceBase,
)
@@ -177,17 +177,13 @@ class ShareableService:
def _send_notification_to_users(
self, users_to_notify: set[User], item: SharableItem, status: ShareWithStatus, galaxy_url: Optional[str] = None
):
if (
self.notification_service.notification_manager.notifications_enabled
and not status.errors
and users_to_notify
):
if self.notification_service.notifications_enabled and not status.errors and users_to_notify:
request = SharedItemNotificationFactory.build_notification_request(
item, users_to_notify, status, galaxy_url
)
# We can set force_sync=True here because we already have the set of users to notify
# and there is no need to resolve them asynchronously as no groups or roles are involved.
self.notification_service.send_notification_internal(request, force_sync=True)
self.notification_service.send_internal_notification(request, force_sync=True)
class SharedItemNotificationFactory:
+2 -4
View File
@@ -9,6 +9,7 @@ from galaxy import (
exceptions as glx_exceptions,
util,
)
from galaxy.celery.helpers import async_task_summary
from galaxy.managers import api_keys
from galaxy.managers.context import (
ProvidesHistoryContext,
@@ -34,10 +35,7 @@ from galaxy.schema.schema import (
UserModel,
)
from galaxy.security.idencoding import IdEncodingHelper
from galaxy.webapps.galaxy.services.base import (
async_task_summary,
ServiceBase,
)
from galaxy.webapps.galaxy.services.base import ServiceBase
from galaxy.webapps.galaxy.services.roles import role_to_model
if TYPE_CHECKING:
+155
View File
@@ -0,0 +1,155 @@
"""Shared helpers for SSE integration tests.
The stream layer emits a ``ready`` event as the first frame on every connection
so tests can synchronize on the server-side subscription rather than the
underlying TCP socket. ``SSELineListener`` waits for that ``ready`` event before
``start()`` returns, and propagates listener-thread exceptions back to the main
thread instead of silently swallowing them.
"""
import queue
import threading
from collections.abc import Callable
from typing import Optional
import requests
from galaxy.util.wait import wait_on
CONNECT_TIMEOUT = 15
DEFAULT_WAIT_TIMEOUT = 15
def parse_sse_events(raw: str) -> list[dict]:
"""Parse raw SSE text into a list of event dicts with ``event``, ``data``, and ``id`` keys."""
events: list[dict] = []
current: dict[str, str] = {}
for line in raw.split("\n"):
if line.startswith(":"):
continue # comment / keepalive
if line == "":
if current:
events.append(current)
current = {}
continue
if ": " in line:
field, _, value = line.partition(": ")
else:
field, value = line.rstrip(":"), ""
if field in ("event", "data", "id"):
current[field] = value
if current:
events.append(current)
return events
class SSEListenerError(Exception):
"""Wraps an exception raised inside the listener thread."""
class SSELineListener:
"""Runs an SSE connection on a background thread and collects raw chunks.
``start()`` blocks until the server-side ``ready`` event has been received,
guaranteeing that any event *posted after* ``start()`` returns will be seen
by this listener. Failures in the background thread are surfaced via
``wait_for_event`` instead of silently timing out.
"""
def __init__(
self,
url: str,
api_key: str,
headers: Optional[dict] = None,
timeout: int = 30,
) -> None:
self.url = url
self.api_key = api_key
self.headers = headers or {}
self.timeout = timeout
self._collected: list[str] = []
self._stop = threading.Event()
self._ready = threading.Event()
self._errors: queue.Queue[BaseException] = queue.Queue()
self._thread = threading.Thread(target=self._listen, daemon=True)
def start(self) -> None:
self._thread.start()
wait_on(
lambda: True if self._ready.is_set() else None,
"SSE `ready` event",
timeout=CONNECT_TIMEOUT,
)
self._raise_if_errored()
def stop(self) -> None:
self._stop.set()
self._thread.join(timeout=5)
def wait_for_event(self, event_type: str, timeout: int = DEFAULT_WAIT_TIMEOUT) -> list[dict]:
"""Block until at least one event of ``event_type`` has been observed, then return all such events."""
def _check():
self._raise_if_errored()
events = self.get_events(event_type)
return events if events else None
return wait_on(_check, f"SSE {event_type} event", timeout=timeout)
def wait_for_event_where(
self,
event_type: str,
predicate: Callable[[dict], bool],
timeout: int = DEFAULT_WAIT_TIMEOUT,
) -> list[dict]:
"""Block until at least one ``event_type`` event matches ``predicate``.
Returns every ``event_type`` event observed so far, not just matches, so
callers can still inspect the surrounding stream (e.g. assert what else
did or didn't appear) after the wait resolves.
"""
def _check():
self._raise_if_errored()
events = self.get_events(event_type)
return events if any(predicate(e) for e in events) else None
return wait_on(_check, f"SSE {event_type} matching predicate", timeout=timeout)
def get_events(self, event_type: Optional[str] = None) -> list[dict]:
"""Return all collected events so far, optionally filtered by type."""
all_events = parse_sse_events("".join(self._collected))
if event_type is None:
return all_events
return [e for e in all_events if e.get("event") == event_type]
def _raise_if_errored(self) -> None:
try:
err = self._errors.get_nowait()
except queue.Empty:
return
raise SSEListenerError(f"SSE listener thread failed: {err!r}") from err
def _listen(self) -> None:
try:
resp = requests.get(
self.url,
params={"key": self.api_key},
headers=self.headers,
stream=True,
timeout=self.timeout,
)
if resp.status_code != 200:
raise RuntimeError(f"SSE connect returned HTTP {resp.status_code}: {resp.text[:200]}")
for chunk in resp.iter_content(chunk_size=None, decode_unicode=True):
if chunk:
self._collected.append(chunk)
if not self._ready.is_set() and "event: ready" in "".join(self._collected):
self._ready.set()
if self._stop.is_set():
break
resp.close()
except Exception as exc:
self._errors.put(exc)
# Ensure start() doesn't hang forever on connection failure.
self._ready.set()
@@ -22,8 +22,10 @@ from unittest import (
skip,
SkipTest,
)
from urllib.parse import urljoin
import pytest
import requests
from galaxy.app import UniverseApplication
from galaxy.tool_util.verify.test_data import TestDataResolver
@@ -238,6 +240,16 @@ class IntegrationInstance(UsesApiTestCaseMixin, UsesCeleryTasks):
if not self._app.config.database_connection.startswith("post"):
raise SkipTest("Test only valid for postgres")
def _decode_id(self, encoded_id: str) -> int:
"""Decode an encoded API id to its raw int via the live app's security helper."""
return self._app.security.decode_id(encoded_id)
def _user_id_for_api_key(self, api_key: str) -> int:
"""Return the raw integer ``User.id`` for the user owning ``api_key``."""
response = requests.get(urljoin(self.url, "api/users/current"), params={"key": api_key})
response.raise_for_status()
return self._decode_id(response.json()["id"])
def _run_tool_test(self, *args, **kwargs):
return self._test_driver.run_tool_test(*args, **kwargs)
+19 -4
View File
@@ -3,6 +3,7 @@
import datetime
import errno
import json
import logging
import os
import traceback
import unittest
@@ -76,6 +77,8 @@ try:
except ImportError:
GalaxyTestDriver = None # type: ignore[assignment, misc, unused-ignore]
logger = logging.getLogger(__name__)
def _load_config_file() -> None:
"""
@@ -433,10 +436,22 @@ class TestWithSeleniumMixin(GalaxyTestSeleniumContext, UsesApiTestCaseMixin, Use
self.target_url_from_selenium = self._target_url_from_selenium()
self.snapshots = []
self.setup_driver_and_session()
if self.run_as_admin and GALAXY_TEST_SELENIUM_ADMIN_USER_EMAIL == DEFAULT_ADMIN_USER:
self._setup_interactor()
self._setup_user(GALAXY_TEST_SELENIUM_ADMIN_USER_EMAIL)
self._try_setup_with_driver()
# Once the driver is allocated, any subsequent failure must still
# tear it down: pytest does not call tearDown when setUp raises, so
# without this the Playwright asyncio loop would stay registered as
# "running" on the main thread and cascade every subsequent test's
# setUp with "Sync API inside the asyncio loop".
try:
if self.run_as_admin and GALAXY_TEST_SELENIUM_ADMIN_USER_EMAIL == DEFAULT_ADMIN_USER:
self._setup_interactor()
self._setup_user(GALAXY_TEST_SELENIUM_ADMIN_USER_EMAIL)
self._try_setup_with_driver()
except Exception:
try:
self.tear_down_driver()
except Exception:
logger.exception("Error tearing down driver after setup_selenium failure")
raise
def _try_setup_with_driver(self):
try:
+1
View File
@@ -101,6 +101,7 @@ test =
console_scripts =
galaxy-main = galaxy.main:main
galaxy-dependencies = galaxy.dependencies.script:main
galaxy-sse-monitor = galaxy.sse_monitor.__main__:main
[options.packages.find]
where = src
+1
View File
@@ -0,0 +1 @@
../../../../lib/galaxy/sse_monitor
+1
View File
@@ -176,6 +176,7 @@ typecheck = [
"pydantic>=2.7.4", # for pydantic.mypy plugin
"types-bleach",
"types-boto",
"types-cachetools",
"types-contextvars",
"types-dataclasses",
"types-docutils",
+146
View File
@@ -0,0 +1,146 @@
"""Integration tests for SSE-based interactive-tool entry-point update notifications.
Mirrors ``test_history_sse.py``. Rather than spin up a real containerized
interactive tool (which would require docker), these tests exercise the
dispatch path by creating a ``Job`` and ``InteractiveToolEntryPoint`` rows
directly via the live app's SQLAlchemy session and invoking
``InteractiveToolManager.configure_entry_points`` with a stub ``ports_dict``.
This is exactly the moment the event fires in production — the job runner's
port-routing hook is the only upstream caller, and the SSE dispatch happens
after the DB commit regardless of how the ports were obtained.
"""
import time
from urllib.parse import urljoin
from uuid import uuid4
from galaxy.model import (
InteractiveToolEntryPoint,
Job,
)
from galaxy_test.base.populators import DatasetPopulator
from galaxy_test.base.sse import SSELineListener
from galaxy_test.driver.integration_util import IntegrationTestCase
def _make_ports_dict(tool_port: int) -> dict:
"""Stub the runner's port-routing payload — one tool_port, fake host/proto."""
return {
str(tool_port): {
"host": "host.invalid",
"port": 12345,
"protocol": "http",
}
}
class TestEntryPointSSEIntegration(IntegrationTestCase):
dataset_populator: DatasetPopulator
framework_tool_and_types = True
@classmethod
def handle_galaxy_config_kwds(cls, config):
super().handle_galaxy_config_kwds(config)
config["enable_celery_tasks"] = False
def setUp(self):
super().setUp()
self.dataset_populator = DatasetPopulator(self.galaxy_interactor)
def _events_stream_url(self) -> str:
return urljoin(self.url, "api/events/stream")
def _create_it_job_with_entry_point(self, user_id: int, tool_port: int = 8888) -> tuple[int, int]:
"""Create a minimal Job + unconfigured InteractiveToolEntryPoint row pair.
Returns ``(job_id, entry_point_id)``. The session used is the live
app's; the rows are real and survive the call.
"""
sa_session = self._app.model.context
job = Job()
job.user_id = user_id
job.tool_id = "interactivetool_simple"
job.state = Job.states.RUNNING
sa_session.add(job)
sa_session.flush()
ep = InteractiveToolEntryPoint(
job=job,
tool_port=tool_port,
entry_url="/",
name="test entry point",
label="test",
requires_domain=True,
requires_path_in_url=False,
requires_path_in_header_named=None,
)
sa_session.add(ep)
sa_session.commit()
return job.id, ep.id
def test_entry_point_update_event_fires_on_configure(self):
"""configure_entry_points should fire an ``entry_point_update`` wake-up event."""
api_key = self.galaxy_interactor.api_key
assert api_key is not None
user_id = self._user_id_for_api_key(api_key)
job_id, _ = self._create_it_job_with_entry_point(user_id)
listener = SSELineListener(self._events_stream_url(), api_key)
listener.start()
try:
sa_session = self._app.model.context
job = sa_session.get(Job, job_id)
assert job is not None
self._app.interactivetool_manager.configure_entry_points(job, _make_ports_dict(8888))
entry_point_events = listener.wait_for_event("entry_point_update")
finally:
listener.stop()
# The event carries no payload: the client refetches ``/api/entry_points``
# (the canonical source) on receipt, so the event just needs to arrive.
assert len(entry_point_events) >= 1, f"Expected entry_point_update wake-up, got: {entry_point_events}"
assert entry_point_events[0]["event"] == "entry_point_update"
def test_entry_point_update_is_scoped_to_owning_user(self):
"""User A must not see entry_point_update events for user B's jobs.
The event has no payload to cross-check with, so we assert on event
count: user A's stream should receive exactly one event for its own
``configure_entry_points`` call and none for user B's.
"""
user_b = self._setup_user(f"{uuid4()}@galaxy.test")
_, user_b_api_key = self._setup_user_get_key(user_b["email"])
user_b_id = self._user_id_for_api_key(user_b_api_key)
user_a_api_key = self.galaxy_interactor.api_key
assert user_a_api_key is not None
user_a_id = self._user_id_for_api_key(user_a_api_key)
job_a_id, _ = self._create_it_job_with_entry_point(user_a_id, tool_port=7001)
job_b_id, _ = self._create_it_job_with_entry_point(user_b_id, tool_port=7002)
listener = SSELineListener(self._events_stream_url(), user_a_api_key)
listener.start()
try:
sa_session = self._app.model.context
job_b = sa_session.get(Job, job_b_id)
assert job_b is not None
# User B's job — user A must NOT see this.
self._app.interactivetool_manager.configure_entry_points(job_b, _make_ports_dict(7002))
# Give the broker a moment so a leaked event (if any) would arrive
# before we fire user A's event — the assertion would then catch
# more than one event on user A's stream.
time.sleep(0.5)
job_a = sa_session.get(Job, job_a_id)
assert job_a is not None
# User A's own job — this is what A's stream must observe.
self._app.interactivetool_manager.configure_entry_points(job_a, _make_ports_dict(7001))
entry_point_events = listener.wait_for_event("entry_point_update")
finally:
listener.stop()
assert (
len(entry_point_events) == 1
), f"User A expected exactly one entry_point_update (own job); saw {len(entry_point_events)}: {entry_point_events}"
+104
View File
@@ -0,0 +1,104 @@
"""Integration tests for SSE-based history update notifications."""
import json
from urllib.parse import urljoin
from uuid import uuid4
import requests
from galaxy_test.base.populators import DatasetPopulator
from galaxy_test.base.sse import SSELineListener
from galaxy_test.driver.integration_util import IntegrationTestCase
class TestHistorySSEIntegration(IntegrationTestCase):
dataset_populator: DatasetPopulator
framework_tool_and_types = True
@classmethod
def handle_galaxy_config_kwds(cls, config):
super().handle_galaxy_config_kwds(config)
config["enable_celery_tasks"] = False
config["enable_sse_updates"] = True
def setUp(self):
super().setUp()
self.dataset_populator = DatasetPopulator(self.galaxy_interactor)
def _events_stream_url(self) -> str:
return urljoin(self.url, "api/events/stream")
def _create_history(self, name=None) -> str:
name = name or f"test_history_{uuid4()}"
response = self._post("histories", data={"name": name}, json=True)
self._assert_status_code_is_ok(response)
return response.json()["id"]
def test_history_update_contains_current_history_id(self):
"""The history_update event should contain the history's encoded ID."""
history_id = self._create_history()
api_key = self.galaxy_interactor.api_key
assert api_key is not None
listener = SSELineListener(self._events_stream_url(), api_key)
listener.start()
try:
self.dataset_populator.new_dataset(history_id, wait=False)
history_events = listener.wait_for_event_where(
"history_update",
lambda e: history_id in json.loads(e["data"]).get("history_ids", []),
)
found = any(history_id in json.loads(e["data"]).get("history_ids", []) for e in history_events)
assert found, f"Expected history_id '{history_id}' in history_update events, got: {history_events}"
finally:
listener.stop()
def test_history_update_is_scoped_to_owning_user(self):
"""User A must only see history_update events for their own histories.
Inverted positive assertion: after user B's upload, user A uploads to
their own history and we assert A's stream contains A's encoded id and
not B's. This avoids a sleep-based "no events" test that was prone to
flaking under slow CI.
"""
user_b = self._setup_user(f"{uuid4()}@galaxy.test")
_, user_b_api_key = self._setup_user_get_key(user_b["email"])
user_a_history_id = self._create_history()
api_key = self.galaxy_interactor.api_key
assert api_key is not None
listener = SSELineListener(self._events_stream_url(), api_key)
listener.start()
try:
# User B creates a history and uploads to it. User A must NOT see this.
create_resp = requests.post(
urljoin(self.url, "api/histories"),
params={"key": user_b_api_key},
json={"name": "User B History"},
)
assert create_resp.status_code == 200
user_b_history_id = create_resp.json()["id"]
requests.post(
urljoin(self.url, f"api/histories/{user_b_history_id}/contents"),
params={"key": user_b_api_key},
json={"from_hda_id": None, "source": "pasted", "content": "user b content"},
)
# User A uploads to their own history — this is what A's stream must observe.
self.dataset_populator.new_dataset(user_a_history_id, wait=False)
history_events = listener.wait_for_event_where(
"history_update",
lambda e: user_a_history_id in json.loads(e["data"]).get("history_ids", []),
)
finally:
listener.stop()
seen_ids: set[str] = set()
for event in history_events:
seen_ids.update(json.loads(event["data"]).get("history_ids", []))
assert user_a_history_id in seen_ids, f"User A missed its own history_update: {history_events}"
assert (
user_b_history_id not in seen_ids
), f"User A received history_update for user B's history ({user_b_history_id}): {history_events}"
+173
View File
@@ -0,0 +1,173 @@
"""Integration tests for the notification SSE (Server-Sent Events) endpoint."""
import json
from typing import Optional
from urllib.parse import urljoin
from uuid import uuid4
from galaxy_test.base.populators import DatasetPopulator
from galaxy_test.base.sse import SSELineListener
from galaxy_test.driver.integration_util import IntegrationTestCase
def notification_test_data(subject: Optional[str] = None, message: Optional[str] = None) -> dict:
return {
"source": "integration_tests",
"variant": "info",
"category": "message",
"content": {
"category": "message",
"subject": subject or "Testing Subject",
"message": message or "Testing Message",
},
}
def notification_broadcast_test_data(subject: Optional[str] = None, message: Optional[str] = None) -> dict:
return {
"source": "integration_tests",
"variant": "info",
"category": "broadcast",
"content": {
"category": "broadcast",
"subject": subject or "Testing Broadcast Subject",
"message": message or "Testing Broadcast Message",
},
}
def _notification_subjects(events: list[dict]) -> list[str]:
"""Extract ``content.subject`` from each SSE ``data`` payload.
Verifies JSON shape rather than substring-matching raw ``data`` strings — a
regression in the envelope (missing id, wrong serializer, content key
renamed) fails here instead of silently passing. Each ``data`` payload is
a ``NotificationResponse`` dump with a top-level ``content.subject``.
"""
subjects = []
for event in events:
payload = json.loads(event["data"])
subjects.append(payload["content"]["subject"])
return subjects
class TestNotificationSSEIntegration(IntegrationTestCase):
dataset_populator: DatasetPopulator
framework_tool_and_types = False
@classmethod
def handle_galaxy_config_kwds(cls, config):
super().handle_galaxy_config_kwds(config)
config["enable_celery_tasks"] = False
config["enable_notification_system"] = True
def setUp(self):
super().setUp()
self.dataset_populator = DatasetPopulator(self.galaxy_interactor)
def _stream_url(self) -> str:
return urljoin(self.url, "api/events/stream")
def test_sse_receives_notification_events(self):
"""When a notification is created, the SSE stream should receive it."""
user = self._setup_user(f"{uuid4()}@galaxy.test")
_, user_api_key = self._setup_user_get_key(user["email"])
listener = SSELineListener(self._stream_url(), user_api_key)
listener.start()
try:
subject = f"sse_test_{uuid4()}"
request = {
"recipients": {"user_ids": [user["id"]]},
"notification": notification_test_data(subject=subject, message="SSE test notification"),
}
response = self._post("notifications", data=request, admin=True, json=True)
self._assert_status_code_is_ok(response)
notification_events = listener.wait_for_event("notification_update")
finally:
listener.stop()
assert subject in _notification_subjects(
notification_events
), f"Expected subject '{subject}' in SSE events, got: {notification_events}"
def test_sse_receives_broadcast_events(self):
"""When a broadcast is created, the SSE stream should receive it."""
api_key = self.galaxy_interactor.api_key
assert api_key is not None
listener = SSELineListener(self._stream_url(), api_key)
listener.start()
try:
subject = f"broadcast_sse_test_{uuid4()}"
payload = notification_broadcast_test_data(subject=subject)
response = self._post("notifications/broadcast", data=payload, admin=True, json=True)
self._assert_status_code_is_ok(response)
broadcast_events = listener.wait_for_event("broadcast_update")
finally:
listener.stop()
# Broadcast events carry a BroadcastNotificationResponse, which shares
# the top-level content.subject shape with per-user notifications.
broadcast_subjects = [json.loads(e["data"])["content"]["subject"] for e in broadcast_events]
assert subject in broadcast_subjects, f"Expected subject '{subject}' in broadcast events: {broadcast_events}"
def test_sse_catchup_on_reconnect(self):
"""Reconnecting with Last-Event-ID should replay a catch-up notification_status event.
The ``Last-Event-ID`` value is the server-issued ID from a prior event,
not a client-side timestamp. This avoids clock-skew flake between the
test runner and the app in containerized CI.
"""
user = self._setup_user(f"{uuid4()}@galaxy.test")
_, user_api_key = self._setup_user_get_key(user["email"])
# First connection: capture the server-issued event id of the first notification.
listener_1 = SSELineListener(self._stream_url(), user_api_key)
listener_1.start()
try:
subject_1 = f"first_{uuid4()}"
request = {
"recipients": {"user_ids": [user["id"]]},
"notification": notification_test_data(subject=subject_1),
}
response = self._post("notifications", data=request, admin=True, json=True)
self._assert_status_code_is_ok(response)
first_events = listener_1.wait_for_event("notification_update")
finally:
listener_1.stop()
last_event_id = next((e["id"] for e in first_events if e.get("id")), None)
assert last_event_id, f"No server-issued id on first notification event: {first_events}"
# Emit a second notification while disconnected; it should appear in the catch-up.
subject_2 = f"catchup_{uuid4()}"
request = {
"recipients": {"user_ids": [user["id"]]},
"notification": notification_test_data(subject=subject_2, message="Catch-up test"),
}
response = self._post("notifications", data=request, admin=True, json=True)
self._assert_status_code_is_ok(response)
# Reconnect with Last-Event-ID = the captured id. The catch-up must include
# the notification sent after that id but not the one that produced it.
listener_2 = SSELineListener(
self._stream_url(),
user_api_key,
headers={"Last-Event-ID": last_event_id},
)
listener_2.start()
try:
status_events = listener_2.wait_for_event("notification_status")
finally:
listener_2.stop()
replayed_subjects: list[str] = []
for event in status_events:
payload = json.loads(event["data"])
replayed_subjects.extend(n["content"]["subject"] for n in payload.get("notifications", []))
assert subject_2 in replayed_subjects, f"Missed catch-up of '{subject_2}': {status_events}"
assert (
subject_1 not in replayed_subjects
), f"Last-Event-ID did not filter — '{subject_1}' replayed: {status_events}"
@@ -0,0 +1,113 @@
"""Playwright E2E test for the interactive-tool entry-point SSE pipeline.
Verifies that when an interactive-tool entry point transitions to ``configured``
server-side (the runner's port-routing hook calls ``configure_entry_points``),
a logged-in user's browser receives the ``entry_point_update`` SSE event and
the entry-point store refetches, without the 10 s polling interval.
This test stubs the server-side runner callback: it creates the Job and entry
point directly and calls ``InteractiveToolManager.configure_entry_points`` on
the live app. That invocation is the exact dispatch site in production.
"""
from uuid import uuid4
from galaxy.model import (
InteractiveToolEntryPoint,
Job,
)
from galaxy.util.wait import wait_on
from galaxy_test.selenium.framework import (
managed_history,
selenium_test,
)
from .framework import SeleniumIntegrationTestCase
SSE_CONNECT_TIMEOUT_SECONDS = 15
SSE_EVENT_TIMEOUT_SECONDS = 15
class TestEntryPointSSESeleniumIntegration(SeleniumIntegrationTestCase):
ensure_registered = True
@classmethod
def handle_galaxy_config_kwds(cls, config):
super().handle_galaxy_config_kwds(config)
config["enable_celery_tasks"] = False
# App.vue only calls entryPointStore.startWatchingEntryPoints() when
# interactivetools_enable is True, and the store only opens an SSE
# connection when enable_sse_updates is True. Without both,
# __galaxy_sse_connected never becomes true and the gate below times out.
config["interactivetools_enable"] = True
config["enable_sse_updates"] = True
def _wait_for_sse_connected(self) -> None:
"""Block until the frontend confirms the SSE pipeline is live."""
wait_on(
lambda: True if self.execute_script("return window.__galaxy_sse_connected === true") else None,
"window.__galaxy_sse_connected === true",
timeout=SSE_CONNECT_TIMEOUT_SECONDS,
)
def _last_sse_event_ts(self) -> int:
return self.execute_script("return window.__galaxy_sse_last_event_ts || 0") or 0
def _wait_for_sse_event_after(self, baseline_ts: int) -> None:
wait_on(
lambda: True if self._last_sse_event_ts() > baseline_ts else None,
"window.__galaxy_sse_last_event_ts advanced past baseline",
timeout=SSE_EVENT_TIMEOUT_SECONDS,
)
def _create_it_job_with_entry_point(self, tool_port: int = 8888) -> tuple[int, int]:
# Use the browser's cookie-authenticated user, not the API interactor's
# default: SSE connects under the Selenium-registered user, and the
# dispatch's user_id must match or push_to_user finds no queues.
user_info = self.api_get("users/current")
user_id = self._decode_id(user_info["id"])
sa_session = self._app.model.context
job = Job()
job.user_id = user_id
job.tool_id = "interactivetool_simple"
job.state = Job.states.RUNNING
sa_session.add(job)
sa_session.flush()
ep = InteractiveToolEntryPoint(
job=job,
tool_port=tool_port,
entry_url="/",
name=f"selenium entry {uuid4()}",
label="selenium",
requires_domain=True,
requires_path_in_url=False,
requires_path_in_header_named=None,
)
sa_session.add(ep)
sa_session.commit()
return job.id, ep.id
@selenium_test
@managed_history
def test_entry_point_update_pushed_via_sse(self):
"""configure_entry_points should trigger an SSE push the client observes."""
# Navigate home so the entry-point store is mounted and subscribed.
self.home()
self._wait_for_sse_connected()
baseline_ts = self._last_sse_event_ts()
self.screenshot("entry_point_sse_before")
job_id, _ep_id = self._create_it_job_with_entry_point(tool_port=8888)
# Stub the runner hook: call configure_entry_points on the live app.
sa_session = self._app.model.context
job = sa_session.get(Job, job_id)
assert job is not None
self._app.interactivetool_manager.configure_entry_points(
job,
{"8888": {"host": "host.invalid", "port": 12345, "protocol": "http"}},
)
# Prove the update arrived via SSE (not polling): the composable's
# event-timestamp hook only advances when useSSE's listener fires.
self._wait_for_sse_event_after(baseline_ts)
self.screenshot("entry_point_sse_after")
@@ -0,0 +1,145 @@
"""Playwright E2E test for the notification SSE pipeline.
Verifies that when an admin creates a notification via the API,
a logged-in user sees it appear in the UI in real-time (within seconds)
without a page refresh, proving the SSE push pipeline works end-to-end.
"""
from uuid import uuid4
from galaxy.util.wait import wait_on
from galaxy_test.selenium.framework import (
managed_history,
selenium_test,
)
from .framework import SeleniumIntegrationTestCase
SSE_CONNECT_TIMEOUT_SECONDS = 15
SSE_EVENT_TIMEOUT_SECONDS = 15
class TestNotificationSSESeleniumIntegration(SeleniumIntegrationTestCase):
ensure_registered = True
@classmethod
def handle_galaxy_config_kwds(cls, config):
super().handle_galaxy_config_kwds(config)
config["enable_notification_system"] = True
config["enable_sse_updates"] = True
config["enable_celery_tasks"] = False
def _wait_for_sse_connected(self) -> None:
"""Block until the frontend confirms the SSE pipeline is live.
Without this gate, the 30 s polling fallback silently masks a broken
SSE pipeline — the UI would still update, but via polling, and the
test would falsely pass.
"""
wait_on(
lambda: True if self.execute_script("return window.__galaxy_sse_connected === true") else None,
"window.__galaxy_sse_connected === true",
timeout=SSE_CONNECT_TIMEOUT_SECONDS,
)
def _last_sse_event_ts(self) -> int:
"""Return the last SSE event timestamp recorded by the composable, or 0."""
return self.execute_script("return window.__galaxy_sse_last_event_ts || 0") or 0
def _wait_for_sse_event_after(self, baseline_ts: int) -> None:
"""Block until an SSE event arrives after ``baseline_ts``.
Guards against a silent regression where the UI update originates from
the polling fallback rather than the SSE push: ``__galaxy_sse_last_event_ts``
only advances when the composable's event listener fires.
"""
wait_on(
lambda: True if self._last_sse_event_ts() > baseline_ts else None,
"window.__galaxy_sse_last_event_ts advanced past baseline",
timeout=SSE_EVENT_TIMEOUT_SECONDS,
)
@selenium_test
@managed_history
def test_notification_appears_via_sse(self):
"""Send a notification via the API and verify it appears in the UI without refresh."""
# Get the browser-logged-in user's info via the browser's cookie. ``self._get``
# uses the API interactor's default-user key, which does not match the Selenium
# user created by ``ensure_registered``, so the SSE push would target a
# different user than the one watching the stream.
user_info = self.api_get("users/current")
user_id = user_info["id"]
# Navigate to notifications page so the store is watching.
# ``get()`` uses ``build_url()`` which handles trailing slashes on the
# base correctly; concatenating against ``target_url_from_selenium``
# can produce a double-slash that Galaxy routes differently.
self.get("user/notifications")
self._wait_for_sse_connected()
baseline_ts = self._last_sse_event_ts()
self.screenshot("notification_sse_before")
# Send a notification to this user via the admin API
subject = f"SSE E2E Test {uuid4()}"
notification_request = {
"recipients": {"user_ids": [user_id]},
"notification": {
"source": "integration_tests",
"variant": "info",
"category": "message",
"content": {
"category": "message",
"subject": subject,
"message": "This notification was pushed via SSE",
},
},
}
response = self._post("notifications", data=notification_request, admin=True, json=True)
self._assert_status_code_is_ok(response)
# Prove the incoming update arrived via SSE: the event-timestamp hook
# only advances when useSSE's listener fires. If this times out while
# the UI still shows the notification, polling picked it up — a silent
# regression this assertion catches.
self._wait_for_sse_event_after(baseline_ts)
# Use xpath (works on both Selenium and Playwright backends); `text=` is a
# Playwright engine prefix that Selenium's css-selector path rejects.
self.wait_for_xpath_visible(f'//*[contains(text(), "{subject}")]', timeout=SSE_EVENT_TIMEOUT_SECONDS)
self.screenshot("notification_sse_after")
@selenium_test
@managed_history
def test_notification_bell_updates_via_sse(self):
"""The notification bell indicator should update when a new notification arrives via SSE."""
# See ``test_notification_appears_via_sse`` — must use the browser's user,
# not the API interactor's default user.
user_info = self.api_get("users/current")
user_id = user_info["id"]
# Go to home page (bell is in masthead)
self.home()
self._wait_for_sse_connected()
baseline_ts = self._last_sse_event_ts()
# Send a notification
subject = f"Bell Test {uuid4()}"
notification_request = {
"recipients": {"user_ids": [user_id]},
"notification": {
"source": "integration_tests",
"variant": "info",
"category": "message",
"content": {
"category": "message",
"subject": subject,
"message": "Testing bell indicator update via SSE",
},
},
}
response = self._post("notifications", data=notification_request, admin=True, json=True)
self._assert_status_code_is_ok(response)
self._wait_for_sse_event_after(baseline_ts)
# The activity-bar notifications item renders its unread-count badge as
# ``.nav-indicator`` (see ``ActivityItem.vue``) once ``totalUnreadCount > 0``.
self.wait_for_selector_visible("#activity-notifications .nav-indicator", timeout=SSE_EVENT_TIMEOUT_SECONDS)
self.screenshot("notification_bell_indicator")
@@ -0,0 +1,227 @@
"""Unit tests for :mod:`galaxy.managers.queue_metrics`.
The ``SSEConnectionManager`` and the kombu connection are small hand-built
fakes — no broker or database is required. Assertions are on the recorded
state of the statsd client (counters, timings) rather than on mock call-lists
so a regression that stops emitting a gauge fails the test for the right
reason.
The failure-isolation test drives real sub-emitters into their error paths by
handing them genuinely broken collaborators (a connection whose ``clone()``
raises, a model whose ``new_session()`` raises). That way the test exercises
the real ``_run`` wrapper rather than asserting a monkey-patched side_effect.
"""
from dataclasses import (
dataclass,
field,
)
from types import SimpleNamespace
from typing import (
cast,
Optional,
)
from unittest.mock import MagicMock
import pytest
from galaxy.managers import queue_metrics
from galaxy.managers.sse import SSEConnectionManager
from galaxy.model.mapping import GalaxyModelMapping
from galaxy.web.statsd_client import VanillaGalaxyStatsdClient
from galaxy.web_stack import ApplicationStack
# ---------------------------------------------------------------------------
# Fakes
# ---------------------------------------------------------------------------
@dataclass
class FakeStatsdClient:
"""In-memory statsd recorder — see docstring in ``test_sse_dispatch.py``."""
counters: dict[tuple[str, tuple[tuple[str, str], ...]], int] = field(default_factory=dict)
timings: list[tuple[str, float, tuple[tuple[str, str], ...]]] = field(default_factory=list)
def incr(self, metric: str, tags: Optional[dict[str, str]] = None) -> None:
key = (metric, tuple(sorted((tags or {}).items())))
self.counters[key] = self.counters.get(key, 0) + 1
def timing(self, metric: str, value: float, tags: Optional[dict[str, str]] = None) -> None:
self.timings.append((metric, value, tuple(sorted((tags or {}).items()))))
def counter(self, metric: str, tags: Optional[dict[str, str]] = None) -> int:
return self.counters.get((metric, tuple(sorted((tags or {}).items()))), 0)
def timings_for(self, metric: str) -> list[tuple[float, dict[str, str]]]:
return [(v, dict(t)) for m, v, t in self.timings if m == metric]
def _fake_sse_manager(broadcast: int = 3, per_user: int = 5):
m = MagicMock(spec=SSEConnectionManager)
m.total_broadcast_connections = broadcast
m.total_per_user_connections = per_user
return m
def _make_fake_queue(name: str, count: int):
"""Fake kombu Queue exposing ``.bind(channel).queue_declare(passive=True).message_count``."""
declared = SimpleNamespace(message_count=count)
bound = SimpleNamespace(queue_declare=lambda passive: declared)
return SimpleNamespace(name=name, bind=lambda channel: bound)
def _make_fake_connection(channel=None):
"""Fake kombu Connection whose ``.clone()`` is usable as a context manager."""
channel = channel or MagicMock()
conn_cm = MagicMock()
conn_cm.channel.return_value = channel
cm = MagicMock()
cm.__enter__ = lambda self: conn_cm
cm.__exit__ = lambda self, *a: False
connection = MagicMock()
connection.clone.return_value = cm
return connection
# ---------------------------------------------------------------------------
# Sub-emitter tests — assert on recorded state
# ---------------------------------------------------------------------------
def test_emit_sse_connection_gauges_emits_both_kinds():
statsd = FakeStatsdClient()
queue_metrics.emit_sse_connection_gauges(
cast(VanillaGalaxyStatsdClient, statsd), _fake_sse_manager(broadcast=4, per_user=7)
)
assert statsd.timings_for("galaxy.sse.connections.active") == [
(4, {"kind": "broadcast"}),
(7, {"kind": "per_user"}),
]
def test_emit_control_queue_depth_emits_per_queue(monkeypatch):
statsd = FakeStatsdClient()
fake_queues = [
_make_fake_queue("control.main@h", 3),
_make_fake_queue("control.main.1@h", 0),
]
monkeypatch.setattr(
queue_metrics,
"all_control_queues_for_declare",
lambda application_stack: fake_queues,
)
queue_metrics.emit_control_queue_depth(
cast(VanillaGalaxyStatsdClient, statsd),
_make_fake_connection(),
cast(ApplicationStack, MagicMock()),
)
assert statsd.timings_for("galaxy.control_queue.depth") == [
(3, {"queue_name": "control.main@h"}),
(0, {"queue_name": "control.main.1@h"}),
]
def test_emit_control_queue_depth_skips_failed_passive_declare(monkeypatch):
"""One bad queue → we skip it and keep going for the rest."""
statsd = FakeStatsdClient()
def bad_declare(passive):
raise RuntimeError("queue does not exist yet")
good_queue = _make_fake_queue("control.good@h", 9)
bad_queue = SimpleNamespace(
name="control.bad@h",
bind=lambda channel: SimpleNamespace(queue_declare=bad_declare),
)
monkeypatch.setattr(queue_metrics, "all_control_queues_for_declare", lambda stack: [good_queue, bad_queue])
queue_metrics.emit_control_queue_depth(
cast(VanillaGalaxyStatsdClient, statsd),
_make_fake_connection(),
cast(ApplicationStack, MagicMock()),
)
assert statsd.timings_for("galaxy.control_queue.depth") == [
(9, {"queue_name": "control.good@h"}),
]
def test_emit_control_queue_depth_no_broker_connection_is_noop():
statsd = FakeStatsdClient()
queue_metrics.emit_control_queue_depth(
cast(VanillaGalaxyStatsdClient, statsd), None, cast(ApplicationStack, MagicMock())
)
assert statsd.timings == []
# ---------------------------------------------------------------------------
# emit_queue_metrics — aggregate entry-point
# ---------------------------------------------------------------------------
def test_emit_queue_metrics_is_silent_when_statsd_is_none():
"""No statsd client → every sub-call is skipped, no DB or broker access."""
# Would raise AttributeError if the short-circuit didn't fire before any
# real collaborator was touched.
queue_metrics.emit_queue_metrics(
statsd_client=None,
connection=None,
application_stack=cast(ApplicationStack, MagicMock()),
model=cast(GalaxyModelMapping, MagicMock()),
sse_manager=None,
)
def test_emit_queue_metrics_isolates_real_subemitter_failures(monkeypatch):
"""When real sub-emitters raise, ``_run`` contains the failure and logs an error counter.
We drive the failures through the actual sub-emitter bodies — not monkey-
patched side_effects — by handing in a connection whose ``.clone()`` raises
(for ``emit_control_queue_depth``) and a model whose ``.new_session()``
raises (for ``emit_worker_process_gauge``). The SSE gauge is given healthy
collaborators and should still land.
"""
statsd = FakeStatsdClient()
sse_manager = _fake_sse_manager(broadcast=2, per_user=1)
broken_connection = MagicMock()
broken_connection.clone.side_effect = RuntimeError("broker is gone")
broken_model = MagicMock()
broken_model.new_session.side_effect = RuntimeError("db is gone")
# Ensure the broker path reaches .clone() rather than short-circuiting on
# an empty queue list.
monkeypatch.setattr(
queue_metrics,
"all_control_queues_for_declare",
lambda stack: [_make_fake_queue("control.main@h", 0)],
)
# Must not raise — the SSE gauge still lands.
queue_metrics.emit_queue_metrics(
statsd_client=cast(VanillaGalaxyStatsdClient, statsd),
connection=broken_connection,
application_stack=cast(ApplicationStack, MagicMock()),
model=cast(GalaxyModelMapping, broken_model),
sse_manager=sse_manager,
)
# SSE gauge landed despite the other two failing.
sse_timings = statsd.timings_for("galaxy.sse.connections.active")
assert (2, {"kind": "broadcast"}) in sse_timings
assert (1, {"kind": "per_user"}) in sse_timings
# Each failing sub-emitter bumped its error counter tagged by name.
assert statsd.counter("galaxy.queue_metrics.error", {"emitter": "control_queue_depth"}) == 1
assert statsd.counter("galaxy.queue_metrics.error", {"emitter": "worker_process"}) == 1
# The healthy SSE sub-emitter did not.
assert statsd.counter("galaxy.queue_metrics.error", {"emitter": "sse_connections"}) == 0
if __name__ == "__main__":
pytest.main([__file__, "-v"])
+236
View File
@@ -0,0 +1,236 @@
"""Unit tests for :class:`galaxy.managers.sse_dispatch.SSEEventDispatcher` observability.
Focus is on the statsd instrumentation contract AND the effect of dispatch:
counters/timers fire on the happy path and the ``_queue_worker is None``
early-return, payloads reach the broker with the expected task+kwargs, and the
dispatcher is a silent no-op when ``statsd_client`` is ``None``.
These tests use lightweight fakes (``FakeStatsdClient``, ``FakeControlTask``)
that record state we can assert against — rather than ``MagicMock`` call-lists —
so a regression that silently drops dispatch or stops recording metrics fails
the test for the right reason.
"""
from dataclasses import (
dataclass,
field,
)
from typing import (
Any,
Optional,
)
from unittest.mock import MagicMock
import pytest
from galaxy.managers.sse_dispatch import SSEEventDispatcher
@dataclass
class FakeStatsdClient:
"""In-memory stand-in for ``VanillaGalaxyStatsdClient``.
Records ``incr`` and ``timing`` calls as plain data so tests assert on
observable state (counter totals, recorded timings) instead of mock
call-lists.
"""
counters: dict[tuple[str, tuple[tuple[str, str], ...]], int] = field(default_factory=dict)
timings: list[tuple[str, float, tuple[tuple[str, str], ...]]] = field(default_factory=list)
def incr(self, metric: str, tags: Optional[dict[str, str]] = None) -> None:
key = (metric, tuple(sorted((tags or {}).items())))
self.counters[key] = self.counters.get(key, 0) + 1
def timing(self, metric: str, value: float, tags: Optional[dict[str, str]] = None) -> None:
self.timings.append((metric, value, tuple(sorted((tags or {}).items()))))
def counter(self, metric: str, tags: Optional[dict[str, str]] = None) -> int:
return self.counters.get((metric, tuple(sorted((tags or {}).items()))), 0)
@dataclass
class RecordedTask:
payload: dict[str, Any]
routing_key: str
expiration: Optional[int]
declare_queues: Any
class FakeControlTask:
"""Stand-in for ``ControlTask`` that records dispatches instead of touching AMQP."""
instances: list["FakeControlTask"] = []
def __init__(self, queue_worker) -> None:
self.queue_worker = queue_worker
self.sent: list[RecordedTask] = []
FakeControlTask.instances.append(self)
def send_task(
self,
payload: dict[str, Any],
routing_key: str,
expiration: Optional[int] = None,
declare_queues: Any = None,
**_: Any,
) -> None:
self.sent.append(
RecordedTask(
payload=payload,
routing_key=routing_key,
expiration=expiration,
declare_queues=declare_queues,
)
)
class BoomControlTask:
"""``ControlTask`` fake whose ``send_task`` always raises — exercises the finally block."""
def __init__(self, queue_worker) -> None:
self.queue_worker = queue_worker
def send_task(self, **kwargs) -> None:
raise RuntimeError("broker down")
@pytest.fixture
def application_stack():
return MagicMock(name="ApplicationStack")
@pytest.fixture
def queue_worker():
return MagicMock(name="GalaxyQueueWorker")
@pytest.fixture
def statsd() -> FakeStatsdClient:
return FakeStatsdClient()
@pytest.fixture(autouse=True)
def _reset_fake_control_task_instances():
FakeControlTask.instances.clear()
yield
FakeControlTask.instances.clear()
@pytest.fixture
def fake_control_task() -> type[FakeControlTask]:
"""Returns the fake class so tests can pass it as ``control_task_factory``."""
return FakeControlTask
def _dispatcher_with_fakes(
queue_worker,
application_stack,
statsd,
control_task_factory=FakeControlTask,
queues=None,
) -> SSEEventDispatcher:
"""Build a dispatcher wired to injected fakes — no monkeypatching required."""
return SSEEventDispatcher(
queue_worker=queue_worker,
application_stack=application_stack,
statsd_client=statsd,
control_task_factory=control_task_factory,
queues_provider=lambda: queues if queues is not None else [],
)
def test_dispatcher_no_op_when_queue_worker_is_none_and_no_statsd(application_stack):
"""No statsd client set and no queue_worker → silent no-op, no AttributeError."""
dispatcher = SSEEventDispatcher(
queue_worker=None,
application_stack=application_stack,
statsd_client=None,
)
# Must not raise, must not attempt to declare queues.
dispatcher.notify_users([1, 2], "hello")
dispatcher.notify_broadcast("hi")
dispatcher.history_update({"1": [42]})
def test_dispatcher_records_skipped_counter_when_queue_worker_is_none(application_stack, statsd):
"""Two dispatches with no queue_worker → two skipped_no_qw increments, no timings."""
dispatcher = SSEEventDispatcher(
queue_worker=None,
application_stack=application_stack,
statsd_client=statsd,
)
dispatcher.notify_users([1], "hello")
dispatcher.notify_broadcast("world")
assert statsd.counter("galaxy.sse.dispatch.skipped_no_qw") == 2
# No latency timing — we never got as far as the broker call.
assert statsd.timings == []
def test_dispatcher_enqueues_payload_and_records_metrics_on_send(
application_stack, queue_worker, statsd, fake_control_task
):
"""Happy path: payload reaches the broker AND counter+timer are recorded."""
dispatcher = _dispatcher_with_fakes(
queue_worker=queue_worker,
application_stack=application_stack,
statsd=statsd,
control_task_factory=fake_control_task,
)
dispatcher.notify_users([1, 2], "hello")
# Exactly one ControlTask constructed, one send_task recorded with the right
# payload — asserts the dispatch *effect*, not just that a mock was called.
assert len(fake_control_task.instances) == 1
sent = fake_control_task.instances[0].sent
assert len(sent) == 1
assert sent[0].payload["task"] == "notify_users"
assert sent[0].payload["kwargs"]["user_ids"] == [1, 2]
assert sent[0].payload["kwargs"]["payload"] == "hello"
assert "event_id" in sent[0].payload["kwargs"]
assert sent[0].routing_key == "control.*"
assert sent[0].expiration == 10
# Counter + timer both recorded with matching task tag.
assert statsd.counter("galaxy.sse.dispatch.count", {"task": "notify_users"}) == 1
assert len(statsd.timings) == 1
metric, _value, tags = statsd.timings[0]
assert metric == "galaxy.sse.dispatch.latency_ms"
assert dict(tags) == {"task": "notify_users"}
def test_dispatcher_timer_still_fires_on_send_exception(application_stack, queue_worker, statsd):
"""Timer lives in ``finally`` — broker errors don't mask the latency metric."""
dispatcher = _dispatcher_with_fakes(
queue_worker=queue_worker,
application_stack=application_stack,
statsd=statsd,
control_task_factory=BoomControlTask,
)
with pytest.raises(RuntimeError):
dispatcher.history_update({"7": [1]})
assert statsd.counter("galaxy.sse.dispatch.count", {"task": "history_update"}) == 1
assert len(statsd.timings) == 1
metric, _value, tags = statsd.timings[0]
assert metric == "galaxy.sse.dispatch.latency_ms"
assert dict(tags) == {"task": "history_update"}
def test_dispatcher_no_statsd_means_no_instrumentation(application_stack, queue_worker, fake_control_task):
"""When ``statsd_client`` is ``None`` instrumentation is bypassed entirely.
The dispatch still happens — we assert via the ControlTask fake — but there
is nothing to observe on the (absent) statsd side.
"""
dispatcher = _dispatcher_with_fakes(
queue_worker=queue_worker,
application_stack=application_stack,
statsd=None,
control_task_factory=fake_control_task,
)
dispatcher.notify_broadcast("hi")
assert len(fake_control_task.instances) == 1
assert len(fake_control_task.instances[0].sent) == 1
assert fake_control_task.instances[0].sent[0].payload["task"] == "notify_broadcast"
@@ -0,0 +1,144 @@
"""Tests for the TTL cache on ``SSEEventDispatcher._get_declare_queues``.
The cache exists because ``_send`` is on a hot path (1000+ events/s at target
load) and without it each dispatch fires a ``WorkerProcess`` DB query. The
underlying data only changes on a 60 s heartbeat cadence, so a 30 s TTL is safe.
"""
from concurrent.futures import (
as_completed,
ThreadPoolExecutor,
)
from typing import Any
from unittest.mock import MagicMock
import pytest
from galaxy.managers.sse_dispatch import SSEEventDispatcher
@pytest.fixture
def fake_declare():
"""Call-counting provider passed to ``SSEEventDispatcher`` via DI.
Returns a non-empty list by default so the cache stores a value — empty
results are intentionally not cached (see ``test_empty_result_not_cached``).
"""
calls: dict[str, Any] = {"count": 0, "returns": [MagicMock(name="queue")]}
def _provider():
calls["count"] += 1
return calls["returns"]
calls["provider"] = _provider
return calls
class FakeClock:
"""Controllable time source passed to ``SSEEventDispatcher``.
Lets tests advance the dispatcher's TTL cache deterministically without
reaching into ``cachetools`` internals.
"""
def __init__(self, start: float = 0.0) -> None:
self.now = start
def __call__(self) -> float:
return self.now
def advance(self, seconds: float) -> None:
self.now += seconds
@pytest.fixture
def clock() -> FakeClock:
return FakeClock()
@pytest.fixture
def dispatcher(clock, fake_declare):
"""Build a dispatcher with a stub queue_worker, a counting queues_provider,
and a no-op ``ControlTask`` factory — no monkey-patching required.
"""
queue_worker = MagicMock(name="queue_worker")
application_stack = MagicMock(name="application_stack")
class NoopControlTask:
def __init__(self, qw):
self.qw = qw
def send_task(self, **kwargs):
pass
return SSEEventDispatcher(
queue_worker=queue_worker,
application_stack=application_stack,
clock=clock,
control_task_factory=NoopControlTask,
queues_provider=fake_declare["provider"],
)
def test_declare_queues_cached_within_ttl(dispatcher, fake_declare):
"""Repeated dispatches inside the TTL window only hit the DB once."""
for _ in range(10):
dispatcher.notify_broadcast("payload")
assert fake_declare["count"] == 1
def test_declare_queues_refetched_after_ttl(dispatcher, fake_declare, clock):
"""Once the TTL expires, the next call refetches exactly once."""
dispatcher.notify_broadcast("payload")
assert fake_declare["count"] == 1
# Advance the injected clock past the TTL so the cache sees the entry as
# expired on the next read.
clock.advance(dispatcher._DECLARE_QUEUES_TTL_SECONDS + 1)
dispatcher.notify_broadcast("payload")
assert fake_declare["count"] == 2
# Further dispatches at the advanced time reuse the newly populated entry.
dispatcher.notify_broadcast("payload")
assert fake_declare["count"] == 2
def test_empty_result_not_cached(dispatcher, fake_declare):
"""An empty list must not be pinned in the cache for the full TTL.
Empty results arise during startup (before ``DatabaseHeartbeat`` writes the
row) and on swallowed DB errors. Caching them would silently drop every SSE
event until the next TTL expiry.
"""
fake_declare["returns"] = []
dispatcher.notify_broadcast("payload")
dispatcher.notify_broadcast("payload")
assert fake_declare["count"] == 2
# Once the upstream starts returning a non-empty result, caching resumes.
fake_declare["returns"] = [MagicMock(name="queue")]
dispatcher.notify_broadcast("payload")
dispatcher.notify_broadcast("payload")
assert fake_declare["count"] == 3
def test_declare_queues_thread_safe_single_query_under_load(dispatcher, fake_declare):
"""Concurrent ``_send`` from many threads still only triggers one DB query.
With stampede protection (RLock around the miss) all 500 dispatches should
collapse to a single ``all_control_queues_for_declare`` call inside one TTL
window. The assertion is exact (== 1), not loose, because the lock
serializes the miss.
"""
iterations = 500
def work():
dispatcher.notify_broadcast("payload")
with ThreadPoolExecutor(max_workers=16) as pool:
futures = [pool.submit(work) for _ in range(iterations)]
for future in as_completed(futures):
future.result()
assert fake_declare["count"] == 1
@@ -1,6 +1,13 @@
import datetime
import time
from dataclasses import (
dataclass,
field,
)
from math import inf
from types import SimpleNamespace
from typing import Optional
from unittest.mock import MagicMock
import pytest
@@ -14,6 +21,27 @@ from galaxy.queues import connection_from_config
from galaxy.web_stack import application_stack_instance
@dataclass
class FakeStatsdClient:
"""In-memory statsd recorder — see docstring in ``test_sse_dispatch.py``."""
counters: dict[tuple[str, tuple[tuple[str, str], ...]], int] = field(default_factory=dict)
timings: list[tuple[str, float, tuple[tuple[str, str], ...]]] = field(default_factory=list)
def incr(self, metric: str, tags: Optional[dict[str, str]] = None) -> None:
key = (metric, tuple(sorted((tags or {}).items())))
self.counters[key] = self.counters.get(key, 0) + 1
def timing(self, metric: str, value: float, tags: Optional[dict[str, str]] = None) -> None:
self.timings.append((metric, value, tuple(sorted((tags or {}).items()))))
def counter(self, metric: str, tags: Optional[dict[str, str]] = None) -> int:
return self.counters.get((metric, tuple(sorted((tags or {}).items()))), 0)
def timings_for(self, metric: str) -> list[tuple[float, dict[str, str]]]:
return [(v, dict(t)) for m, v, t in self.timings if m == metric]
def bar(app, **kwargs):
app.some_var = "bar"
app.tasks_executed.append("echo")
@@ -119,3 +147,86 @@ def wait_for_var(obj, var, value, tries=10, sleep=0.25):
tries -= 1
time.sleep(sleep)
assert getattr(obj, var) == value
# ---------------------------------------------------------------------------
# process_task observability tests
#
# These don't need a real broker or DB — we just instantiate GalaxyQueueWorker
# via ``__new__`` and drive ``process_task`` directly with fake ``body`` /
# ``message`` objects. The statsd client is pulled through
# ``app.execution_timer_factory.galaxy_statsd_client``.
# ---------------------------------------------------------------------------
def _make_fake_worker(task_fn, statsd_client):
worker = GalaxyQueueWorker.__new__(GalaxyQueueWorker)
worker.app = SimpleNamespace(
config=SimpleNamespace(server_name="test.server"),
execution_timer_factory=SimpleNamespace(galaxy_statsd_client=statsd_client),
)
worker.task_mapping = {"echo": task_fn}
worker.epoch = 0
# ``producer`` is a read-only property on the mixin; we avoid the publisher
# path entirely by leaving ``reply_to`` out of the fake message properties.
return worker
def _fake_message():
message = MagicMock()
message.headers = {"epoch": inf} # always greater than worker.epoch
message.properties = {} # no reply_to — skip publisher path
return message
def test_process_task_emits_counter_and_ok_timer():
statsd = FakeStatsdClient()
calls: list[dict] = []
def handler(app, **kwargs):
calls.append(kwargs)
return "done"
worker = _make_fake_worker(handler, statsd)
worker.process_task({"task": "echo", "kwargs": {"x": 1}}, _fake_message())
# Handler actually ran — if the worker silently dropped the task, this list
# would be empty and the test would fail loudly.
assert calls == [{"x": 1}]
assert statsd.counter("galaxy.control_queue.task.count", {"task": "echo"}) == 1
assert len(statsd.timings_for("galaxy.control_queue.task.latency_ms")) == 1
_value, tags = statsd.timings_for("galaxy.control_queue.task.latency_ms")[0]
assert tags == {"task": "echo", "outcome": "ok"}
def test_process_task_emits_error_timer_on_handler_exception():
statsd = FakeStatsdClient()
invocations: list[bool] = []
def boom(app, **kwargs):
invocations.append(True)
raise RuntimeError("handler failed")
worker = _make_fake_worker(boom, statsd)
# process_task swallows handler exceptions (logged, not raised).
worker.process_task({"task": "echo", "kwargs": {}}, _fake_message())
# Handler was actually invoked before raising.
assert invocations == [True]
assert statsd.counter("galaxy.control_queue.task.count", {"task": "echo"}) == 1
assert len(statsd.timings_for("galaxy.control_queue.task.latency_ms")) == 1
_value, tags = statsd.timings_for("galaxy.control_queue.task.latency_ms")[0]
assert tags == {"task": "echo", "outcome": "error"}
def test_process_task_no_statsd_is_silent_no_op():
calls: list[dict] = []
def handler(app, **kwargs):
calls.append(kwargs)
return "done"
worker = _make_fake_worker(handler, statsd_client=None)
worker.process_task({"task": "echo", "kwargs": {"x": 2}}, _fake_message())
# Handler ran — the test guards both "no exception" AND "task not silently dropped".
assert calls == [{"x": 2}]
+12 -13
View File
@@ -9,10 +9,7 @@ from datetime import (
)
from types import SimpleNamespace
from typing import Optional
from unittest.mock import (
MagicMock,
patch,
)
from unittest.mock import MagicMock
import jwt
import pytest
@@ -400,15 +397,16 @@ def test_oidc_config_custom_auth_pipeline_and_extra(mock_oidc_config_file, mock_
def test_sync_user_profile_skips_when_account_interface_enabled():
manager = MagicMock()
session = MagicMock()
notify = MagicMock()
app_config = SimpleNamespace(enable_account_interface=True, enable_notification_system=True)
app = SimpleNamespace(config=app_config, user_manager=manager, notification_manager=SimpleNamespace())
notification_manager = SimpleNamespace(send_notification_internal=notify)
app = SimpleNamespace(config=app_config, user_manager=manager, notification_manager=notification_manager)
trans = SimpleNamespace(app=app, sa_session=session)
strategy = SimpleNamespace(config={"GALAXY_TRANS": trans, "FIXED_DELEGATED_AUTH": True})
user = SimpleNamespace(id=1, preferences={})
details = {"email": "new@example.com", "username": "newname"}
with patch("galaxy.webapps.galaxy.services.notifications.NotificationService.send_notification_internal") as notify:
sync_user_profile(strategy=strategy, details=details, user=user)
sync_user_profile(strategy=strategy, details=details, user=user)
manager.update_email.assert_not_called()
manager.update_username.assert_not_called()
@@ -419,15 +417,16 @@ def test_sync_user_profile_skips_when_account_interface_enabled():
def test_sync_user_profile_skips_when_fixed_delegated_auth_disabled():
manager = MagicMock()
session = MagicMock()
notify = MagicMock()
app_config = SimpleNamespace(enable_account_interface=False, enable_notification_system=True)
app = SimpleNamespace(config=app_config, user_manager=manager, notification_manager=SimpleNamespace())
notification_manager = SimpleNamespace(send_notification_internal=notify)
app = SimpleNamespace(config=app_config, user_manager=manager, notification_manager=notification_manager)
trans = SimpleNamespace(app=app, sa_session=session)
strategy = SimpleNamespace(config={"GALAXY_TRANS": trans, "FIXED_DELEGATED_AUTH": False})
user = SimpleNamespace(id=2, email="old@example.com", username="oldname", preferences={})
details = {"email": "new@example.com", "username": "newname"}
with patch("galaxy.webapps.galaxy.services.notifications.NotificationService.send_notification_internal") as notify:
sync_user_profile(strategy=strategy, details=details, user=user)
sync_user_profile(strategy=strategy, details=details, user=user)
manager.update_email.assert_not_called()
manager.update_username.assert_not_called()
@@ -438,16 +437,16 @@ def test_sync_user_profile_skips_when_fixed_delegated_auth_disabled():
def test_sync_user_profile_updates_when_account_interface_disabled():
manager = MagicMock()
session = MagicMock()
notify = MagicMock()
app_config = SimpleNamespace(enable_account_interface=False, enable_notification_system=True)
notification_manager = SimpleNamespace(notifications_enabled=True)
notification_manager = SimpleNamespace(notifications_enabled=True, send_notification_internal=notify)
app = SimpleNamespace(config=app_config, user_manager=manager, notification_manager=notification_manager)
trans = SimpleNamespace(app=app, sa_session=session)
strategy = SimpleNamespace(config={"GALAXY_TRANS": trans, "FIXED_DELEGATED_AUTH": True})
user = SimpleNamespace(id=2, email="old@example.com", username="oldname", preferences={})
details = {"email": "new@example.com", "username": "newname"}
with patch("galaxy.webapps.galaxy.services.notifications.NotificationService.send_notification_internal") as notify:
sync_user_profile(strategy=strategy, details=details, user=user)
sync_user_profile(strategy=strategy, details=details, user=user)
manager.update_email.assert_called_once_with(
trans, user, "new@example.com", commit=False, send_activation_email=False