mirror of
https://github.com/galaxyproject/galaxy.git
synced 2026-09-24 16:30:27 +08:00
Merge pull request #22513 from mvdbeek/sse-notifications
Server-Sent Events for history + notification updates
This commit is contained in:
@@ -19,6 +19,10 @@ env:
|
||||
YARN_INSTALL_OPTS: --frozen-lockfile
|
||||
GALAXY_CONFIG_SQLALCHEMY_WARN_20: '1'
|
||||
GALAXY_DEPENDENCIES_INSTALL_WEASYPRINT: '1'
|
||||
# TEMP: shake down SSE/notification system across full UI surface — revert before merge
|
||||
GALAXY_CONFIG_OVERRIDE_ENABLE_NOTIFICATION_SYSTEM: '1'
|
||||
GALAXY_CONFIG_OVERRIDE_ENABLE_SSE_HISTORY_UPDATES: '1'
|
||||
GALAXY_CONFIG_OVERRIDE_ENABLE_SSE_ENTRY_POINT_UPDATES: '1'
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
@@ -20,6 +20,10 @@ env:
|
||||
GALAXY_TEST_SELENIUM_HEADLESS: 1
|
||||
YARN_INSTALL_OPTS: --frozen-lockfile
|
||||
GALAXY_CONFIG_SQLALCHEMY_WARN_20: '1'
|
||||
# TEMP: shake down SSE/notification system across full UI surface — revert before merge
|
||||
GALAXY_CONFIG_OVERRIDE_ENABLE_NOTIFICATION_SYSTEM: '1'
|
||||
GALAXY_CONFIG_OVERRIDE_ENABLE_SSE_HISTORY_UPDATES: '1'
|
||||
GALAXY_CONFIG_OVERRIDE_ENABLE_SSE_ENTRY_POINT_UPDATES: '1'
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
@@ -19,6 +19,10 @@ env:
|
||||
GALAXY_TEST_SKIP_FLAKEY_TESTS_ON_ERROR: 1
|
||||
YARN_INSTALL_OPTS: --frozen-lockfile
|
||||
GALAXY_CONFIG_SQLALCHEMY_WARN_20: '1'
|
||||
# TEMP: shake down SSE/notification system across full UI surface — revert before merge
|
||||
GALAXY_CONFIG_OVERRIDE_ENABLE_NOTIFICATION_SYSTEM: '1'
|
||||
GALAXY_CONFIG_OVERRIDE_ENABLE_SSE_HISTORY_UPDATES: '1'
|
||||
GALAXY_CONFIG_OVERRIDE_ENABLE_SSE_ENTRY_POINT_UPDATES: '1'
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
@@ -18,6 +18,8 @@ export const pendingRequestsMiddleware: Middleware = {
|
||||
return new Request(request, { headers });
|
||||
}
|
||||
const shared = getPendingAbortSignal();
|
||||
// Combine with any signal the caller may have set so we don't silently
|
||||
// drop their cancellation semantics.
|
||||
const signal = typeof AbortSignal.any === "function" ? AbortSignal.any([request.signal, shared]) : shared;
|
||||
return new Request(request, { signal });
|
||||
},
|
||||
|
||||
@@ -1292,6 +1292,33 @@ export interface paths {
|
||||
patch?: never;
|
||||
trace?: never;
|
||||
};
|
||||
"/api/events/stream": {
|
||||
parameters: {
|
||||
query?: never;
|
||||
header?: never;
|
||||
path?: never;
|
||||
cookie?: never;
|
||||
};
|
||||
/**
|
||||
* Server-Sent Events stream for real-time updates.
|
||||
* @description Opens a Server-Sent Events (SSE) connection that pushes real-time
|
||||
* updates for notifications, history changes, and other events.
|
||||
*
|
||||
* On reconnect, the browser sends the ``Last-Event-ID`` header automatically.
|
||||
* If the notification system is enabled, any notifications created since that
|
||||
* timestamp are delivered as a catch-up ``notification_status`` event.
|
||||
*
|
||||
* Anonymous users receive only broadcast events.
|
||||
*/
|
||||
get: operations["stream_events_api_events_stream_get"];
|
||||
put?: never;
|
||||
post?: never;
|
||||
delete?: never;
|
||||
options?: never;
|
||||
head?: never;
|
||||
patch?: never;
|
||||
trace?: never;
|
||||
};
|
||||
"/api/exports": {
|
||||
parameters: {
|
||||
query?: never;
|
||||
@@ -33451,6 +33478,46 @@ export interface operations {
|
||||
};
|
||||
};
|
||||
};
|
||||
stream_events_api_events_stream_get: {
|
||||
parameters: {
|
||||
query?: never;
|
||||
header?: {
|
||||
"Last-Event-ID"?: string | null;
|
||||
/** @description The user ID that will be used to effectively make this API call. Only admins and designated users can make API calls on behalf of other users. */
|
||||
"run-as"?: string | null;
|
||||
};
|
||||
path?: never;
|
||||
cookie?: never;
|
||||
};
|
||||
requestBody?: never;
|
||||
responses: {
|
||||
/** @description Successful Response */
|
||||
200: {
|
||||
headers: {
|
||||
[name: string]: unknown;
|
||||
};
|
||||
content?: never;
|
||||
};
|
||||
/** @description Request Error */
|
||||
"4XX": {
|
||||
headers: {
|
||||
[name: string]: unknown;
|
||||
};
|
||||
content: {
|
||||
"application/json": components["schemas"]["MessageExceptionModel"];
|
||||
};
|
||||
};
|
||||
/** @description Server Error */
|
||||
"5XX": {
|
||||
headers: {
|
||||
[name: string]: unknown;
|
||||
};
|
||||
content: {
|
||||
"application/json": components["schemas"]["MessageExceptionModel"];
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
index_api_exports_get: {
|
||||
parameters: {
|
||||
query?: {
|
||||
|
||||
@@ -16,8 +16,9 @@ import { useNotificationsStore } from "@/stores/notificationsStore";
|
||||
* will use a fresh signal and is not affected.
|
||||
*/
|
||||
export function discardActiveConnectionsBeforeAuthNavigation() {
|
||||
// Stop polling watchers first so they can't kick off new fetches, then
|
||||
// abort any requests still in flight via the shared AbortController.
|
||||
// Order: close SSE streams first (synchronous TCP close), then stop the
|
||||
// polling watchers so they can't kick off new fetches, then abort any
|
||||
// requests still in flight via the shared AbortController.
|
||||
useHistoryStore().stopWatchingHistory();
|
||||
useEntryPointStore().stopWatchingEntryPoints();
|
||||
useNotificationsStore().stopWatchingNotifications();
|
||||
|
||||
@@ -0,0 +1,191 @@
|
||||
import { onScopeDispose, ref } from "vue";
|
||||
|
||||
import { withPrefix } from "@/utils/redirect";
|
||||
|
||||
/**
|
||||
* All SSE event types the server may emit.
|
||||
*/
|
||||
export const SSE_EVENT_TYPES = [
|
||||
"notification_update",
|
||||
"broadcast_update",
|
||||
"notification_status",
|
||||
"history_update",
|
||||
"entry_point_update",
|
||||
] as const;
|
||||
|
||||
export type SSEEventType = (typeof SSE_EVENT_TYPES)[number];
|
||||
|
||||
interface SSEDebugGlobals {
|
||||
__galaxy_sse_connected?: boolean;
|
||||
__galaxy_sse_last_event_ts?: number;
|
||||
}
|
||||
|
||||
function sseGlobals(): SSEDebugGlobals {
|
||||
return window as unknown as SSEDebugGlobals;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Module-level shared EventSource.
|
||||
//
|
||||
// Every call to ``useSSE`` registers its handler against this one socket so
|
||||
// the tab opens a single ``/api/events/stream`` connection no matter how many
|
||||
// stores listen. HTTP/1.1 caps simultaneous connections per origin at six;
|
||||
// before this consolidation we burned three slots on SSE alone (history,
|
||||
// notifications, entry points), which is what starved the scratchbook iframe
|
||||
// flow — see the fix in ``client/src/entry/analysis/App.vue``.
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
type Handler = (event: MessageEvent) => void;
|
||||
|
||||
let sharedSource: EventSource | null = null;
|
||||
const sharedConnected = ref(false);
|
||||
const subscribers: Map<SSEEventType, Set<Handler>> = new Map();
|
||||
// Track the per-type dispatchers we registered so ``closeSource`` removes the
|
||||
// exact same listeners (``addEventListener`` matches by reference).
|
||||
const dispatchers: Map<SSEEventType, Handler> = new Map();
|
||||
|
||||
function openSourceIfNeeded() {
|
||||
if (sharedSource) {
|
||||
return;
|
||||
}
|
||||
sharedSource = new EventSource(withPrefix("/api/events/stream"));
|
||||
|
||||
for (const eventType of SSE_EVENT_TYPES) {
|
||||
const dispatcher: Handler = (event) => {
|
||||
// Selenium tests watch ``__galaxy_sse_last_event_ts`` to prove that
|
||||
// an observable state change came from an SSE push and not the
|
||||
// polling fallback (where the global would never advance).
|
||||
sseGlobals().__galaxy_sse_last_event_ts = Date.now();
|
||||
const subs = subscribers.get(eventType);
|
||||
if (!subs) {
|
||||
return;
|
||||
}
|
||||
for (const handler of subs) {
|
||||
handler(event);
|
||||
}
|
||||
};
|
||||
dispatchers.set(eventType, dispatcher);
|
||||
sharedSource.addEventListener(eventType, dispatcher);
|
||||
}
|
||||
|
||||
sharedSource.onopen = () => {
|
||||
sharedConnected.value = true;
|
||||
// Global readiness flag so Selenium tests can distinguish a working
|
||||
// SSE pipeline from the polling fallback.
|
||||
sseGlobals().__galaxy_sse_connected = true;
|
||||
};
|
||||
|
||||
sharedSource.onerror = () => {
|
||||
// EventSource auto-reconnects natively; SSE-vs-polling is a
|
||||
// config-level decision (see historyStore / notificationsStore), so
|
||||
// we must not give up on transient errors here — doing so would leave
|
||||
// the client with no updates at all.
|
||||
sharedConnected.value = false;
|
||||
sseGlobals().__galaxy_sse_connected = false;
|
||||
};
|
||||
|
||||
// Browser EventSource teardown during a full-page navigation
|
||||
// (``window.location.href = …``) is not guaranteed to happen before the
|
||||
// browser issues requests for the new page — we've seen Chrome keep the
|
||||
// stream alive long enough that a login/register POST reload races the
|
||||
// close, and the new page then loads with a stale auth view. Force a
|
||||
// synchronous ``close()`` during ``pagehide`` (fires for both reloads and
|
||||
// tab-close, unlike ``beforeunload``) to close that window.
|
||||
if (typeof window !== "undefined") {
|
||||
window.addEventListener("pagehide", closeSource);
|
||||
}
|
||||
}
|
||||
|
||||
function closeSource() {
|
||||
if (!sharedSource) {
|
||||
return;
|
||||
}
|
||||
for (const [eventType, dispatcher] of dispatchers) {
|
||||
sharedSource.removeEventListener(eventType, dispatcher);
|
||||
}
|
||||
dispatchers.clear();
|
||||
sharedSource.close();
|
||||
sharedSource = null;
|
||||
sharedConnected.value = false;
|
||||
sseGlobals().__galaxy_sse_connected = false;
|
||||
if (typeof window !== "undefined") {
|
||||
window.removeEventListener("pagehide", closeSource);
|
||||
}
|
||||
}
|
||||
|
||||
function addSubscriber(onEvent: Handler, eventTypes: readonly SSEEventType[]) {
|
||||
for (const eventType of eventTypes) {
|
||||
let subs = subscribers.get(eventType);
|
||||
if (!subs) {
|
||||
subs = new Set();
|
||||
subscribers.set(eventType, subs);
|
||||
}
|
||||
subs.add(onEvent);
|
||||
}
|
||||
}
|
||||
|
||||
function removeSubscriber(onEvent: Handler, eventTypes: readonly SSEEventType[]): boolean {
|
||||
let anyRemaining = false;
|
||||
for (const eventType of eventTypes) {
|
||||
const subs = subscribers.get(eventType);
|
||||
if (subs) {
|
||||
subs.delete(onEvent);
|
||||
if (subs.size === 0) {
|
||||
subscribers.delete(eventType);
|
||||
}
|
||||
}
|
||||
}
|
||||
for (const subs of subscribers.values()) {
|
||||
if (subs.size > 0) {
|
||||
anyRemaining = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
return anyRemaining;
|
||||
}
|
||||
|
||||
/**
|
||||
* Composable for subscribing to events on the shared SSE stream.
|
||||
*
|
||||
* The browser's EventSource handles reconnection automatically and sends the
|
||||
* ``Last-Event-ID`` header so the server can catch up on missed events. Only
|
||||
* one EventSource is opened per tab regardless of how many callers invoke
|
||||
* this composable; the composable multiplexes dispatch per event type.
|
||||
*
|
||||
* @param onEvent - callback invoked for every matching SSE event
|
||||
* @param eventTypes - subset of event types to listen to (defaults to all)
|
||||
*/
|
||||
export function useSSE(onEvent: Handler, eventTypes: readonly SSEEventType[] = SSE_EVENT_TYPES) {
|
||||
let connected_: boolean = false;
|
||||
|
||||
function connect() {
|
||||
if (connected_) {
|
||||
return;
|
||||
}
|
||||
connected_ = true;
|
||||
addSubscriber(onEvent, eventTypes);
|
||||
openSourceIfNeeded();
|
||||
}
|
||||
|
||||
function disconnect() {
|
||||
if (!connected_) {
|
||||
return;
|
||||
}
|
||||
connected_ = false;
|
||||
const anyRemaining = removeSubscriber(onEvent, eventTypes);
|
||||
if (!anyRemaining) {
|
||||
closeSource();
|
||||
}
|
||||
}
|
||||
|
||||
onScopeDispose(() => {
|
||||
disconnect();
|
||||
});
|
||||
|
||||
return { connect, disconnect, connected: sharedConnected };
|
||||
}
|
||||
|
||||
/**
|
||||
* @deprecated Use `useSSE` instead. This alias exists for backward compatibility.
|
||||
*/
|
||||
export const useNotificationSSE = useSSE;
|
||||
@@ -48,7 +48,7 @@
|
||||
</template>
|
||||
<script>
|
||||
import { storeToRefs } from "pinia";
|
||||
import { ref, watch } from "vue";
|
||||
import { computed, ref, watch } from "vue";
|
||||
import { useRoute } from "vue-router/composables";
|
||||
|
||||
import { getGalaxyInstance } from "@/app";
|
||||
@@ -105,9 +105,29 @@ export default {
|
||||
const uploadModal = ref(null);
|
||||
setGlobalUploadModal(uploadModal);
|
||||
|
||||
const embedded = useRouteQueryBool("embed");
|
||||
// Treat any iframe context as embedded: scratchbook pops dataset
|
||||
// displays into ``WinBox`` iframes that hit the same routes without
|
||||
// an ``embed`` query param, and each one would otherwise open its own
|
||||
// SSE + polling traffic, quickly saturating the HTTP/1.1 per-origin
|
||||
// connection pool (e.g. ``test_scratchbook_window_persistence`` hangs
|
||||
// indefinitely after two windows are open).
|
||||
const inIframe = (() => {
|
||||
if (typeof window === "undefined") {
|
||||
return false;
|
||||
}
|
||||
try {
|
||||
return window.top !== window.self;
|
||||
} catch {
|
||||
// Cross-origin access throws — that's definitely an iframe.
|
||||
return true;
|
||||
}
|
||||
})();
|
||||
const embeddedQuery = useRouteQueryBool("embed");
|
||||
const embedded = computed(() => embeddedQuery.value || inIframe);
|
||||
const historyStore = useHistoryStore();
|
||||
historyStore.startWatchingHistory();
|
||||
if (!embedded.value) {
|
||||
historyStore.startWatchingHistory();
|
||||
}
|
||||
|
||||
watch(
|
||||
() => embedded.value,
|
||||
|
||||
@@ -0,0 +1,74 @@
|
||||
/**
|
||||
* Shared test helpers for the SSE-driven stores (historyStore, notificationsStore).
|
||||
*
|
||||
* Both stores consume the same `useSSE` composable and need:
|
||||
* - a mock that captures the onEvent callback so tests can synthesize SSE messages;
|
||||
* - visibility-state patching without leaking across tests (JSDOM's `document`
|
||||
* is shared by every test in the same worker, so an unrestored
|
||||
* `Object.defineProperty` causes silent bleed).
|
||||
*
|
||||
* Because ``vi.mock`` is hoisted above module-level variables, tests must
|
||||
* construct the SSE-mock state via ``vi.hoisted`` and then hand it to
|
||||
* ``sseMockFactory`` from inside the ``vi.mock`` factory. See the ``.test.ts``
|
||||
* files in this directory for the pattern.
|
||||
*/
|
||||
|
||||
import { vi } from "vitest";
|
||||
import { type Ref, ref } from "vue";
|
||||
|
||||
export interface SSEMockState {
|
||||
onEvent: ((event: MessageEvent) => void) | null;
|
||||
connect: ReturnType<typeof vi.fn>;
|
||||
disconnect: ReturnType<typeof vi.fn>;
|
||||
connected?: Ref<boolean>;
|
||||
}
|
||||
|
||||
/** Build the factory used with ``vi.mock("@/composables/useNotificationSSE", ...)``. */
|
||||
export function sseMockFactory(state: SSEMockState) {
|
||||
// Lazily initialize ``connected`` so existing callers that don't pass it
|
||||
// still get a working ref.
|
||||
if (!state.connected) {
|
||||
state.connected = ref(false);
|
||||
}
|
||||
return {
|
||||
useSSE: vi.fn((onEvent: (event: MessageEvent) => void) => {
|
||||
state.onEvent = onEvent;
|
||||
return { connect: state.connect, disconnect: state.disconnect, connected: state.connected };
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
/** Synthesize an SSE message through the captured handler. */
|
||||
export function emitSse(state: SSEMockState, type: string, payload: unknown): void {
|
||||
if (!state.onEvent) {
|
||||
throw new Error("useSSE was not called by the store under test — cannot emit an SSE event");
|
||||
}
|
||||
state.onEvent(new MessageEvent(type, { data: JSON.stringify(payload) }));
|
||||
}
|
||||
|
||||
/**
|
||||
* Save the current ``document.visibilityState`` descriptor and return a restorer.
|
||||
* Call the restorer in ``afterEach`` to prevent patching from leaking into later tests.
|
||||
*/
|
||||
export function useVisibilityPatch(): {
|
||||
set: (state: "visible" | "hidden") => void;
|
||||
restore: () => void;
|
||||
} {
|
||||
const original = Object.getOwnPropertyDescriptor(document, "visibilityState");
|
||||
return {
|
||||
set(state: "visible" | "hidden") {
|
||||
Object.defineProperty(document, "visibilityState", {
|
||||
configurable: true,
|
||||
get: () => state,
|
||||
});
|
||||
document.dispatchEvent(new Event("visibilitychange"));
|
||||
},
|
||||
restore() {
|
||||
if (original) {
|
||||
Object.defineProperty(document, "visibilityState", original);
|
||||
} else {
|
||||
delete (document as unknown as Record<string, unknown>).visibilityState;
|
||||
}
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -1,12 +1,25 @@
|
||||
import flushPromises from "flush-promises";
|
||||
import { createPinia, setActivePinia } from "pinia";
|
||||
import { beforeEach, describe, expect, it } from "vitest";
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
import { HttpResponse, useServerMock } from "@/api/client/__mocks__";
|
||||
|
||||
import testInteractiveToolsResponse from "../components/InteractiveTools/testData/testInteractiveToolsResponse";
|
||||
import { sseMockFactory } from "./_testing/sseStoreSupport";
|
||||
import { useEntryPointStore } from "./entryPointStore";
|
||||
|
||||
// ``vi.mock`` is hoisted above module-level declarations, so the capture-state
|
||||
// has to be built via ``vi.hoisted`` to be visible to the factory. Prevents
|
||||
// these tests from opening a real EventSource against ``/api/events/stream``
|
||||
// when ``useEntryPointStore()`` is invoked.
|
||||
const sseState = vi.hoisted(() => ({
|
||||
onEvent: null,
|
||||
connect: vi.fn(),
|
||||
disconnect: vi.fn(),
|
||||
connected: null,
|
||||
}));
|
||||
vi.mock("@/composables/useNotificationSSE", () => sseMockFactory(sseState));
|
||||
|
||||
const { server, http } = useServerMock();
|
||||
|
||||
describe("stores/EntryPointStore", () => {
|
||||
|
||||
@@ -1,10 +1,12 @@
|
||||
import axios from "axios";
|
||||
import isEqual from "lodash.isequal";
|
||||
import { defineStore } from "pinia";
|
||||
import { computed, ref } from "vue";
|
||||
import { computed, ref, watch } from "vue";
|
||||
|
||||
import { useResourceWatcher } from "@/composables/resourceWatcher";
|
||||
import { useSSE } from "@/composables/useNotificationSSE";
|
||||
import { getAppRoot } from "@/onload/loadConfig";
|
||||
import { useConfigStore } from "@/stores/configurationStore";
|
||||
import { rethrowSimple } from "@/utils/simple-error";
|
||||
|
||||
const ACTIVE_POLLING_INTERVAL = 10000;
|
||||
@@ -23,23 +25,8 @@ interface EntryPoint {
|
||||
}
|
||||
|
||||
export const useEntryPointStore = defineStore("entryPointStore", () => {
|
||||
const { startWatchingResource: startWatchingEntryPoints, stopWatchingResource: stopWatchingEntryPoints } =
|
||||
useResourceWatcher(fetchEntryPoints, {
|
||||
shortPollingInterval: ACTIVE_POLLING_INTERVAL,
|
||||
enableBackgroundPolling: false, // No need to poll in the background
|
||||
});
|
||||
|
||||
const entryPoints = ref<EntryPoint[]>([]);
|
||||
|
||||
const entryPointsForJob = computed(() => {
|
||||
return (jobId: string) => entryPoints.value.filter((entryPoint) => entryPoint["job_id"] === jobId);
|
||||
});
|
||||
|
||||
const entryPointsForHda = computed(() => {
|
||||
return (hdaId: string) =>
|
||||
entryPoints.value.filter((entryPoint) => entryPoint["output_datasets_ids"].includes(hdaId));
|
||||
});
|
||||
|
||||
async function fetchEntryPoints() {
|
||||
const url = `${getAppRoot()}api/entry_points`;
|
||||
const params = { running: true };
|
||||
@@ -51,6 +38,80 @@ export const useEntryPointStore = defineStore("entryPointStore", () => {
|
||||
}
|
||||
}
|
||||
|
||||
// SSE-driven path: on each entry_point_update signal, refetch the canonical
|
||||
// list from REST. The event carries no data — it's a pure wake-up.
|
||||
function handleEntryPointSSEEvent(_event: MessageEvent) {
|
||||
fetchEntryPoints().catch((err) => console.error("Error refreshing entry points from SSE push:", err));
|
||||
}
|
||||
const {
|
||||
connect: sseConnect,
|
||||
disconnect: sseDisconnect,
|
||||
connected: sseConnected,
|
||||
} = useSSE(handleEntryPointSSEEvent, ["entry_point_update"]);
|
||||
let stopPolling: (() => void) | null = null;
|
||||
let stopConnectedWatcher: (() => void) | null = null;
|
||||
|
||||
let watchingInitialized = false;
|
||||
|
||||
// Callers opt in via ``startWatchingEntryPoints()`` (App.vue gates this on
|
||||
// ``interactivetools_enable``). We then pick SSE or polling based on the
|
||||
// server flag — mutually exclusive, mirroring historyStore / notificationsStore.
|
||||
// ``useConfigStore`` is resolved lazily here so tests that only exercise
|
||||
// the data methods don't need a ``/api/configuration`` handler registered.
|
||||
function startWatchingEntryPoints() {
|
||||
if (watchingInitialized) {
|
||||
return;
|
||||
}
|
||||
watchingInitialized = true;
|
||||
const configStore = useConfigStore();
|
||||
|
||||
const decide = () => {
|
||||
if (configStore.config?.enable_sse_updates) {
|
||||
// Baseline fetch + SSE. Reconnect-refetch closes the "user
|
||||
// navigated away and missed events" window.
|
||||
fetchEntryPoints().catch((err) => console.warn("Initial entry-point load failed", err));
|
||||
sseConnect();
|
||||
stopConnectedWatcher = watch(sseConnected, (isConnected, wasConnected) => {
|
||||
if (isConnected && !wasConnected) {
|
||||
fetchEntryPoints().catch((err) =>
|
||||
console.error("Error refreshing entry points on SSE reconnect:", err),
|
||||
);
|
||||
}
|
||||
});
|
||||
} else {
|
||||
const { startWatchingResource, stopWatchingResource } = useResourceWatcher(fetchEntryPoints, {
|
||||
shortPollingInterval: ACTIVE_POLLING_INTERVAL,
|
||||
enableBackgroundPolling: false,
|
||||
});
|
||||
stopPolling = stopWatchingResource;
|
||||
startWatchingResource();
|
||||
}
|
||||
};
|
||||
|
||||
if (configStore.isLoaded) {
|
||||
decide();
|
||||
} else {
|
||||
const stop = watch(
|
||||
() => configStore.isLoaded,
|
||||
(loaded) => {
|
||||
if (loaded) {
|
||||
stop();
|
||||
decide();
|
||||
}
|
||||
},
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
const entryPointsForJob = computed(() => {
|
||||
return (jobId: string) => entryPoints.value.filter((entryPoint) => entryPoint["job_id"] === jobId);
|
||||
});
|
||||
|
||||
const entryPointsForHda = computed(() => {
|
||||
return (hdaId: string) =>
|
||||
entryPoints.value.filter((entryPoint) => entryPoint["output_datasets_ids"].includes(hdaId));
|
||||
});
|
||||
|
||||
function updateEntryPoints(data: EntryPoint[]) {
|
||||
let hasChanged = entryPoints.value.length !== data.length ? true : false;
|
||||
if (entryPoints.value.length === 0) {
|
||||
@@ -85,6 +146,22 @@ export const useEntryPointStore = defineStore("entryPointStore", () => {
|
||||
}
|
||||
}
|
||||
|
||||
// Closes the SSE stream and stops the polling watcher; paired with login
|
||||
// /register flows so background traffic doesn't outlive the navigation
|
||||
// and clobber the freshly authenticated session cookie.
|
||||
function stopWatchingEntryPoints() {
|
||||
sseDisconnect();
|
||||
if (stopPolling) {
|
||||
stopPolling();
|
||||
stopPolling = null;
|
||||
}
|
||||
if (stopConnectedWatcher) {
|
||||
stopConnectedWatcher();
|
||||
stopConnectedWatcher = null;
|
||||
}
|
||||
watchingInitialized = false;
|
||||
}
|
||||
|
||||
return {
|
||||
entryPoints,
|
||||
entryPointsForJob,
|
||||
|
||||
@@ -0,0 +1,194 @@
|
||||
import flushPromises from "flush-promises";
|
||||
import { createPinia, setActivePinia } from "pinia";
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
import { useServerMock } from "@/api/client/__mocks__";
|
||||
|
||||
import { emitSse, sseMockFactory, useVisibilityPatch } from "./_testing/sseStoreSupport";
|
||||
import { useHistoryStore } from "./historyStore";
|
||||
|
||||
// ``vi.mock`` is hoisted above module-level ``const`` declarations, so the
|
||||
// capture-state has to be built via ``vi.hoisted`` to be visible to the factory.
|
||||
const sseState = vi.hoisted(() => {
|
||||
return {
|
||||
onEvent: null as ((event: MessageEvent) => void) | null,
|
||||
connect: vi.fn(),
|
||||
disconnect: vi.fn(),
|
||||
};
|
||||
});
|
||||
|
||||
vi.mock("@/composables/useNotificationSSE", () => sseMockFactory(sseState));
|
||||
|
||||
// `watchHistory(app)` is the polling handler invoked on the short/long
|
||||
// interval. We mock it so each invocation is observable without pulling in
|
||||
// the history-items store, dataset store, and Galaxy app instance.
|
||||
const mockWatchHistory = vi.fn().mockResolvedValue(undefined);
|
||||
const mockRefreshHistoryFromPush = vi.fn().mockResolvedValue(undefined);
|
||||
vi.mock("@/watch/watchHistory", () => ({
|
||||
ACTIVE_POLLING_INTERVAL: 3000,
|
||||
INACTIVE_POLLING_INTERVAL: 60_000,
|
||||
watchHistory: (app: unknown) => mockWatchHistory(app),
|
||||
refreshHistoryFromPush: (app: unknown) => mockRefreshHistoryFromPush(app),
|
||||
}));
|
||||
|
||||
vi.mock("@/app", () => ({
|
||||
getGalaxyInstance: () => ({ name: "fake-galaxy" }),
|
||||
}));
|
||||
|
||||
const { server, http } = useServerMock();
|
||||
|
||||
function registerDefaultHandlers({ enableSse }: { enableSse: boolean }) {
|
||||
server.use(
|
||||
http.get("/api/configuration", ({ response }) => {
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
return response(200).json({ enable_sse_updates: enableSse } as any);
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
async function primeStore(startFn: () => void): Promise<void> {
|
||||
startFn();
|
||||
// Config load is async; let the watch fire and the initial fetch complete.
|
||||
await flushPromises();
|
||||
await vi.runOnlyPendingTimersAsync();
|
||||
await flushPromises();
|
||||
}
|
||||
|
||||
describe("historyStore — config-driven SSE vs polling", () => {
|
||||
let visibility: ReturnType<typeof useVisibilityPatch>;
|
||||
|
||||
beforeEach(() => {
|
||||
setActivePinia(createPinia());
|
||||
sseState.connect.mockClear();
|
||||
sseState.disconnect.mockClear();
|
||||
sseState.onEvent = null;
|
||||
mockWatchHistory.mockClear();
|
||||
mockRefreshHistoryFromPush.mockClear();
|
||||
vi.useFakeTimers();
|
||||
visibility = useVisibilityPatch();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
visibility.restore();
|
||||
vi.useRealTimers();
|
||||
});
|
||||
|
||||
describe("when enable_sse_updates is true (SSE scenario)", () => {
|
||||
beforeEach(() => {
|
||||
registerDefaultHandlers({ enableSse: true });
|
||||
});
|
||||
|
||||
it("primes the store with one initial load, connects SSE, and does not keep polling", async () => {
|
||||
const store = useHistoryStore();
|
||||
await primeStore(() => store.startWatchingHistory());
|
||||
|
||||
expect(sseState.connect).toHaveBeenCalledTimes(1);
|
||||
// One-shot initial fetch so the history panel isn't empty before
|
||||
// the first SSE event arrives.
|
||||
expect(mockWatchHistory).toHaveBeenCalledTimes(1);
|
||||
|
||||
// Advance past the short polling interval (3s) several times and
|
||||
// confirm the polling handler is not invoked a second time in SSE mode.
|
||||
vi.advanceTimersByTime(30_000);
|
||||
await flushPromises();
|
||||
expect(mockWatchHistory).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("does not start polling when the tab regains visibility", async () => {
|
||||
const store = useHistoryStore();
|
||||
await primeStore(() => store.startWatchingHistory());
|
||||
expect(mockWatchHistory).toHaveBeenCalledTimes(1);
|
||||
|
||||
// Simulate a tab hide/show cycle. `useResourceWatcher` registers
|
||||
// a `visibilitychange` listener whose handler calls
|
||||
// `startWatchingResourceIfNeeded` — in SSE mode that would
|
||||
// silently resume polling. Because we never instantiated the
|
||||
// watcher, no listener should exist and no poll should fire.
|
||||
visibility.set("hidden");
|
||||
visibility.set("visible");
|
||||
|
||||
await flushPromises();
|
||||
vi.advanceTimersByTime(30_000);
|
||||
await flushPromises();
|
||||
expect(mockWatchHistory).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("triggers refreshHistoryFromPush when an SSE event names the current history", async () => {
|
||||
// This test asserts the store's *decision* to refresh, not the refresh
|
||||
// itself — ``refreshHistoryFromPush`` is mocked so we can observe the
|
||||
// dispatch. The real refresh is covered end-to-end in the Selenium
|
||||
// SSE integration tests (see test/integration_selenium/test_history_sse.py).
|
||||
const store = useHistoryStore();
|
||||
await primeStore(() => store.startWatchingHistory());
|
||||
// Drive the store to a known current-history id so the handler has
|
||||
// something to match against. ``currentHistoryId`` is a computed
|
||||
// that only returns the stored id when the history is present in
|
||||
// ``storedHistories``, so the history has to be registered too.
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
store.setHistory({ id: "hist-1" } as any);
|
||||
store.setCurrentHistoryId("hist-1");
|
||||
|
||||
mockRefreshHistoryFromPush.mockClear();
|
||||
emitSse(sseState, "history_update", { history_ids: ["hist-1", "hist-2"] });
|
||||
await flushPromises();
|
||||
|
||||
expect(mockRefreshHistoryFromPush).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("ignores SSE history events that do not include the current history", async () => {
|
||||
const store = useHistoryStore();
|
||||
await primeStore(() => store.startWatchingHistory());
|
||||
store.setCurrentHistoryId("hist-1");
|
||||
|
||||
mockRefreshHistoryFromPush.mockClear();
|
||||
emitSse(sseState, "history_update", { history_ids: ["hist-2"] });
|
||||
await flushPromises();
|
||||
|
||||
expect(mockRefreshHistoryFromPush).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
describe("when enable_sse_updates is false (polling scenario)", () => {
|
||||
beforeEach(() => {
|
||||
registerDefaultHandlers({ enableSse: false });
|
||||
});
|
||||
|
||||
it("does not connect SSE and polls on the configured interval", async () => {
|
||||
const store = useHistoryStore();
|
||||
await primeStore(() => store.startWatchingHistory());
|
||||
|
||||
expect(sseState.connect).not.toHaveBeenCalled();
|
||||
|
||||
// The resource watcher invokes the handler immediately on start
|
||||
// and then re-schedules after each completion. Advance past the
|
||||
// short interval and confirm repeated invocations.
|
||||
const initialCalls = mockWatchHistory.mock.calls.length;
|
||||
expect(initialCalls).toBeGreaterThanOrEqual(1);
|
||||
|
||||
await vi.advanceTimersByTimeAsync(3000);
|
||||
await flushPromises();
|
||||
expect(mockWatchHistory.mock.calls.length).toBeGreaterThan(initialCalls);
|
||||
});
|
||||
|
||||
it("calling startWatchingHistory again is idempotent (no second SSE, polling tick count +1 only)", async () => {
|
||||
const store = useHistoryStore();
|
||||
await primeStore(() => store.startWatchingHistory());
|
||||
|
||||
const pollsAfterFirst = mockWatchHistory.mock.calls.length;
|
||||
|
||||
store.startWatchingHistory();
|
||||
await flushPromises();
|
||||
|
||||
expect(sseState.connect).not.toHaveBeenCalled();
|
||||
// Calling again must not schedule a second independent polling loop.
|
||||
// Advance past one interval and confirm only one handler tick fires,
|
||||
// not two.
|
||||
await vi.advanceTimersByTimeAsync(3000);
|
||||
await flushPromises();
|
||||
// Exactly one additional poll after the 3000ms advance — anything
|
||||
// else means a second independent polling loop was scheduled.
|
||||
const deltaAfterSecond = mockWatchHistory.mock.calls.length - pollsAfterFirst;
|
||||
expect(deltaAfterSecond).toBe(1);
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -1,5 +1,5 @@
|
||||
import { defineStore } from "pinia";
|
||||
import { computed, del, ref, set } from "vue";
|
||||
import { computed, del, ref, set, watch } from "vue";
|
||||
|
||||
import {
|
||||
type AnyHistory,
|
||||
@@ -15,7 +15,9 @@ import type { ArchivedHistoryDetailed } from "@/api/histories.archived";
|
||||
import { getGalaxyInstance } from "@/app";
|
||||
import { HistoryFilters } from "@/components/History/HistoryFilters";
|
||||
import { useResourceWatcher } from "@/composables/resourceWatcher";
|
||||
import { useSSE } from "@/composables/useNotificationSSE";
|
||||
import { useUserLocalStorage } from "@/composables/userLocalStorage";
|
||||
import { useConfigStore } from "@/stores/configurationStore";
|
||||
import {
|
||||
createAndSelectNewHistory,
|
||||
getCurrentHistoryFromServer,
|
||||
@@ -30,6 +32,7 @@ import { sortByObjectProp } from "@/utils/sorting";
|
||||
import {
|
||||
ACTIVE_POLLING_INTERVAL,
|
||||
INACTIVE_POLLING_INTERVAL,
|
||||
refreshHistoryFromPush as refreshHistoryFromPushSuppliedApp,
|
||||
watchHistory as watchHistorySuppliedApp,
|
||||
} from "@/watch/watchHistory";
|
||||
|
||||
@@ -391,14 +394,94 @@ export const useHistoryStore = defineStore("historyStore", () => {
|
||||
return watchHistorySuppliedApp(app);
|
||||
}
|
||||
|
||||
const {
|
||||
startWatchingResource: startWatchingHistory,
|
||||
stopWatchingResource: stopWatchingHistory,
|
||||
isWatchingResource: isWatchingHistory,
|
||||
} = useResourceWatcher(watchHistory, {
|
||||
shortPollingInterval: ACTIVE_POLLING_INTERVAL,
|
||||
longPollingInterval: INACTIVE_POLLING_INTERVAL,
|
||||
});
|
||||
// SSE-driven history updates: when we receive a history_update event,
|
||||
// immediately trigger a refresh of the current history
|
||||
const SSE_HISTORY_EVENT_TYPES = ["history_update"] as const;
|
||||
const { connect: sseHistoryConnect, disconnect: sseHistoryDisconnect } = useSSE(
|
||||
handleHistorySSEEvent,
|
||||
SSE_HISTORY_EVENT_TYPES,
|
||||
);
|
||||
let stopHistoryPolling: (() => void) | null = null;
|
||||
let stopIsWatchingWatcher: (() => void) | null = null;
|
||||
|
||||
function handleHistorySSEEvent(event: MessageEvent) {
|
||||
try {
|
||||
const data = JSON.parse(event.data);
|
||||
const changedHistoryIds: string[] = data.history_ids ?? [];
|
||||
if (currentHistoryId.value && changedHistoryIds.includes(currentHistoryId.value)) {
|
||||
// SSE is itself the signal that the history changed — force the
|
||||
// refresh so the update_time short-circuit in watchHistoryOnce
|
||||
// can't suppress the contents fetch.
|
||||
const app = getGalaxyInstance();
|
||||
refreshHistoryFromPushSuppliedApp(app).catch((err) =>
|
||||
console.error("Error refreshing history from SSE push:", err),
|
||||
);
|
||||
}
|
||||
} catch (e) {
|
||||
console.error("Error handling history SSE event:", e);
|
||||
}
|
||||
}
|
||||
|
||||
// Choose between SSE and polling based on the server config flag
|
||||
// `enable_sse_updates`. SSE success at the socket level is not a
|
||||
// reliable proxy: the `/api/events/stream` endpoint accepts connections
|
||||
// even when the HistoryAuditMonitor is disabled, so relying on the
|
||||
// EventSource `connected` state would silently stop polling without any
|
||||
// events ever arriving.
|
||||
//
|
||||
// `useResourceWatcher` is instantiated lazily because it registers a
|
||||
// `visibilitychange` listener that calls `startWatchingResourceIfNeeded`
|
||||
// every time the tab regains focus — in SSE mode that would re-start
|
||||
// polling we explicitly don't want.
|
||||
const isWatchingHistory = ref(false);
|
||||
let watchingInitialized = false;
|
||||
function startWatchingHistoryWithSSE() {
|
||||
if (watchingInitialized) {
|
||||
return;
|
||||
}
|
||||
watchingInitialized = true;
|
||||
|
||||
const configStore = useConfigStore();
|
||||
const decide = () => {
|
||||
if (configStore.config?.enable_sse_updates) {
|
||||
// SSE delivers incremental updates only; the store still needs
|
||||
// a baseline fetch so the history panel isn't empty until the
|
||||
// first change arrives.
|
||||
watchHistory().catch((err) => console.warn("Initial history load failed", err));
|
||||
sseHistoryConnect();
|
||||
} else {
|
||||
// The resource watcher fires its handler once immediately and
|
||||
// then re-schedules on the polling interval, which covers the
|
||||
// initial load as well as ongoing updates.
|
||||
const { startWatchingResource, stopWatchingResource, isWatchingResource } = useResourceWatcher(
|
||||
watchHistory,
|
||||
{
|
||||
shortPollingInterval: ACTIVE_POLLING_INTERVAL,
|
||||
longPollingInterval: INACTIVE_POLLING_INTERVAL,
|
||||
},
|
||||
);
|
||||
stopHistoryPolling = stopWatchingResource;
|
||||
stopIsWatchingWatcher = watch(isWatchingResource, (v) => (isWatchingHistory.value = v), {
|
||||
immediate: true,
|
||||
});
|
||||
startWatchingResource();
|
||||
}
|
||||
};
|
||||
|
||||
if (configStore.isLoaded) {
|
||||
decide();
|
||||
} else {
|
||||
const stop = watch(
|
||||
() => configStore.isLoaded,
|
||||
(loaded) => {
|
||||
if (loaded) {
|
||||
stop();
|
||||
decide();
|
||||
}
|
||||
},
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async function loadHistoryById(historyId: string) {
|
||||
if (!isLoadingHistory.has(historyId)) {
|
||||
@@ -497,6 +580,23 @@ export const useHistoryStore = defineStore("historyStore", () => {
|
||||
return contentStats;
|
||||
}
|
||||
|
||||
// Closes SSE and stops polling so the watcher can't emit a trailing
|
||||
// anonymous-cookie request that would overwrite the authenticated
|
||||
// ``galaxysession`` cookie set by the login/register response.
|
||||
function stopWatchingHistory() {
|
||||
sseHistoryDisconnect();
|
||||
if (stopHistoryPolling) {
|
||||
stopHistoryPolling();
|
||||
stopHistoryPolling = null;
|
||||
}
|
||||
if (stopIsWatchingWatcher) {
|
||||
stopIsWatchingWatcher();
|
||||
stopIsWatchingWatcher = null;
|
||||
}
|
||||
isWatchingHistory.value = false;
|
||||
watchingInitialized = false;
|
||||
}
|
||||
|
||||
return {
|
||||
histories,
|
||||
changingCurrentHistory,
|
||||
@@ -525,7 +625,7 @@ export const useHistoryStore = defineStore("historyStore", () => {
|
||||
restoreHistory,
|
||||
restoreHistories,
|
||||
handleTotalCountChange,
|
||||
startWatchingHistory,
|
||||
startWatchingHistory: startWatchingHistoryWithSSE,
|
||||
stopWatchingHistory,
|
||||
isWatchingHistory,
|
||||
loadCurrentHistory,
|
||||
|
||||
@@ -0,0 +1,185 @@
|
||||
import flushPromises from "flush-promises";
|
||||
import { createPinia, setActivePinia } from "pinia";
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
import { useServerMock } from "@/api/client/__mocks__";
|
||||
import type { UserNotification } from "@/api/notifications";
|
||||
|
||||
import { emitSse, sseMockFactory, useVisibilityPatch } from "./_testing/sseStoreSupport";
|
||||
import { useNotificationsStore } from "./notificationsStore";
|
||||
|
||||
// ``vi.mock`` is hoisted above module-level ``const`` declarations, so the
|
||||
// capture-state has to be built via ``vi.hoisted`` to be visible to the factory.
|
||||
const sseState = vi.hoisted(() => {
|
||||
return {
|
||||
onEvent: null as ((event: MessageEvent) => void) | null,
|
||||
connect: vi.fn(),
|
||||
disconnect: vi.fn(),
|
||||
};
|
||||
});
|
||||
|
||||
vi.mock("@/composables/useNotificationSSE", () => sseMockFactory(sseState));
|
||||
|
||||
// Realistic fixture: a single unread notification, as returned by
|
||||
// GET /api/notifications. Shape mirrors UserNotification.
|
||||
function makeNotificationFixture(overrides: Partial<UserNotification> = {}): UserNotification {
|
||||
return {
|
||||
id: "notif-1",
|
||||
source: "galaxy_test",
|
||||
category: "message",
|
||||
variant: "info",
|
||||
create_time: "2026-01-01T00:00:00",
|
||||
update_time: "2026-01-01T00:00:00",
|
||||
publication_time: "2026-01-01T00:00:00",
|
||||
expiration_time: null,
|
||||
seen_time: null,
|
||||
deleted: false,
|
||||
content: { category: "message", subject: "hello", message: "welcome" },
|
||||
...overrides,
|
||||
} as UserNotification;
|
||||
}
|
||||
|
||||
const SCENARIO_NOTIFICATION = makeNotificationFixture();
|
||||
const SCENARIO_STATUS_SINCE = {
|
||||
total_unread_count: 1,
|
||||
notifications: [SCENARIO_NOTIFICATION],
|
||||
broadcasts: [],
|
||||
};
|
||||
|
||||
const { server, http } = useServerMock();
|
||||
|
||||
const statusSpy = vi.fn();
|
||||
|
||||
function registerDefaultHandlers({ enableSseUpdates }: { enableSseUpdates: boolean }) {
|
||||
server.use(
|
||||
http.get("/api/configuration", ({ response }) => {
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
return response(200).json({
|
||||
enable_notification_system: true,
|
||||
enable_sse_updates: enableSseUpdates,
|
||||
} as any);
|
||||
}),
|
||||
http.get("/api/notifications", ({ response }) => {
|
||||
return response(200).json([SCENARIO_NOTIFICATION]);
|
||||
}),
|
||||
http.get("/api/notifications/broadcast", ({ response }) => {
|
||||
return response(200).json([]);
|
||||
}),
|
||||
http.get("/api/notifications/status", ({ response }) => {
|
||||
statusSpy();
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
return response(200).json(SCENARIO_STATUS_SINCE as any);
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
/** Config load + initial fetch + store-decision watch needs a couple of ticks. */
|
||||
async function primeStore(startFn: () => Promise<void> | void): Promise<void> {
|
||||
// Let the config-store fetch resolve before the store's `watch` runs.
|
||||
await vi.runOnlyPendingTimersAsync();
|
||||
await startFn();
|
||||
// Two flush cycles: one for the config watch, one for the resulting fetch.
|
||||
await flushPromises();
|
||||
await vi.runOnlyPendingTimersAsync();
|
||||
await flushPromises();
|
||||
}
|
||||
|
||||
describe("notificationsStore — config-driven SSE vs polling", () => {
|
||||
let visibility: ReturnType<typeof useVisibilityPatch>;
|
||||
|
||||
beforeEach(() => {
|
||||
setActivePinia(createPinia());
|
||||
sseState.connect.mockClear();
|
||||
sseState.disconnect.mockClear();
|
||||
sseState.onEvent = null;
|
||||
statusSpy.mockClear();
|
||||
vi.useFakeTimers();
|
||||
visibility = useVisibilityPatch();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
visibility.restore();
|
||||
vi.useRealTimers();
|
||||
});
|
||||
|
||||
describe("when enable_sse_updates is true (SSE scenario)", () => {
|
||||
beforeEach(() => {
|
||||
registerDefaultHandlers({ enableSseUpdates: true });
|
||||
});
|
||||
|
||||
it("connects SSE and does not poll the status endpoint", async () => {
|
||||
const store = useNotificationsStore();
|
||||
await primeStore(() => store.startWatchingNotifications());
|
||||
|
||||
expect(sseState.connect).toHaveBeenCalledTimes(1);
|
||||
|
||||
// Advance well past the polling interval (30s) and confirm
|
||||
// the status endpoint is never polled while SSE is the active channel.
|
||||
vi.advanceTimersByTime(120_000);
|
||||
await flushPromises();
|
||||
expect(statusSpy).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("does not start polling when the tab regains visibility", async () => {
|
||||
const store = useNotificationsStore();
|
||||
await primeStore(() => store.startWatchingNotifications());
|
||||
|
||||
visibility.set("hidden");
|
||||
visibility.set("visible");
|
||||
|
||||
await flushPromises();
|
||||
vi.advanceTimersByTime(120_000);
|
||||
await flushPromises();
|
||||
expect(statusSpy).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("ingests notification_update events into the store state", async () => {
|
||||
const store = useNotificationsStore();
|
||||
await primeStore(() => store.startWatchingNotifications());
|
||||
|
||||
const pushed = makeNotificationFixture({
|
||||
id: "notif-2",
|
||||
content: { category: "message", subject: "pushed via sse", message: "hi" },
|
||||
});
|
||||
emitSse(sseState, "notification_update", pushed);
|
||||
await flushPromises();
|
||||
|
||||
expect(store.notifications.map((n) => n.id)).toContain("notif-2");
|
||||
expect(store.totalUnreadCount).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it("ingests notification_status catch-up events on reconnect", async () => {
|
||||
const store = useNotificationsStore();
|
||||
await primeStore(() => store.startWatchingNotifications());
|
||||
|
||||
emitSse(sseState, "notification_status", {
|
||||
total_unread_count: 42,
|
||||
notifications: [makeNotificationFixture({ id: "notif-catchup" })],
|
||||
broadcasts: [],
|
||||
});
|
||||
await flushPromises();
|
||||
|
||||
expect(store.totalUnreadCount).toBe(42);
|
||||
expect(store.notifications.map((n) => n.id)).toContain("notif-catchup");
|
||||
});
|
||||
});
|
||||
|
||||
describe("when enable_sse_updates is false (polling scenario)", () => {
|
||||
beforeEach(() => {
|
||||
registerDefaultHandlers({ enableSseUpdates: false });
|
||||
});
|
||||
|
||||
it("does not connect SSE and polls the status endpoint on the configured interval", async () => {
|
||||
const store = useNotificationsStore();
|
||||
await primeStore(() => store.startWatchingNotifications());
|
||||
|
||||
expect(sseState.connect).not.toHaveBeenCalled();
|
||||
|
||||
// Advance past the short polling interval (30s) and confirm
|
||||
// the status endpoint is hit by the resource watcher.
|
||||
vi.advanceTimersByTime(30_000);
|
||||
await flushPromises();
|
||||
expect(statusSpy).toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -1,9 +1,11 @@
|
||||
import { defineStore } from "pinia";
|
||||
import { computed, ref } from "vue";
|
||||
import { computed, ref, watch } from "vue";
|
||||
|
||||
import { GalaxyApi } from "@/api";
|
||||
import type { NotificationChanges, UserNotification, UserNotificationsBatchUpdateRequest } from "@/api/notifications";
|
||||
import { useResourceWatcher } from "@/composables/resourceWatcher";
|
||||
import { useSSE } from "@/composables/useNotificationSSE";
|
||||
import { useConfigStore } from "@/stores/configurationStore";
|
||||
import { rethrowSimple } from "@/utils/simple-error";
|
||||
import { mergeObjectListsById } from "@/utils/utils";
|
||||
|
||||
@@ -13,11 +15,6 @@ const ACTIVE_POLLING_INTERVAL = 30000; // 30 seconds
|
||||
const INACTIVE_POLLING_INTERVAL = ACTIVE_POLLING_INTERVAL * 20; // 10 minutes
|
||||
|
||||
export const useNotificationsStore = defineStore("notificationsStore", () => {
|
||||
const { startWatchingResource: startWatchingNotifications, stopWatchingResource: stopWatchingNotifications } =
|
||||
useResourceWatcher(getNotificationStatus, {
|
||||
shortPollingInterval: ACTIVE_POLLING_INTERVAL,
|
||||
longPollingInterval: INACTIVE_POLLING_INTERVAL,
|
||||
});
|
||||
const broadcastsStore = useBroadcastsStore();
|
||||
|
||||
const totalUnreadCount = ref<number>(0);
|
||||
@@ -28,6 +25,45 @@ export const useNotificationsStore = defineStore("notificationsStore", () => {
|
||||
|
||||
const unreadNotifications = computed(() => notifications.value.filter((n) => !n.seen_time));
|
||||
|
||||
// --- SSE setup (listen only for notification event types) ---
|
||||
const NOTIFICATION_EVENT_TYPES = ["notification_update", "broadcast_update", "notification_status"] as const;
|
||||
const { connect: sseConnect, disconnect: sseDisconnect } = useSSE(handleSSEEvent, NOTIFICATION_EVENT_TYPES);
|
||||
let stopPolling: (() => void) | null = null;
|
||||
|
||||
function handleSSEEvent(event: MessageEvent) {
|
||||
try {
|
||||
const data = JSON.parse(event.data);
|
||||
switch (event.type) {
|
||||
case "notification_update":
|
||||
notifications.value = mergeObjectListsById(
|
||||
notifications.value,
|
||||
[data as UserNotification],
|
||||
"create_time",
|
||||
"desc",
|
||||
);
|
||||
updateUnreadCount();
|
||||
break;
|
||||
case "broadcast_update":
|
||||
broadcastsStore.updateBroadcasts([data]);
|
||||
break;
|
||||
case "notification_status":
|
||||
// Full catch-up on reconnect (same shape as GET /api/notifications/status)
|
||||
totalUnreadCount.value = data.total_unread_count;
|
||||
notifications.value = mergeObjectListsById(
|
||||
notifications.value,
|
||||
data.notifications as UserNotification[],
|
||||
"create_time",
|
||||
"desc",
|
||||
);
|
||||
broadcastsStore.updateBroadcasts(data.broadcasts);
|
||||
break;
|
||||
}
|
||||
lastNotificationUpdate.value = new Date();
|
||||
} catch (e) {
|
||||
console.error("Error handling SSE event:", e);
|
||||
}
|
||||
}
|
||||
|
||||
async function loadNotifications() {
|
||||
const { data, error } = await GalaxyApi().GET("/api/notifications");
|
||||
|
||||
@@ -76,6 +112,75 @@ export const useNotificationsStore = defineStore("notificationsStore", () => {
|
||||
}
|
||||
}
|
||||
|
||||
// Choose between SSE and polling based on the server config flag
|
||||
// `enable_sse_updates`. The `/api/events/stream` endpoint accepts
|
||||
// connections regardless of the flag, so we cannot rely on EventSource
|
||||
// connectivity to decide — config is the source of truth. Callers gate
|
||||
// on `enable_notification_system` before reaching here, so we only have
|
||||
// to pick the transport.
|
||||
//
|
||||
// `useResourceWatcher` is instantiated lazily because it registers a
|
||||
// `visibilitychange` listener that calls `startWatchingResourceIfNeeded`
|
||||
// every time the tab regains focus — in SSE mode that would re-start
|
||||
// polling we explicitly don't want.
|
||||
let watchingInitialized = false;
|
||||
function ensureWatchingWithConfig() {
|
||||
if (watchingInitialized) {
|
||||
return;
|
||||
}
|
||||
watchingInitialized = true;
|
||||
|
||||
const configStore = useConfigStore();
|
||||
const decide = () => {
|
||||
if (configStore.config?.enable_sse_updates) {
|
||||
sseConnect();
|
||||
} else {
|
||||
const { startWatchingResource: startPolling, stopWatchingResource } = useResourceWatcher(
|
||||
getNotificationStatus,
|
||||
{
|
||||
shortPollingInterval: ACTIVE_POLLING_INTERVAL,
|
||||
longPollingInterval: INACTIVE_POLLING_INTERVAL,
|
||||
},
|
||||
);
|
||||
stopPolling = stopWatchingResource;
|
||||
startPolling();
|
||||
}
|
||||
};
|
||||
|
||||
if (configStore.isLoaded) {
|
||||
decide();
|
||||
} else {
|
||||
const stop = watch(
|
||||
() => configStore.isLoaded,
|
||||
(loaded) => {
|
||||
if (loaded) {
|
||||
stop();
|
||||
decide();
|
||||
}
|
||||
},
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async function startWatchingNotifications() {
|
||||
// Always do an initial load first
|
||||
if (!lastNotificationUpdate.value) {
|
||||
try {
|
||||
loadingNotifications.value = true;
|
||||
await broadcastsStore.loadBroadcasts();
|
||||
await loadNotifications();
|
||||
updateUnreadCount();
|
||||
lastNotificationUpdate.value = new Date();
|
||||
} catch (e) {
|
||||
console.error(e);
|
||||
} finally {
|
||||
loadingNotifications.value = false;
|
||||
}
|
||||
}
|
||||
|
||||
ensureWatchingWithConfig();
|
||||
}
|
||||
|
||||
async function updateBatchNotification(request: UserNotificationsBatchUpdateRequest) {
|
||||
const { error } = await GalaxyApi().PUT("/api/notifications", {
|
||||
body: request,
|
||||
@@ -88,7 +193,11 @@ export const useNotificationsStore = defineStore("notificationsStore", () => {
|
||||
if (request.changes.deleted) {
|
||||
notifications.value = notifications.value.filter((n) => !request.notification_ids.includes(n.id));
|
||||
}
|
||||
startWatchingNotifications();
|
||||
// When polling is the active transport, trigger a poll to refresh
|
||||
// state after a local mutation. SSE delivers the update on its own.
|
||||
if (!useConfigStore().config?.enable_sse_updates) {
|
||||
startWatchingNotifications();
|
||||
}
|
||||
}
|
||||
|
||||
async function updateNotification(notification: UserNotification, changes: NotificationChanges) {
|
||||
@@ -99,6 +208,19 @@ export const useNotificationsStore = defineStore("notificationsStore", () => {
|
||||
totalUnreadCount.value = notifications.value.filter((n) => !n.seen_time).length;
|
||||
}
|
||||
|
||||
// Closes the SSE stream and stops the polling watcher so nothing running
|
||||
// in the background can outlive a full-page navigation (login/register).
|
||||
// A late-arriving response from an anonymous-cookie request would otherwise
|
||||
// overwrite the just-issued authenticated ``galaxysession`` cookie.
|
||||
function stopWatchingNotifications() {
|
||||
sseDisconnect();
|
||||
if (stopPolling) {
|
||||
stopPolling();
|
||||
stopPolling = null;
|
||||
}
|
||||
watchingInitialized = false;
|
||||
}
|
||||
|
||||
return {
|
||||
notifications,
|
||||
totalUnreadCount,
|
||||
|
||||
@@ -39,13 +39,29 @@ export async function watchHistory(app) {
|
||||
}
|
||||
|
||||
export async function watchHistoryOnce(app) {
|
||||
return _fetchHistoryAndChangedItems(app, { force: false });
|
||||
}
|
||||
|
||||
/**
|
||||
* Forces a fresh history + changed-items fetch, ignoring the `lastUpdateTime`
|
||||
* short-circuit. Use this when an out-of-band signal (SSE `history_update`)
|
||||
* already told us the history changed — we shouldn't re-gate on update_time.
|
||||
*/
|
||||
export async function refreshHistoryFromPush(app) {
|
||||
return _fetchHistoryAndChangedItems(app, { force: true });
|
||||
}
|
||||
|
||||
async function _fetchHistoryAndChangedItems(app, { force }) {
|
||||
const historyStore = useHistoryStore();
|
||||
const historyItemsStore = useHistoryItemsStore();
|
||||
const datasetStore = useDatasetStore();
|
||||
const collectionElementsStore = useCollectionElementsStore();
|
||||
|
||||
// get current history
|
||||
const checkForUpdate = new Date();
|
||||
// Always pass the `since` cursor so the server can short-circuit cheaply
|
||||
// when nothing has changed; SSE-driven `force` only bypasses the
|
||||
// client-side update_time equality gate, not the server-side cursor —
|
||||
// we don't want to refetch items the client already has.
|
||||
const history = await historyStore.loadCurrentHistory(lastUpdateTime);
|
||||
const { lastCheckedTime } = storeToRefs(historyItemsStore);
|
||||
lastCheckedTime.value = checkForUpdate;
|
||||
@@ -53,39 +69,35 @@ export async function watchHistoryOnce(app) {
|
||||
return;
|
||||
}
|
||||
|
||||
// continue if the history update time has changed
|
||||
if (!lastUpdateTime || lastUpdateTime < history.update_time) {
|
||||
const historyId = history.id;
|
||||
lastUpdateTime = history.update_time;
|
||||
historyItemsStore.setLastUpdateTime();
|
||||
// execute request to obtain recently changed items
|
||||
const params = {
|
||||
v: "dev",
|
||||
limit: limit,
|
||||
q: "update_time-ge",
|
||||
qv: lastRequestDate.toISOString(),
|
||||
};
|
||||
// request detailed info only for the expanded datasets
|
||||
const detailedIds = getCurrentlyExpandedHistoryContentIds();
|
||||
if (detailedIds.length) {
|
||||
params["details"] = detailedIds.join(",");
|
||||
}
|
||||
const url = `/api/histories/${historyId}/contents`;
|
||||
lastRequestDate = new Date();
|
||||
const payload = await urlData({ url, params });
|
||||
// show warning that not all changes have been obtained
|
||||
if (payload && payload.length == limit) {
|
||||
console.debug(`Reached limit of monitored changes (limit=${limit}).`);
|
||||
}
|
||||
// pass changed items to attached stores
|
||||
historyStore.setHistory(history);
|
||||
datasetStore.saveDatasets(payload);
|
||||
historyItemsStore.saveHistoryItems(historyId, payload);
|
||||
collectionElementsStore.saveCollections(payload);
|
||||
// trigger changes in legacy handler
|
||||
if (app) {
|
||||
app.user.loadFromApi(app.user.id || "current");
|
||||
}
|
||||
if (!force && lastUpdateTime && lastUpdateTime >= history.update_time) {
|
||||
return;
|
||||
}
|
||||
|
||||
const historyId = history.id;
|
||||
lastUpdateTime = history.update_time;
|
||||
historyItemsStore.setLastUpdateTime();
|
||||
const params = {
|
||||
v: "dev",
|
||||
limit: limit,
|
||||
q: "update_time-ge",
|
||||
qv: lastRequestDate.toISOString(),
|
||||
};
|
||||
const detailedIds = getCurrentlyExpandedHistoryContentIds();
|
||||
if (detailedIds.length) {
|
||||
params["details"] = detailedIds.join(",");
|
||||
}
|
||||
const url = `/api/histories/${historyId}/contents`;
|
||||
lastRequestDate = new Date();
|
||||
const payload = await urlData({ url, params });
|
||||
if (payload && payload.length == limit) {
|
||||
console.debug(`Reached limit of monitored changes (limit=${limit}).`);
|
||||
}
|
||||
historyStore.setHistory(history);
|
||||
datasetStore.saveDatasets(payload);
|
||||
historyItemsStore.saveHistoryItems(historyId, payload);
|
||||
collectionElementsStore.saveCollections(payload);
|
||||
if (app) {
|
||||
app.user.loadFromApi(app.user.id || "current");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -99,16 +99,33 @@ export function galaxyDevServerPlugin() {
|
||||
const originalWrite = res.write.bind(res);
|
||||
const originalEnd = res.end.bind(res);
|
||||
|
||||
// Buffer to collect response body
|
||||
// Buffer to collect response body. We only buffer when the
|
||||
// upstream response is HTML; everything else (JSON, binary,
|
||||
// and crucially `text/event-stream`) must pass straight
|
||||
// through, because streaming responses never call
|
||||
// `res.end()` and would otherwise stall indefinitely.
|
||||
const chunks = [];
|
||||
let isHtml = false;
|
||||
// Tri-state: null = undecided (first write hasn't landed yet),
|
||||
// true = stream it through untransformed,
|
||||
// false = buffer for HTML rewrite on end().
|
||||
let passthrough = null;
|
||||
|
||||
function decidePassthrough() {
|
||||
if (passthrough !== null) {
|
||||
return;
|
||||
}
|
||||
const contentType = res.getHeader("content-type");
|
||||
passthrough = !contentType || !contentType.toString().includes("text/html");
|
||||
}
|
||||
|
||||
// Override write to collect chunks
|
||||
res.write = function (chunk, encoding, callback) {
|
||||
decidePassthrough();
|
||||
if (passthrough) {
|
||||
return originalWrite(chunk, encoding, callback);
|
||||
}
|
||||
if (chunk) {
|
||||
chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk, encoding));
|
||||
}
|
||||
// Don't write yet - we'll write in end()
|
||||
if (typeof encoding === "function") {
|
||||
encoding(); // encoding is actually the callback
|
||||
} else if (typeof callback === "function") {
|
||||
@@ -119,41 +136,39 @@ export function galaxyDevServerPlugin() {
|
||||
|
||||
// Override end to transform and send response
|
||||
res.end = function (chunk, encoding, callback) {
|
||||
decidePassthrough();
|
||||
if (passthrough) {
|
||||
return originalEnd(chunk, encoding, callback);
|
||||
}
|
||||
|
||||
if (chunk) {
|
||||
chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk, encoding));
|
||||
}
|
||||
|
||||
// Check content type
|
||||
const contentType = res.getHeader("content-type");
|
||||
isHtml = contentType && contentType.toString().includes("text/html");
|
||||
|
||||
// Combine all chunks
|
||||
let body = Buffer.concat(chunks);
|
||||
|
||||
// Transform HTML responses that contain Galaxy bundles
|
||||
if (isHtml) {
|
||||
// Decompress gzip responses (common from remote Galaxy servers)
|
||||
const contentEncoding = res.getHeader("content-encoding");
|
||||
if (contentEncoding === "gzip") {
|
||||
try {
|
||||
body = gunzipSync(body);
|
||||
} catch (e) {
|
||||
// If decompression fails, continue with original body
|
||||
console.warn("[galaxy-dev-server] Failed to decompress gzip response:", e.message);
|
||||
}
|
||||
// Decompress gzip responses (common from remote Galaxy servers)
|
||||
const contentEncoding = res.getHeader("content-encoding");
|
||||
if (contentEncoding === "gzip") {
|
||||
try {
|
||||
body = gunzipSync(body);
|
||||
} catch (e) {
|
||||
// If decompression fails, continue with original body
|
||||
console.warn("[galaxy-dev-server] Failed to decompress gzip response:", e.message);
|
||||
}
|
||||
}
|
||||
|
||||
let htmlString = body.toString("utf-8");
|
||||
if (htmlString.includes("bundled.js") || htmlString.includes("/static/dist/")) {
|
||||
htmlString = transformGalaxyHtml(htmlString);
|
||||
body = Buffer.from(htmlString, "utf-8");
|
||||
let htmlString = body.toString("utf-8");
|
||||
if (htmlString.includes("bundled.js") || htmlString.includes("/static/dist/")) {
|
||||
htmlString = transformGalaxyHtml(htmlString);
|
||||
body = Buffer.from(htmlString, "utf-8");
|
||||
|
||||
// Update content-length header
|
||||
res.setHeader("content-length", body.length);
|
||||
// Update content-length header
|
||||
res.setHeader("content-length", body.length);
|
||||
|
||||
// Remove content-encoding since we've decompressed it
|
||||
res.removeHeader("content-encoding");
|
||||
}
|
||||
// Remove content-encoding since we've decompressed it
|
||||
res.removeHeader("content-encoding");
|
||||
}
|
||||
|
||||
// Send the response
|
||||
|
||||
@@ -281,6 +281,23 @@
|
||||
:Type: int
|
||||
|
||||
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
``kombu_sqla_transport_cleanup_interval``
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
:Description:
|
||||
Time (in seconds) between attempts to delete fully-consumed rows
|
||||
from the Kombu SQLAlchemy transport tables (``kombu_message``).
|
||||
Only relevant when ``amqp_internal_connection`` uses a
|
||||
``sqlalchemy+*`` scheme (the default with an on-disk
|
||||
control.sqlite); the SQLAlchemy transport has no built-in TTL, so
|
||||
without this task the tables grow unbounded. The task no-ops on
|
||||
non-SQLAlchemy brokers (RabbitMQ/Redis honor per-message
|
||||
expiration natively). Set to 0 to disable the cleanup task.
|
||||
:Default: ``900``
|
||||
:Type: int
|
||||
|
||||
|
||||
~~~~~~~~~~~~~
|
||||
``file_path``
|
||||
~~~~~~~~~~~~~
|
||||
@@ -3408,6 +3425,18 @@
|
||||
:Type: bool
|
||||
|
||||
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
``queue_metrics_interval``
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
:Description:
|
||||
How often (in seconds) the Celery beat task emits queue-depth,
|
||||
SSE-connection, and WorkerProcess gauges. Only active when
|
||||
statsd_host is set. Set to 0 to disable.
|
||||
:Default: ``15``
|
||||
:Type: int
|
||||
|
||||
|
||||
~~~~~~~~~~~~~~~~~~~~~~
|
||||
``library_import_dir``
|
||||
~~~~~~~~~~~~~~~~~~~~~~
|
||||
@@ -5804,6 +5833,41 @@
|
||||
:Type: str
|
||||
|
||||
|
||||
~~~~~~~~~~~~~~~~~~~~~~
|
||||
``enable_sse_updates``
|
||||
~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
:Description:
|
||||
Enables real-time updates via Server-Sent Events (SSE), replacing
|
||||
the history (3 s), entry-point (10 s) and notification (30 s)
|
||||
polling loops with push events delivered over a single
|
||||
``/api/events/stream`` connection per browser tab. A background
|
||||
monitor watches for history changes (via PostgreSQL LISTEN/NOTIFY,
|
||||
or audit-table polling as a fallback for SQLite); entry-point
|
||||
changes are dispatched directly from the code paths that mutate
|
||||
them; in-app notifications and broadcasts are pushed when
|
||||
``enable_notification_system`` is also true. When disabled,
|
||||
polling remains the source of updates for all three. See the admin
|
||||
guide "Server-Sent Events for real-time updates" for the full
|
||||
architecture, monitoring guidance and proxy configuration.
|
||||
:Default: ``false``
|
||||
:Type: bool
|
||||
|
||||
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
``history_audit_monitor_poll_interval``
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
:Description:
|
||||
The interval in seconds between history audit table polls when
|
||||
using the polling fallback (SQLite or when PostgreSQL
|
||||
LISTEN/NOTIFY is unavailable). Only used when enable_sse_updates
|
||||
is true. Lower values mean faster updates but more database
|
||||
queries. Recommended range: 1-5 seconds.
|
||||
:Default: ``2``
|
||||
:Type: int
|
||||
|
||||
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
``enable_notification_system``
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
@@ -5815,6 +5879,9 @@
|
||||
finished, etc.
|
||||
The system allows notification scheduling and expiration, and
|
||||
users can opt-out of specific notification categories or channels.
|
||||
Delivery is push-based via Server-Sent Events when
|
||||
``enable_sse_updates`` is also true, and falls back to 30-second
|
||||
polling against ``/api/notifications/status`` otherwise.
|
||||
Admins can schedule and broadcast notifications that will be
|
||||
visible to all users, including special server-wide announcements
|
||||
such as scheduled maintenance, high load warnings, and event
|
||||
|
||||
@@ -14,6 +14,7 @@ Galaxy Deployment & Administration
|
||||
nginx
|
||||
apache
|
||||
scaling
|
||||
sse_updates
|
||||
cluster
|
||||
jobs
|
||||
job_metrics
|
||||
|
||||
@@ -233,6 +233,17 @@ previous section:
|
||||
|
||||
## Advanced Configuration Topics
|
||||
|
||||
### Server-Sent Events (real-time updates)
|
||||
|
||||
Galaxy can push history, entry-point and notification updates to the browser
|
||||
via a long-lived Server-Sent Events stream at ``/api/events/stream``. nginx
|
||||
will buffer that response by default, which breaks the stream — either rely
|
||||
on the ``X-Accel-Buffering: no`` header Galaxy already sets, or add an
|
||||
explicit ``location /api/events/stream`` block that disables buffering and
|
||||
raises the read/send timeouts. The full configuration block, monitoring
|
||||
guidance, and the architecture overview live in
|
||||
[Server-Sent Events for real-time updates](sse_updates.md#configuring-nginx).
|
||||
|
||||
### Sending Files With Nginx
|
||||
|
||||
Galaxy sends files (e.g. dataset downloads) by opening the file and streaming it in chunks through the proxy server.
|
||||
|
||||
@@ -190,6 +190,14 @@ Some of these options deserve explanation:
|
||||
4-12 workers should be able to handle hundreds if not thousands of requests per second.
|
||||
* `extra_args`: You can specify additional arguments to pass to gunicorn here.
|
||||
|
||||
```{note}
|
||||
If you have enabled [Server-Sent Events for real-time updates](sse_updates.md), every connected browser tab holds an
|
||||
open connection to one Gunicorn worker for the lifetime of the tab. Gravity configures Gunicorn with the
|
||||
``uvicorn.workers.UvicornWorker`` async worker by default, so a single worker process can hold thousands of idle SSE
|
||||
connections without blocking — the limit you actually need to watch is memory headroom and the per-process
|
||||
file-descriptor ulimit, not the worker count.
|
||||
```
|
||||
|
||||
Note that the performance option values given above are just examples and should be tuned per your specific needs.
|
||||
However, as given, they are a good place to start.
|
||||
|
||||
|
||||
@@ -0,0 +1,280 @@
|
||||
# Server-Sent Events (SSE) for real-time updates
|
||||
|
||||
Galaxy can push history changes, interactive-tool entry-point changes, and
|
||||
in-app notifications to connected browsers via [Server-Sent Events][sse-mdn]
|
||||
instead of polling. This replaces the legacy 3-second history poll and
|
||||
10-second entry-point poll with a single long-lived HTTP connection per
|
||||
browser tab, dramatically reducing API load on busy servers and giving users
|
||||
sub-second update latency.
|
||||
|
||||
This document describes the architecture, the configuration knobs, the
|
||||
metrics admins should watch, and how to configure NGINX so the long-lived
|
||||
event connection is not buffered or prematurely terminated.
|
||||
|
||||
[sse-mdn]: https://developer.mozilla.org/en-US/docs/Web/API/Server-sent_events
|
||||
|
||||
## How it works
|
||||
|
||||
A high-level view of the moving parts on the server:
|
||||
|
||||
```
|
||||
┌────────────────┐ LISTEN/NOTIFY ┌──────────────────────┐
|
||||
│ Postgres │ ─────────────────► │ HistoryAuditMonitor │
|
||||
│ history_audit │ (or audit poll │ (one elected process) │
|
||||
│ │ on SQLite) └──────────┬───────────┘
|
||||
└────────────────┘ │ Kombu
|
||||
▼
|
||||
┌─────────────────────────────┐
|
||||
│ galaxy_queue_worker │
|
||||
│ control task fan-out: │
|
||||
│ • history_update │
|
||||
│ • entry_point_update │
|
||||
│ • notification_update │
|
||||
│ • broadcast_update │
|
||||
└──────────────┬──────────────┘
|
||||
│ in-process
|
||||
▼
|
||||
┌─────────────────────────────────────┐
|
||||
│ Each gunicorn worker: │
|
||||
│ SSEConnectionManager → asyncio │
|
||||
│ queue per connected browser tab │
|
||||
└──────────────┬──────────────────────┘
|
||||
│ HTTP chunked
|
||||
▼
|
||||
┌─────────────────────────────────────┐
|
||||
│ Browser EventSource │
|
||||
│ (single stream, multiplexed events)│
|
||||
└─────────────────────────────────────┘
|
||||
```
|
||||
|
||||
Concretely:
|
||||
|
||||
1. **One stream per browser, many event types.** The browser opens a single
|
||||
`EventSource` against `/api/events/stream`. The same connection carries
|
||||
`history_update`, `entry_point_update`, `notification_update`,
|
||||
`broadcast_update`, and `notification_status` events.
|
||||
2. **Per-process registries.** Each Gunicorn worker keeps an
|
||||
`SSEConnectionManager` that holds an `asyncio.Queue` per connected tab,
|
||||
indexed by user id (and Galaxy session id, so anonymous users still
|
||||
receive their own history's updates).
|
||||
3. **Producers.**
|
||||
- History updates come from a `HistoryAuditMonitor` that watches
|
||||
`history_audit` via PostgreSQL `LISTEN/NOTIFY` (instant) or by polling
|
||||
the audit table on SQLite. Only one process in the cluster is the
|
||||
producer, picked by `DatabaseHeartbeat` leader election. If a
|
||||
standalone `galaxy-sse-monitor` process is running it always wins;
|
||||
otherwise one webapp picks it up.
|
||||
- Entry-point updates are dispatched directly from the code paths that
|
||||
mutate interactive-tool entry points — there is no separate watcher.
|
||||
- Notifications dispatch SSE events from `NotificationManager` whenever
|
||||
a notification or broadcast is created.
|
||||
4. **Cross-process fan-out.** Producers don't know which worker holds the
|
||||
recipient's connection, so all events go through a Kombu control task
|
||||
broadcast on the internal AMQP bus. Every worker receives every event
|
||||
and locally drops the ones for users it doesn't currently hold a
|
||||
connection for.
|
||||
5. **Reconnect catch-up.** When a browser reconnects after a network blip,
|
||||
it sends `Last-Event-ID`. The server replays an aggregated
|
||||
`notification_status` covering everything since that timestamp. History
|
||||
updates come with an `update_time` cursor in the payload, so the client
|
||||
can request the delta itself.
|
||||
|
||||
### Standalone monitor (recommended for production)
|
||||
|
||||
The `galaxy-sse-monitor` console script (installed by the `galaxy-app`
|
||||
package) runs the `HistoryAuditMonitor` outside the webapp processes. This
|
||||
is the recommended layout for production because:
|
||||
|
||||
- The webapp processes never compete with each other for the audit-monitor
|
||||
role on cold starts.
|
||||
- Restarting the webapp tier doesn't briefly stall history updates while
|
||||
another worker is elected.
|
||||
- The monitor needs only DB + AMQP access, so it can be sized
|
||||
independently and runs with a much smaller resident-set than a webapp.
|
||||
|
||||
A typical Gravity supervisor entry looks like the existing
|
||||
`galaxy-celery-worker` block — point at the same `galaxy.yml` and run
|
||||
`galaxy-sse-monitor` on its own. With the daemon present, the
|
||||
`HistoryAuditMonitor` registered on the webapp side stays idle (the
|
||||
heartbeat election picks the daemon) but is still wired up so it can take
|
||||
over if the daemon goes away.
|
||||
|
||||
If `enable_sse_updates` is `false`, `galaxy-sse-monitor` will start, log a
|
||||
warning, and idle — it does no work and produces no events.
|
||||
|
||||
## Configuration
|
||||
|
||||
There is a single admin-facing flag for SSE-driven updates:
|
||||
|
||||
```yaml
|
||||
galaxy:
|
||||
enable_sse_updates: true
|
||||
```
|
||||
|
||||
This controls **all three** SSE-driven paths (history, entry-point,
|
||||
notifications). When `false`:
|
||||
|
||||
- `HistoryAuditMonitor` is not registered, so the cluster does no
|
||||
`LISTEN/NOTIFY` or audit-table polling for history changes.
|
||||
- The browser falls back to its existing 3-second history poll and
|
||||
10-second entry-point poll.
|
||||
- Notifications fall back to the existing 30-second polling against
|
||||
`/api/notifications/status`.
|
||||
|
||||
`enable_notification_system` is independent: it gates whether the
|
||||
notification system is available at all (notification creation, delivery,
|
||||
preferences, broadcasts). With `enable_notification_system: true`:
|
||||
|
||||
- `enable_sse_updates: true` → notifications arrive via SSE.
|
||||
- `enable_sse_updates: false` → notifications are polled.
|
||||
|
||||
With `enable_notification_system: false` the entire notification system is
|
||||
off — there is nothing to push or poll.
|
||||
|
||||
The polling-fallback knob for the history audit monitor stays available:
|
||||
|
||||
```yaml
|
||||
galaxy:
|
||||
history_audit_monitor_poll_interval: 2 # seconds, SQLite / no-LISTEN only
|
||||
```
|
||||
|
||||
This only matters when running on SQLite or in setups where PostgreSQL
|
||||
`LISTEN/NOTIFY` is unavailable.
|
||||
|
||||
## What to monitor
|
||||
|
||||
When statsd is configured (via `statsd_host` and friends), the SSE
|
||||
plumbing emits the following metrics. Capture these on the same dashboard
|
||||
you use for Gunicorn worker health:
|
||||
|
||||
| Metric | Type | Source | Meaning |
|
||||
| ------------------------------------------ | ------- | --------------------- | ------------------------------------------------------------ |
|
||||
| `galaxy.sse.connections.dropped` | counter | `SSEConnectionManager` | A per-connection asyncio queue filled up; an event was lost. |
|
||||
| `galaxy.sse.dispatch.count` (tag: `task`) | counter | `SSEEventDispatcher` | Control-task fan-outs by event kind. |
|
||||
| `galaxy.sse.dispatch.latency_ms` (tag: `task`) | timing | `SSEEventDispatcher` | Wall time spent enqueueing the control task. |
|
||||
| `galaxy.sse.dispatch.skipped_no_qw` | counter | `SSEEventDispatcher` | Producer tried to dispatch with no queue worker bound — events would have been dropped. |
|
||||
|
||||
You can also expose a "currently connected SSE clients" gauge if you wire
|
||||
one up: each `SSEConnectionManager` instance publishes
|
||||
`total_broadcast_connections` (all connections, including anonymous) and
|
||||
`total_per_user_connections` (connections bound to a specific user). These
|
||||
are per-worker numbers; sum across workers for a cluster total.
|
||||
|
||||
Alerting recommendations:
|
||||
|
||||
- **`galaxy.sse.connections.dropped` > 0 sustained** indicates a slow or
|
||||
stuck client whose queue filled up. Occasional drops on a network blip
|
||||
are normal; a steady rate is a bug or a misconfigured proxy holding
|
||||
events back too long.
|
||||
- **`galaxy.sse.dispatch.skipped_no_qw` > 0** means events are being lost
|
||||
because the producer process can't reach the AMQP bus. Check the
|
||||
`amqp_internal_connection` config and the AMQP broker health.
|
||||
- **`galaxy.sse.dispatch.latency_ms` p95 climbing** points at an AMQP
|
||||
bottleneck (broker load, network) — events will land late.
|
||||
|
||||
In addition, watch the standard worker metrics. SSE connections are
|
||||
long-lived (tens of minutes is common), but Galaxy runs Gunicorn with
|
||||
`uvicorn.workers.UvicornWorker` (configured by Gravity by default), so
|
||||
each worker process is async and can hold thousands of idle SSE
|
||||
connections without blocking other requests. The practical concern on
|
||||
busy servers is therefore memory and file-descriptor headroom, not
|
||||
worker exhaustion: budget a few KB per connection plus one fd per
|
||||
connection per worker, and raise `ulimit -n` accordingly.
|
||||
|
||||
## Configuring NGINX
|
||||
|
||||
The SSE endpoint is served at `/api/events/stream`. It is a normal
|
||||
HTTP/1.1 chunked response, so it works through NGINX without any special
|
||||
modules — but you must turn off response buffering and raise the
|
||||
read/send timeouts, otherwise events will arrive in batched bursts (or
|
||||
not at all until the connection times out).
|
||||
|
||||
Galaxy already sets `X-Accel-Buffering: no` on the response, which
|
||||
disables NGINX's response buffering for that one endpoint without
|
||||
affecting buffering on the rest of Galaxy. That alone is enough on most
|
||||
setups. The block below adds the read/send timeouts and HTTP/1.1
|
||||
upgrade-friendly headers explicitly so the connection survives long
|
||||
idle periods between events:
|
||||
|
||||
```nginx
|
||||
# Long-lived Server-Sent Events stream.
|
||||
# Galaxy sends ``X-Accel-Buffering: no`` on the response, which
|
||||
# disables nginx response buffering just for this endpoint.
|
||||
# The keepalive comment fires every 30s so the read timeout
|
||||
# only needs to be a comfortable margin above that.
|
||||
location /api/events/stream {
|
||||
proxy_pass http://unix:/srv/galaxy/var/gunicorn.sock;
|
||||
proxy_set_header Host $http_host;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header X-Forwarded-Proto $scheme;
|
||||
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Connection "";
|
||||
|
||||
# Disable buffering and gzip explicitly. ``X-Accel-Buffering``
|
||||
# already does this, but pinning it here also covers setups
|
||||
# where a sub-filter strips upstream headers.
|
||||
proxy_buffering off;
|
||||
proxy_cache off;
|
||||
gzip off;
|
||||
|
||||
# Keepalives fire every 30s; allow generous slack on top.
|
||||
proxy_read_timeout 1h;
|
||||
proxy_send_timeout 1h;
|
||||
}
|
||||
```
|
||||
|
||||
Place this `location` block **above** the catch-all `location /` block in
|
||||
your existing Galaxy `server {}` (see the [NGINX proxy guide](nginx.md)).
|
||||
NGINX matches longest prefix first, so the order doesn't matter for
|
||||
correctness, but keeping all the override blocks together at the top of
|
||||
the server block makes the special-case handling easy to find.
|
||||
|
||||
If you serve Galaxy at a URL prefix (`/galaxy`), prefix the location too:
|
||||
|
||||
```nginx
|
||||
location /galaxy/api/events/stream {
|
||||
proxy_pass http://unix:/srv/galaxy/var/gunicorn.sock:/galaxy;
|
||||
# ...same body as above
|
||||
}
|
||||
```
|
||||
|
||||
### Other proxies
|
||||
|
||||
If you front Galaxy with something other than NGINX, the same rules
|
||||
apply: disable response buffering for `/api/events/stream`, allow the
|
||||
connection to stay open for at least the `keepalive` interval (30 s by
|
||||
default) plus a healthy margin, and pass the request through HTTP/1.1
|
||||
without forcing `Connection: close`.
|
||||
|
||||
- **Apache `mod_proxy_http`**: add `ProxyPass` with
|
||||
`flushpackets=on flushwait=5` and bump `ProxyTimeout` to at least a
|
||||
few minutes. Avoid `mod_deflate` on this endpoint.
|
||||
- **HAProxy**: the connection is a plain HTTP/1.1 chunked response and
|
||||
needs no special handling beyond `timeout server` and
|
||||
`timeout tunnel` raised above 30 s.
|
||||
- **Cloudflare and other CDNs**: many CDNs buffer HTTP/1.1 chunked
|
||||
responses by default. Either bypass the CDN for `/api/events/stream`
|
||||
or follow the CDN's documented pattern for SSE streaming.
|
||||
|
||||
## Verifying the deployment
|
||||
|
||||
After enabling `enable_sse_updates`, three quick checks confirm the
|
||||
stream is healthy end-to-end:
|
||||
|
||||
1. From the browser DevTools Network tab, open Galaxy and look for a
|
||||
`GET /api/events/stream` request that stays in the **pending** state
|
||||
with a constantly incrementing transferred-bytes count. The response
|
||||
`Content-Type` is `text/event-stream`.
|
||||
2. Trigger a history change (run a tool, rename a dataset). The Network
|
||||
tab should show a `history_update` event in the EventStream view of
|
||||
that connection within a second or two, and the history panel
|
||||
refreshes without a polling round-trip.
|
||||
3. On the server side, `galaxy.sse.dispatch.count` should be ticking up
|
||||
for each event kind your users exercise. If you wired up the
|
||||
connection gauges, they should reflect roughly one connection per
|
||||
open browser tab.
|
||||
|
||||
If the connection opens but no events arrive, the most common causes
|
||||
are: a proxy buffering responses (revisit the NGINX section), or a
|
||||
producer that can't reach AMQP (see `galaxy.sse.dispatch.skipped_no_qw`).
|
||||
+109
-23
@@ -65,6 +65,7 @@ from galaxy.managers.file_source_instances import (
|
||||
from galaxy.managers.folders import FolderManager
|
||||
from galaxy.managers.hdas import HDAManager
|
||||
from galaxy.managers.histories import HistoryManager
|
||||
from galaxy.managers.history_audit_monitor import HistoryAuditMonitor
|
||||
from galaxy.managers.interactivetool import InteractiveToolManager
|
||||
from galaxy.managers.jobs import (
|
||||
JobManager as JobQueryManager,
|
||||
@@ -77,6 +78,8 @@ from galaxy.managers.notification import NotificationManager
|
||||
from galaxy.managers.object_store_instances import UserObjectStoreResolverImpl
|
||||
from galaxy.managers.roles import RoleManager
|
||||
from galaxy.managers.session import GalaxySessionManager
|
||||
from galaxy.managers.sse import SSEConnectionManager
|
||||
from galaxy.managers.sse_dispatch import SSEEventDispatcher
|
||||
from galaxy.managers.tasks import (
|
||||
AsyncTasksManager,
|
||||
CeleryAsyncTasksManager,
|
||||
@@ -150,8 +153,12 @@ from galaxy.structured_app import (
|
||||
)
|
||||
from galaxy.tool_shed.cache import ToolShedRepositoryCache
|
||||
from galaxy.tool_shed.galaxy_install.client import InstallationTarget
|
||||
from galaxy.tool_shed.galaxy_install.installed_repository_manager import InstalledRepositoryManager
|
||||
from galaxy.tool_shed.galaxy_install.update_repository_manager import UpdateRepositoryManager
|
||||
from galaxy.tool_shed.galaxy_install.installed_repository_manager import (
|
||||
InstalledRepositoryManager,
|
||||
)
|
||||
from galaxy.tool_shed.galaxy_install.update_repository_manager import (
|
||||
UpdateRepositoryManager,
|
||||
)
|
||||
from galaxy.tool_util.data import ToolDataTableManager as BaseToolDataTableManager
|
||||
from galaxy.tool_util.deps import containers
|
||||
from galaxy.tool_util.deps.dependencies import AppInfo
|
||||
@@ -250,7 +257,11 @@ class SentryClientMixin:
|
||||
# "cannot find 'file_name' while searching for 'species_chromosomes.file_name'"]
|
||||
# If we don't do this issues are never properly grouped since by default the calling stack is inspected,
|
||||
# and that is always unique in cheetah as it is dynamically generated.
|
||||
event["fingerprint"] = [str(exc_value), str(exc_value.tool_version), str(exc_value.__cause__)]
|
||||
event["fingerprint"] = [
|
||||
str(exc_value),
|
||||
str(exc_value.tool_version),
|
||||
str(exc_value.__cause__),
|
||||
]
|
||||
event.setdefault("tags", {}).update(
|
||||
{
|
||||
"tool_is_latest": exc_value.is_latest,
|
||||
@@ -307,7 +318,10 @@ class MinimalGalaxyApplication(BasicSharedApp, HaltableContainer, SentryClientMi
|
||||
config_file = kwargs.get("global_conf", {}).get("__file__", None)
|
||||
if config_file:
|
||||
log.debug('Using "galaxy.ini" config file: %s', config_file)
|
||||
self._configure_models(check_migrate_databases=self.config.check_migrate_databases, config_file=config_file)
|
||||
self._configure_models(
|
||||
check_migrate_databases=self.config.check_migrate_databases,
|
||||
config_file=config_file,
|
||||
)
|
||||
# Security helper
|
||||
self._configure_security()
|
||||
self._register_singleton(IdEncodingHelper, self.security)
|
||||
@@ -411,7 +425,11 @@ class MinimalGalaxyApplication(BasicSharedApp, HaltableContainer, SentryClientMi
|
||||
index_help = getattr(self.config, "index_tool_help", True)
|
||||
self.toolbox_search = self._register_singleton(
|
||||
ToolBoxSearch,
|
||||
ToolBoxSearch(self.toolbox, index_dir=self.config.tool_search_index_dir, index_help=index_help),
|
||||
ToolBoxSearch(
|
||||
self.toolbox,
|
||||
index_dir=self.config.tool_search_index_dir,
|
||||
index_help=index_help,
|
||||
),
|
||||
)
|
||||
|
||||
@property
|
||||
@@ -496,8 +514,9 @@ class MinimalGalaxyApplication(BasicSharedApp, HaltableContainer, SentryClientMi
|
||||
templates = ConfiguredObjectStoreTemplates.from_app_config(self.config, vault_configured=vault_configured)
|
||||
self.object_store_templates = self._register_singleton(ConfiguredObjectStoreTemplates, templates)
|
||||
user_object_store_resolver = self._register_abstract_singleton(
|
||||
UserObjectStoreResolver, UserObjectStoreResolverImpl # type: ignore[type-abstract]
|
||||
) # Ignored because of https://github.com/python/mypy/issues/4717
|
||||
UserObjectStoreResolver, # type: ignore[type-abstract] # https://github.com/python/mypy/issues/4717
|
||||
UserObjectStoreResolverImpl,
|
||||
)
|
||||
kwds["user_object_store_resolver"] = user_object_store_resolver
|
||||
self.object_store = build_object_store_from_config(self.config, **kwds)
|
||||
|
||||
@@ -600,7 +619,13 @@ class GalaxyManagerApplication(MinimalManagerApp, MinimalGalaxyApplication):
|
||||
|
||||
model: GalaxyModelMapping
|
||||
|
||||
def __init__(self, configure_logging=True, use_converters=True, use_display_applications=True, **kwargs):
|
||||
def __init__(
|
||||
self,
|
||||
configure_logging=True,
|
||||
use_converters=True,
|
||||
use_display_applications=True,
|
||||
**kwargs,
|
||||
):
|
||||
super().__init__(**kwargs)
|
||||
self._register_singleton(MinimalManagerApp, self) # type: ignore[type-abstract]
|
||||
self.execution_timer_factory = self._register_singleton(
|
||||
@@ -617,7 +642,8 @@ class GalaxyManagerApplication(MinimalManagerApp, MinimalGalaxyApplication):
|
||||
# Initialize job metrics manager, needs to be in place before
|
||||
# config so per-destination modifications can be made.
|
||||
self.job_metrics = self._register_singleton(
|
||||
JobMetrics, JobMetrics(self.config.job_metrics_config_file, self.config.job_metrics, app=self)
|
||||
JobMetrics,
|
||||
JobMetrics(self.config.job_metrics_config_file, self.config.job_metrics, app=self),
|
||||
)
|
||||
# Initialize the job management configuration
|
||||
self.job_config = self._register_singleton(jobs.JobConfiguration)
|
||||
@@ -655,11 +681,30 @@ class GalaxyManagerApplication(MinimalManagerApp, MinimalGalaxyApplication):
|
||||
self.role_manager = self._register_singleton(RoleManager)
|
||||
self.job_manager = self._register_singleton(JobManager)
|
||||
|
||||
# AMQP connection + a publisher-ready queue worker. Celery worker processes
|
||||
# inherit this via GalaxyManagerApplication so they can fan out SSE events
|
||||
# to web workers (no consumer thread is started here — see bind_publisher).
|
||||
self.amqp_internal_connection_obj = galaxy.queues.connection_from_config(self.config)
|
||||
if self.amqp_internal_connection_obj is not None:
|
||||
self.queue_worker = self._register_singleton(GalaxyQueueWorker, GalaxyQueueWorker(self))
|
||||
self.queue_worker.bind_publisher()
|
||||
|
||||
# SSE dispatcher must be registered before NotificationManager so Lagom
|
||||
# can auto-inject the Optional[SSEEventDispatcher] constructor arg.
|
||||
self._register_singleton(
|
||||
SSEEventDispatcher,
|
||||
SSEEventDispatcher(
|
||||
queue_worker=getattr(self, "queue_worker", None),
|
||||
application_stack=self.application_stack,
|
||||
statsd_client=self.execution_timer_factory.galaxy_statsd_client,
|
||||
),
|
||||
)
|
||||
self.notification_manager = self._register_singleton(NotificationManager)
|
||||
self.interactivetool_manager = InteractiveToolManager(self)
|
||||
|
||||
self.task_manager = self._register_abstract_singleton(
|
||||
AsyncTasksManager, CeleryAsyncTasksManager # type: ignore[type-abstract] # https://github.com/python/mypy/issues/4717
|
||||
AsyncTasksManager, # type: ignore[type-abstract] # https://github.com/python/mypy/issues/4717
|
||||
CeleryAsyncTasksManager,
|
||||
)
|
||||
|
||||
# ConfiguredFileSources
|
||||
@@ -671,10 +716,12 @@ class GalaxyManagerApplication(MinimalManagerApp, MinimalGalaxyApplication):
|
||||
self._register_singleton(FileSourcePluginLoader, file_source_plugin_loader)
|
||||
self.file_source_templates = self._register_singleton(ConfiguredFileSourceTemplates, templates)
|
||||
self._register_singleton(
|
||||
UserDefinedFileSourcesConfig, UserDefinedFileSourcesConfig.from_app_config(self.config)
|
||||
UserDefinedFileSourcesConfig,
|
||||
UserDefinedFileSourcesConfig.from_app_config(self.config),
|
||||
)
|
||||
user_defined_file_sources = self._register_abstract_singleton(
|
||||
UserDefinedFileSources, UserDefinedFileSourcesImpl # type: ignore[type-abstract] # https://github.com/python/mypy/issues/4717
|
||||
UserDefinedFileSources, # type: ignore[type-abstract] # https://github.com/python/mypy/issues/4717
|
||||
UserDefinedFileSourcesImpl,
|
||||
)
|
||||
configured_file_source_conf: ConfiguredFileSourcesConf = ConfiguredFileSourcesConf.from_app_config(self.config)
|
||||
file_sources = ConfiguredFileSources(
|
||||
@@ -690,7 +737,8 @@ class GalaxyManagerApplication(MinimalManagerApp, MinimalGalaxyApplication):
|
||||
# Load security policy.
|
||||
self.security_agent = self.model.security_agent
|
||||
self.host_security_agent = galaxy.model.security.HostAgent(
|
||||
self.security_agent.sa_session, permitted_actions=self.security_agent.permitted_actions
|
||||
self.security_agent.sa_session,
|
||||
permitted_actions=self.security_agent.permitted_actions,
|
||||
)
|
||||
|
||||
# We need the datatype registry for running certain tasks that modify HDAs, and to build the registry we need
|
||||
@@ -791,6 +839,7 @@ class UniverseApplication(StructuredApp, GalaxyManagerApplication, InstallationT
|
||||
("queue worker", self._shutdown_queue_worker),
|
||||
("file watcher", self._shutdown_watcher),
|
||||
("database heartbeat", self._shutdown_database_heartbeat),
|
||||
("history audit monitor", self._shutdown_history_audit_monitor),
|
||||
("workflow scheduler", self._shutdown_scheduling_manager),
|
||||
("object store", self._shutdown_object_store),
|
||||
("job manager", self._shutdown_job_manager),
|
||||
@@ -805,21 +854,33 @@ class UniverseApplication(StructuredApp, GalaxyManagerApplication, InstallationT
|
||||
# A lot of postfork initialization depends on the server name, ensure it is set immediately after forking before other postfork functions
|
||||
self.application_stack.register_postfork_function(self.application_stack.set_postfork_server_name, self)
|
||||
self.config.reload_sanitize_allowlist(explicit="sanitize_allowlist_file" in kwargs)
|
||||
self.amqp_internal_connection_obj = galaxy.queues.connection_from_config(self.config)
|
||||
# queue_worker *can* be initialized with a queue, but here we don't
|
||||
# want to and we'll allow postfork to bind and start it.
|
||||
self.queue_worker = self._register_singleton(GalaxyQueueWorker, GalaxyQueueWorker(self))
|
||||
# amqp_internal_connection_obj and queue_worker are built in GalaxyManagerApplication
|
||||
# (so Celery workers also get a publisher); here we only register the consumer path,
|
||||
# which is started later via the application_stack postfork hook.
|
||||
# SSE connection manager for real-time notification push.
|
||||
# Consumed via ``depends(SSEConnectionManager)`` / ``app[SSEConnectionManager]``,
|
||||
# so no module-level attribute is needed — keep the container wiring only.
|
||||
self._register_singleton(
|
||||
SSEConnectionManager,
|
||||
SSEConnectionManager(
|
||||
statsd_client=self.execution_timer_factory.galaxy_statsd_client,
|
||||
),
|
||||
)
|
||||
|
||||
# AI agent registry and service
|
||||
agent_registry = build_agent_registry(self.config)
|
||||
self._register_singleton(AgentRegistry, agent_registry)
|
||||
self._register_singleton(AgentService, AgentService(self.config, JobQueryManager(self), agent_registry))
|
||||
self._register_singleton(
|
||||
AgentService,
|
||||
AgentService(self.config, JobQueryManager(self), agent_registry),
|
||||
)
|
||||
|
||||
self.dependency_resolvers_view = self._register_singleton(
|
||||
DependencyResolversView, DependencyResolversView(self)
|
||||
)
|
||||
self.test_data_resolver = self._register_singleton(
|
||||
TestDataResolver, TestDataResolver(file_dirs=self.config.tool_test_data_directories)
|
||||
TestDataResolver,
|
||||
TestDataResolver(file_dirs=self.config.tool_test_data_directories),
|
||||
)
|
||||
self.api_keys_manager = self._register_singleton(ApiKeyManager)
|
||||
|
||||
@@ -876,7 +937,9 @@ class UniverseApplication(StructuredApp, GalaxyManagerApplication, InstallationT
|
||||
# Start the heartbeat process if configured and available
|
||||
if self.config.use_heartbeat:
|
||||
self.heartbeat = heartbeat.Heartbeat(
|
||||
self.config, period=self.config.heartbeat_interval, fname=self.config.heartbeat_log
|
||||
self.config,
|
||||
period=self.config.heartbeat_interval,
|
||||
fname=self.config.heartbeat_log,
|
||||
)
|
||||
self.heartbeat.daemon = True
|
||||
self.application_stack.register_postfork_function(self.heartbeat.start)
|
||||
@@ -886,7 +949,9 @@ class UniverseApplication(StructuredApp, GalaxyManagerApplication, InstallationT
|
||||
from galaxy.authnz import managers
|
||||
|
||||
self.authnz_manager = managers.AuthnzManager(
|
||||
self, self.config.oidc_config_file, self.config.oidc_backends_config_file
|
||||
self,
|
||||
self.config.oidc_config_file,
|
||||
self.config.oidc_backends_config_file,
|
||||
)
|
||||
|
||||
# If there is only a single external authentication provider in use
|
||||
@@ -923,7 +988,12 @@ class UniverseApplication(StructuredApp, GalaxyManagerApplication, InstallationT
|
||||
self.workflow_completion_hook_registry,
|
||||
)
|
||||
self.application_stack.register_postfork_function(self.workflow_completion_monitor.start)
|
||||
self.haltables.append(("WorkflowCompletionMonitor", self.workflow_completion_monitor.shutdown_monitor))
|
||||
self.haltables.append(
|
||||
(
|
||||
"WorkflowCompletionMonitor",
|
||||
self.workflow_completion_monitor.shutdown_monitor,
|
||||
)
|
||||
)
|
||||
|
||||
# Start the job manager
|
||||
self.application_stack.register_postfork_function(self.job_manager.start)
|
||||
@@ -942,12 +1012,23 @@ class UniverseApplication(StructuredApp, GalaxyManagerApplication, InstallationT
|
||||
self.database_heartbeat.add_change_callback(self.watchers.change_state)
|
||||
self.application_stack.register_postfork_function(self.database_heartbeat.start)
|
||||
|
||||
# History audit monitor for SSE-based history updates. The monitor only
|
||||
# runs on the single process elected via DatabaseHeartbeat's
|
||||
# is_history_audit_monitor role — a standalone ``galaxy-sse-monitor``
|
||||
# daemon wins that election when present, otherwise one webapp picks it
|
||||
# up. start/stop are driven by heartbeat role transitions rather than
|
||||
# postfork, so the monitor cleanly migrates when the leader dies.
|
||||
if self.config.enable_sse_updates:
|
||||
monitor = self._register_singleton(HistoryAuditMonitor)
|
||||
self.database_heartbeat.add_audit_monitor_change_callback(monitor.on_role_change)
|
||||
|
||||
# Start web stack message handling
|
||||
self.application_stack.register_postfork_function(self.application_stack.start)
|
||||
self.application_stack.register_postfork_function(self.queue_worker.bind_and_start)
|
||||
# Reload toolbox to pick up changes to toolbox made after master was ready
|
||||
self.application_stack.register_postfork_function(
|
||||
lambda: reload_toolbox(self, save_integrated_tool_panel=False), post_fork_only=True
|
||||
lambda: reload_toolbox(self, save_integrated_tool_panel=False),
|
||||
post_fork_only=True,
|
||||
)
|
||||
# Delay toolbox index until after startup
|
||||
self.application_stack.register_postfork_function(
|
||||
@@ -975,6 +1056,11 @@ class UniverseApplication(StructuredApp, GalaxyManagerApplication, InstallationT
|
||||
def _shutdown_database_heartbeat(self):
|
||||
self.database_heartbeat.shutdown()
|
||||
|
||||
def _shutdown_history_audit_monitor(self):
|
||||
if not self.config.enable_sse_updates:
|
||||
return
|
||||
self[HistoryAuditMonitor].shutdown()
|
||||
|
||||
def _shutdown_scheduling_manager(self):
|
||||
self.workflow_scheduling_manager.shutdown()
|
||||
|
||||
|
||||
@@ -119,6 +119,7 @@ class MockApp(di.Container, GalaxyDataTestApp):
|
||||
history_manager: HistoryManager
|
||||
job_metrics: JobMetrics
|
||||
vault: Optional[Vault] = None
|
||||
execution_timer_factory: Any
|
||||
stop: bool
|
||||
is_webapp: bool = True
|
||||
|
||||
@@ -159,7 +160,7 @@ class MockApp(di.Container, GalaxyDataTestApp):
|
||||
self.application_stack = ApplicationStack()
|
||||
self.auth_manager = AuthManager(self.config)
|
||||
self.user_manager = UserManager(cast(BasicSharedApp, self))
|
||||
self.execution_timer_factory = Bunch(get_timer=StructuredExecutionTimer)
|
||||
self.execution_timer_factory = Bunch(get_timer=StructuredExecutionTimer, galaxy_statsd_client=None)
|
||||
self.interactivetool_manager = Bunch(create_interactivetool=lambda *args, **kwargs: None)
|
||||
self.is_job_handler = False
|
||||
self.biotools_metadata_source = None
|
||||
|
||||
@@ -798,7 +798,6 @@ def _send_oidc_profile_update_notification(trans, user, updates: list[str]) -> N
|
||||
NotificationVariant,
|
||||
PersonalNotificationCategory,
|
||||
)
|
||||
from galaxy.webapps.galaxy.services.notifications import NotificationService
|
||||
|
||||
labels: dict[str, str] = {
|
||||
"email": "email address",
|
||||
@@ -819,7 +818,7 @@ def _send_oidc_profile_update_notification(trans, user, updates: list[str]) -> N
|
||||
),
|
||||
galaxy_url=None,
|
||||
)
|
||||
NotificationService(trans.app.notification_manager).send_notification_internal(request, force_sync=True)
|
||||
trans.app.notification_manager.send_notification_internal(request, force_sync=True)
|
||||
except Exception as exc:
|
||||
log.warning("OIDC profile update notification failed for user %s: %s", user.id, exc)
|
||||
|
||||
|
||||
@@ -251,6 +251,10 @@ def setup_periodic_tasks(config, celery_app):
|
||||
beat_schedule: dict[str, dict[str, Any]] = {}
|
||||
schedule_task("prune_history_audit_table", config.history_audit_table_prune_interval)
|
||||
schedule_task("cleanup_short_term_storage", config.short_term_storage_cleanup_interval)
|
||||
schedule_task("prune_kombu_sqla_transport", config.kombu_sqla_transport_cleanup_interval)
|
||||
|
||||
if config.statsd_host:
|
||||
schedule_task("emit_queue_metrics_task", config.queue_metrics_interval)
|
||||
|
||||
if config.enable_notification_system:
|
||||
schedule_task("cleanup_expired_notifications", config.expired_notifications_cleanup_interval)
|
||||
|
||||
@@ -0,0 +1,25 @@
|
||||
from celery.result import AsyncResult
|
||||
|
||||
from galaxy.schema.schema import AsyncTaskResultSummary
|
||||
|
||||
|
||||
def async_task_summary(async_result: AsyncResult) -> AsyncTaskResultSummary:
|
||||
name = None
|
||||
try:
|
||||
name = async_result.name
|
||||
except AttributeError:
|
||||
# if backend is disabled, we won't have this
|
||||
pass
|
||||
queue = None
|
||||
try:
|
||||
queue = async_result.queue
|
||||
except AttributeError:
|
||||
# if backend is disabled, we won't have this
|
||||
pass
|
||||
|
||||
return AsyncTaskResultSummary(
|
||||
id=str(async_result.id),
|
||||
ignored=async_result.ignored,
|
||||
name=name,
|
||||
queue=queue,
|
||||
)
|
||||
@@ -9,13 +9,17 @@ from typing import (
|
||||
Any,
|
||||
Optional,
|
||||
)
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from lagom.exceptions import UnresolvableType
|
||||
from sqlalchemy import (
|
||||
and_,
|
||||
create_engine,
|
||||
delete,
|
||||
exists,
|
||||
false,
|
||||
select,
|
||||
text,
|
||||
update,
|
||||
)
|
||||
|
||||
@@ -40,6 +44,8 @@ from galaxy.managers.lddas import LDDAManager
|
||||
from galaxy.managers.markdown_util import generate_branded_pdf
|
||||
from galaxy.managers.model_stores import ModelStoreManager
|
||||
from galaxy.managers.notification import NotificationManager
|
||||
from galaxy.managers.queue_metrics import emit_queue_metrics
|
||||
from galaxy.managers.sse import SSEConnectionManager
|
||||
from galaxy.managers.tool_data import ToolDataImportManager
|
||||
from galaxy.managers.workflow_completion import WorkflowCompletionManager
|
||||
from galaxy.metadata.set_metadata import set_metadata_portable
|
||||
@@ -76,7 +82,10 @@ from galaxy.security.vault import (
|
||||
Vault,
|
||||
)
|
||||
from galaxy.short_term_storage import ShortTermStorageMonitor
|
||||
from galaxy.structured_app import MinimalManagerApp
|
||||
from galaxy.structured_app import (
|
||||
MinimalManagerApp,
|
||||
StructuredApp,
|
||||
)
|
||||
from galaxy.tools import create_tool_from_representation
|
||||
from galaxy.tools.data_fetch import do_fetch
|
||||
from galaxy.util import galaxy_directory
|
||||
@@ -591,6 +600,38 @@ def prune_history_audit_table(sa_session: galaxy_scoped_session):
|
||||
model.HistoryAudit.prune(sa_session)
|
||||
|
||||
|
||||
@galaxy_task(action="cleaning up Kombu SQLAlchemy transport")
|
||||
def prune_kombu_sqla_transport(config: GalaxyAppConfiguration):
|
||||
"""Delete fully-consumed rows from the Kombu SQLAlchemy transport tables.
|
||||
|
||||
Kombu's SQLAlchemy transport marks consumed messages with ``visible=0`` but
|
||||
never deletes them — without this task the control-queue tables grow
|
||||
without bound on the default on-disk sqlite broker. On AMQP / Redis the
|
||||
broker has native TTL, so this task is a no-op.
|
||||
"""
|
||||
broker_url = config.amqp_internal_connection
|
||||
if not broker_url:
|
||||
log.debug("kombu cleanup: no broker URL configured, skipping")
|
||||
return
|
||||
scheme = urlparse(broker_url).scheme
|
||||
if not scheme.startswith("sqlalchemy"):
|
||||
log.debug("kombu cleanup: broker scheme %s is not sqlalchemy, skipping", scheme)
|
||||
return
|
||||
|
||||
# Kombu's SQLA transport URL is ``sqlalchemy+<dialect>://...``. Strip the
|
||||
# ``sqlalchemy+`` prefix to get an engine URL we can hand to SQLAlchemy.
|
||||
sa_url = broker_url[len("sqlalchemy+") :] if broker_url.startswith("sqlalchemy+") else broker_url
|
||||
engine = create_engine(sa_url)
|
||||
try:
|
||||
with engine.begin() as conn:
|
||||
result = conn.execute(text("DELETE FROM kombu_message WHERE visible = 0"))
|
||||
log.info("kombu cleanup: deleted %s consumed messages", result.rowcount)
|
||||
except Exception:
|
||||
log.exception("kombu cleanup: failed to prune kombu_message")
|
||||
finally:
|
||||
engine.dispose()
|
||||
|
||||
|
||||
@galaxy_task(action="clean up short term storage")
|
||||
def cleanup_short_term_storage(storage_monitor: ShortTermStorageMonitor):
|
||||
"""Cleanup short term storage."""
|
||||
@@ -628,6 +669,28 @@ def dispatch_pending_notifications(notification_manager: NotificationManager):
|
||||
log.info(f"Successfully dispatched {count} notifications.")
|
||||
|
||||
|
||||
@galaxy_task(action="emit queue and SSE observability metrics")
|
||||
def emit_queue_metrics_task(app: StructuredApp):
|
||||
"""Sample control-queue depth, SSE connection count, and worker rows → statsd.
|
||||
|
||||
Resolves the narrow collaborators ``emit_queue_metrics`` needs from the app
|
||||
container and passes them in — keeps the emitter module free of
|
||||
``StructuredApp`` service-locator lookups.
|
||||
"""
|
||||
try:
|
||||
sse_manager: Optional[SSEConnectionManager] = app[SSEConnectionManager]
|
||||
except UnresolvableType:
|
||||
sse_manager = None
|
||||
|
||||
emit_queue_metrics(
|
||||
statsd_client=app.execution_timer_factory.galaxy_statsd_client,
|
||||
connection=app.amqp_internal_connection_obj,
|
||||
application_stack=app.application_stack,
|
||||
model=app.model,
|
||||
sse_manager=sse_manager,
|
||||
)
|
||||
|
||||
|
||||
@galaxy_task(action="clean up job working directories")
|
||||
def cleanup_jwds(sa_session: galaxy_scoped_session, object_store: BaseObjectStore, config: GalaxyAppConfiguration):
|
||||
"""Cleanup job working directories for failed jobs that are older than X days"""
|
||||
|
||||
@@ -512,6 +512,16 @@ galaxy:
|
||||
# history_audit database table. Set to 0 to disable pruning.
|
||||
#history_audit_table_prune_interval: 3600
|
||||
|
||||
# Time (in seconds) between attempts to delete fully-consumed rows
|
||||
# from the Kombu SQLAlchemy transport tables (``kombu_message``). Only
|
||||
# relevant when ``amqp_internal_connection`` uses a ``sqlalchemy+*``
|
||||
# scheme (the default with an on-disk control.sqlite); the SQLAlchemy
|
||||
# transport has no built-in TTL, so without this task the tables grow
|
||||
# unbounded. The task no-ops on non-SQLAlchemy brokers (RabbitMQ/Redis
|
||||
# honor per-message expiration natively). Set to 0 to disable the
|
||||
# cleanup task.
|
||||
#kombu_sqla_transport_cleanup_interval: 900
|
||||
|
||||
# Where dataset files are stored. It must be accessible at the same
|
||||
# path on any cluster nodes that will run Galaxy jobs, unless using
|
||||
# Pulsar. The default value has been changed from 'files' to 'objects'
|
||||
@@ -1962,6 +1972,11 @@ galaxy:
|
||||
# really. Do not set this in production environments.
|
||||
#statsd_mock_calls: false
|
||||
|
||||
# How often (in seconds) the Celery beat task emits queue-depth,
|
||||
# SSE-connection, and WorkerProcess gauges. Only active when
|
||||
# statsd_host is set. Set to 0 to disable.
|
||||
#queue_metrics_interval: 15
|
||||
|
||||
# Add an option to the library upload form which allows administrators
|
||||
# to upload a directory of files.
|
||||
#library_import_dir: null
|
||||
@@ -3129,12 +3144,36 @@ galaxy:
|
||||
# Message to display on the export citations tool page
|
||||
#citations_export_message_html: When writing up your analysis, remember to include all references that should be cited in order to completely describe your work. Also, please remember to <a href="https://galaxyproject.org/citing-galaxy">cite Galaxy</a>.
|
||||
|
||||
# Enables real-time updates via Server-Sent Events (SSE), replacing
|
||||
# the history (3 s), entry-point (10 s) and notification (30 s)
|
||||
# polling loops with push events delivered over a single
|
||||
# ``/api/events/stream`` connection per browser tab. A background
|
||||
# monitor watches for history changes (via PostgreSQL LISTEN/NOTIFY,
|
||||
# or audit-table polling as a fallback for SQLite); entry-point
|
||||
# changes are dispatched directly from the code paths that mutate
|
||||
# them; in-app notifications and broadcasts are pushed when
|
||||
# ``enable_notification_system`` is also true. When disabled, polling
|
||||
# remains the source of updates for all three. See the admin guide
|
||||
# "Server-Sent Events for real-time updates" for the full
|
||||
# architecture, monitoring guidance and proxy configuration.
|
||||
#enable_sse_updates: false
|
||||
|
||||
# The interval in seconds between history audit table polls when using
|
||||
# the polling fallback (SQLite or when PostgreSQL LISTEN/NOTIFY is
|
||||
# unavailable). Only used when enable_sse_updates is true. Lower
|
||||
# values mean faster updates but more database queries. Recommended
|
||||
# range: 1-5 seconds.
|
||||
#history_audit_monitor_poll_interval: 2
|
||||
|
||||
# Enables the Notification System integrated in Galaxy.
|
||||
# Users can receive automatic notifications when a certain resource is
|
||||
# shared with them or when some long running operations have finished,
|
||||
# etc.
|
||||
# The system allows notification scheduling and expiration, and users
|
||||
# can opt-out of specific notification categories or channels.
|
||||
# Delivery is push-based via Server-Sent Events when
|
||||
# ``enable_sse_updates`` is also true, and falls back to 30-second
|
||||
# polling against ``/api/notifications/status`` otherwise.
|
||||
# Admins can schedule and broadcast notifications that will be visible
|
||||
# to all users, including special server-wide announcements such as
|
||||
# scheduled maintenance, high load warnings, and event announcements,
|
||||
|
||||
@@ -222,6 +222,18 @@ mapping:
|
||||
Time (in seconds) between attempts to remove old rows from the history_audit database table.
|
||||
Set to 0 to disable pruning.
|
||||
|
||||
kombu_sqla_transport_cleanup_interval:
|
||||
type: int
|
||||
default: 900
|
||||
required: false
|
||||
desc: |
|
||||
Time (in seconds) between attempts to delete fully-consumed rows from the Kombu SQLAlchemy
|
||||
transport tables (``kombu_message``). Only relevant when ``amqp_internal_connection`` uses a
|
||||
``sqlalchemy+*`` scheme (the default with an on-disk control.sqlite); the SQLAlchemy transport
|
||||
has no built-in TTL, so without this task the tables grow unbounded. The task no-ops on
|
||||
non-SQLAlchemy brokers (RabbitMQ/Redis honor per-message expiration natively). Set to 0 to
|
||||
disable the cleanup task.
|
||||
|
||||
file_path:
|
||||
type: str
|
||||
default: objects
|
||||
@@ -2512,6 +2524,14 @@ mapping:
|
||||
Mock out statsd client calls - only used by testing infrastructure really.
|
||||
Do not set this in production environments.
|
||||
|
||||
queue_metrics_interval:
|
||||
type: int
|
||||
default: 15
|
||||
required: false
|
||||
desc: |
|
||||
How often (in seconds) the Celery beat task emits queue-depth, SSE-connection,
|
||||
and WorkerProcess gauges. Only active when statsd_host is set. Set to 0 to disable.
|
||||
|
||||
library_import_dir:
|
||||
type: str
|
||||
required: false
|
||||
@@ -4293,6 +4313,33 @@ mapping:
|
||||
desc: |
|
||||
Message to display on the export citations tool page
|
||||
|
||||
enable_sse_updates:
|
||||
type: bool
|
||||
default: false
|
||||
required: false
|
||||
desc: |
|
||||
Enables real-time updates via Server-Sent Events (SSE), replacing the
|
||||
history (3 s), entry-point (10 s) and notification (30 s) polling loops
|
||||
with push events delivered over a single ``/api/events/stream``
|
||||
connection per browser tab. A background monitor watches for history
|
||||
changes (via PostgreSQL LISTEN/NOTIFY, or audit-table polling as a
|
||||
fallback for SQLite); entry-point changes are dispatched directly from
|
||||
the code paths that mutate them; in-app notifications and broadcasts
|
||||
are pushed when ``enable_notification_system`` is also true.
|
||||
When disabled, polling remains the source of updates for all three.
|
||||
See the admin guide "Server-Sent Events for real-time updates" for the
|
||||
full architecture, monitoring guidance and proxy configuration.
|
||||
|
||||
history_audit_monitor_poll_interval:
|
||||
type: int
|
||||
default: 2
|
||||
required: false
|
||||
desc: |
|
||||
The interval in seconds between history audit table polls when using the polling
|
||||
fallback (SQLite or when PostgreSQL LISTEN/NOTIFY is unavailable). Only used when
|
||||
enable_sse_updates is true. Lower values mean faster updates but more
|
||||
database queries. Recommended range: 1-5 seconds.
|
||||
|
||||
enable_notification_system:
|
||||
type: bool
|
||||
default: false
|
||||
@@ -4304,6 +4351,10 @@ mapping:
|
||||
|
||||
The system allows notification scheduling and expiration, and users can opt-out of specific notification categories or channels.
|
||||
|
||||
Delivery is push-based via Server-Sent Events when ``enable_sse_updates``
|
||||
is also true, and falls back to 30-second polling against
|
||||
``/api/notifications/status`` otherwise.
|
||||
|
||||
Admins can schedule and broadcast notifications that will be visible to all users, including special server-wide announcements such as scheduled maintenance, high load warnings, and event announcements, to name a few examples.
|
||||
|
||||
enable_mcp_server:
|
||||
|
||||
@@ -18,6 +18,7 @@ tomli==2.4.1 ; python_full_version < '3.11'
|
||||
types-awscrt==0.31.3
|
||||
types-bleach==6.3.0.20260408
|
||||
types-boto==2.49.18.20241019
|
||||
types-cachetools==6.2.0.20260408
|
||||
types-contextvars==2.4.7.3
|
||||
types-dataclasses==0.6.6
|
||||
types-docutils==0.22.3.20260408
|
||||
|
||||
@@ -229,6 +229,7 @@ class ConfigSerializer(base.ModelSerializer):
|
||||
"tool_training_recommendations_link": _use_config,
|
||||
"tool_training_recommendations_api_url": _use_config,
|
||||
"enable_notification_system": _use_config,
|
||||
"enable_sse_updates": _use_config,
|
||||
"instance_resource_url": _use_config,
|
||||
"instance_access_url": _use_config,
|
||||
"organization_name": _use_config,
|
||||
|
||||
@@ -0,0 +1,318 @@
|
||||
"""Monitor for history audit table changes.
|
||||
|
||||
Detects history changes via PostgreSQL LISTEN/NOTIFY (instant) or by polling
|
||||
the history_audit table (SQLite fallback). Dispatches SSE events to connected
|
||||
users via Kombu control tasks.
|
||||
|
||||
Only active when ``enable_sse_updates`` is True in the Galaxy config.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import select
|
||||
import threading
|
||||
import time
|
||||
from collections import (
|
||||
defaultdict,
|
||||
OrderedDict,
|
||||
)
|
||||
from collections.abc import Iterator
|
||||
from datetime import (
|
||||
datetime,
|
||||
timedelta,
|
||||
)
|
||||
from typing import (
|
||||
Any,
|
||||
Optional,
|
||||
)
|
||||
|
||||
from sqlalchemy import select as sa_select
|
||||
from sqlalchemy.engine import Engine
|
||||
|
||||
from galaxy.config import GalaxyAppConfiguration
|
||||
from galaxy.managers.sse_dispatch import SSEEventDispatcher
|
||||
from galaxy.model import (
|
||||
GalaxySessionToHistoryAssociation,
|
||||
History,
|
||||
HistoryAudit,
|
||||
)
|
||||
from galaxy.model.mapping import GalaxyModelMapping
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
CHANNEL_NAME = "galaxy_history_update"
|
||||
OWNER_CACHE_MAX = 10_000
|
||||
DEBOUNCE_SECONDS = 0.2
|
||||
|
||||
|
||||
class _PgListenAdapter:
|
||||
"""Thin DBAPI-level adapter for PostgreSQL LISTEN/NOTIFY.
|
||||
|
||||
Hides the receiving-API differences between psycopg2 (``conn.poll()`` +
|
||||
``conn.notifies`` list, driven by ``select.select``) and psycopg3
|
||||
(``conn.notifies(timeout=...)`` generator). The SA URL is used to inherit
|
||||
DSN / SSL / auth config, but the connection itself is opened directly with
|
||||
the DBAPI driver so it stays outside the SA pool — LISTEN connections must
|
||||
live for the lifetime of the monitor and never be returned to the pool.
|
||||
"""
|
||||
|
||||
# Typed once at the class level so both driver branches can assign without
|
||||
# re-annotating the same attribute.
|
||||
_conn: Any
|
||||
driver: str
|
||||
|
||||
def __init__(self, engine: Engine) -> None:
|
||||
# Strip the SA ``+driver`` suffix so the raw DBAPI libraries accept the URL.
|
||||
dsn = engine.url.set(drivername="postgresql").render_as_string(hide_password=False)
|
||||
driver = engine.dialect.driver
|
||||
if driver == "psycopg":
|
||||
import psycopg # conditional: psycopg3 driver
|
||||
|
||||
self._conn = psycopg.connect(dsn, autocommit=True)
|
||||
self.driver = "psycopg3"
|
||||
else:
|
||||
import psycopg2 # conditional: psycopg2 driver
|
||||
|
||||
self._conn = psycopg2.connect(dsn)
|
||||
self._conn.autocommit = True # same effect as set_isolation_level(AUTOCOMMIT)
|
||||
self.driver = "psycopg2"
|
||||
|
||||
def listen(self, channel: str) -> None:
|
||||
with self._conn.cursor() as cursor:
|
||||
cursor.execute(f"LISTEN {channel};")
|
||||
|
||||
def close(self) -> None:
|
||||
try:
|
||||
self._conn.close()
|
||||
except Exception:
|
||||
log.debug("Error closing LISTEN connection", exc_info=True)
|
||||
|
||||
def poll(self, timeout: float) -> Iterator[str]:
|
||||
"""Block up to ``timeout`` seconds and yield notification payloads.
|
||||
|
||||
Returns an empty iterator on timeout so callers can uniformly treat
|
||||
"nothing received in this tick" regardless of driver.
|
||||
"""
|
||||
if self.driver == "psycopg3":
|
||||
# psycopg3: notifies() is a blocking generator bounded by ``timeout``.
|
||||
yield from (n.payload for n in self._conn.notifies(timeout=timeout))
|
||||
return
|
||||
# psycopg2: block on the socket via select(), then drain notifies list.
|
||||
if select.select([self._conn], [], [], timeout) == ([], [], []):
|
||||
return
|
||||
self._conn.poll()
|
||||
while self._conn.notifies:
|
||||
yield self._conn.notifies.pop(0).payload
|
||||
|
||||
|
||||
class HistoryAuditMonitor:
|
||||
"""Background thread that monitors history_audit for changes and dispatches SSE events.
|
||||
|
||||
On PostgreSQL: uses LISTEN/NOTIFY for instant notification.
|
||||
On SQLite: polls history_audit table at a configurable interval.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: GalaxyAppConfiguration,
|
||||
model: GalaxyModelMapping,
|
||||
sse_dispatcher: SSEEventDispatcher,
|
||||
) -> None:
|
||||
self._config = config
|
||||
self._model = model
|
||||
self._dispatcher = sse_dispatcher
|
||||
self.poll_interval: int = config.history_audit_monitor_poll_interval
|
||||
self._is_postgres: bool = "postgres" in model.engine.name
|
||||
self._exit = threading.Event()
|
||||
self._thread: Optional[threading.Thread] = None
|
||||
self._active = False
|
||||
# Bounded LRU cache: history_id -> (user_id, session_ids), refreshed on miss.
|
||||
# For registered-owned histories: (user_id, ()); for anonymous histories:
|
||||
# (None, (session_id, ...)) — a history can be associated with multiple
|
||||
# sessions via GalaxySessionToHistoryAssociation.
|
||||
self._history_owner_cache: OrderedDict[int, tuple[Optional[int], tuple[int, ...]]] = OrderedDict()
|
||||
|
||||
def start(self) -> None:
|
||||
if self._active:
|
||||
return
|
||||
self._active = True
|
||||
self._exit.clear() # allow restart after a previous shutdown
|
||||
target = self._listen_postgres if self._is_postgres else self._poll_audit_table
|
||||
self._thread = threading.Thread(
|
||||
target=target,
|
||||
name="history_audit_monitor",
|
||||
daemon=True,
|
||||
)
|
||||
self._thread.start()
|
||||
log.info(
|
||||
"HistoryAuditMonitor started (mode=%s, interval=%ds)",
|
||||
"pg_listen" if self._is_postgres else "poll",
|
||||
self.poll_interval,
|
||||
)
|
||||
|
||||
def shutdown(self) -> None:
|
||||
if not self._active:
|
||||
return
|
||||
self._active = False
|
||||
self._exit.set()
|
||||
if self._thread:
|
||||
self._thread.join(timeout=5)
|
||||
self._thread = None
|
||||
log.info("HistoryAuditMonitor stopped")
|
||||
|
||||
def on_role_change(self, is_leader: bool) -> None:
|
||||
"""Heartbeat callback: start/stop the monitor as this process's election state changes."""
|
||||
if is_leader:
|
||||
self.start()
|
||||
else:
|
||||
self.shutdown()
|
||||
|
||||
# --- PostgreSQL LISTEN/NOTIFY mode ---
|
||||
|
||||
def _listen_postgres(self) -> None:
|
||||
"""LISTEN for history update notifications.
|
||||
|
||||
Works against both psycopg2 and psycopg3 — whichever driver the SA
|
||||
engine was built with. Falls back to the SQLite polling path if the
|
||||
DBAPI driver can't be imported or the initial LISTEN fails.
|
||||
"""
|
||||
try:
|
||||
adapter = _PgListenAdapter(self._model.engine)
|
||||
adapter.listen(CHANNEL_NAME)
|
||||
except Exception:
|
||||
log.warning(
|
||||
"Failed to establish PostgreSQL LISTEN connection, falling back to polling",
|
||||
exc_info=True,
|
||||
)
|
||||
self._poll_audit_table()
|
||||
return
|
||||
|
||||
log.debug("LISTEN %s established (driver=%s)", CHANNEL_NAME, adapter.driver)
|
||||
pending: dict[int, float] = {} # history_id -> first_seen_time
|
||||
|
||||
try:
|
||||
while not self._exit.is_set():
|
||||
received_any = False
|
||||
for payload in adapter.poll(self.poll_interval):
|
||||
received_any = True
|
||||
try:
|
||||
history_id = int(payload)
|
||||
except (ValueError, TypeError):
|
||||
continue
|
||||
pending.setdefault(history_id, time.monotonic())
|
||||
|
||||
if not received_any:
|
||||
# Timeout — flush anything that's been pending since last tick
|
||||
if pending:
|
||||
self._dispatch_history_updates(set(pending.keys()))
|
||||
pending.clear()
|
||||
continue
|
||||
|
||||
# Debounce: dispatch events that have been pending long enough
|
||||
now = time.monotonic()
|
||||
ready = {hid for hid, ts in pending.items() if now - ts >= DEBOUNCE_SECONDS}
|
||||
if ready:
|
||||
self._dispatch_history_updates(ready)
|
||||
for hid in ready:
|
||||
del pending[hid]
|
||||
except Exception:
|
||||
log.exception("HistoryAuditMonitor LISTEN loop error")
|
||||
finally:
|
||||
adapter.close()
|
||||
|
||||
# --- SQLite polling fallback ---
|
||||
|
||||
def _poll_audit_table(self) -> None:
|
||||
"""Poll history_audit for recent changes."""
|
||||
last_check = datetime.utcnow() - timedelta(seconds=self.poll_interval)
|
||||
|
||||
while not self._exit.is_set():
|
||||
try:
|
||||
check_time = datetime.utcnow()
|
||||
stmt = (
|
||||
sa_select(HistoryAudit.history_id)
|
||||
.where(HistoryAudit.update_time > last_check)
|
||||
.group_by(HistoryAudit.history_id)
|
||||
)
|
||||
with self._model.new_session() as session:
|
||||
changed_ids = set(session.scalars(stmt).all())
|
||||
|
||||
if changed_ids:
|
||||
self._dispatch_history_updates(changed_ids)
|
||||
|
||||
last_check = check_time
|
||||
except Exception:
|
||||
log.exception("HistoryAuditMonitor poll error")
|
||||
|
||||
self._exit.wait(self.poll_interval)
|
||||
|
||||
# --- Common dispatch logic ---
|
||||
|
||||
def _dispatch_history_updates(self, history_ids: set[int]) -> None:
|
||||
"""Map history_ids to user_ids / session_ids and send Kombu control task.
|
||||
|
||||
Raw integer history IDs are sent across the control queue; encoding is
|
||||
deferred to the ``history_update`` task handler on the receiving side,
|
||||
keeping this manager free of presentation concerns.
|
||||
"""
|
||||
# Resolve owners for unknown history_ids
|
||||
unknown = history_ids - self._history_owner_cache.keys()
|
||||
if unknown:
|
||||
self._refresh_owner_cache(unknown)
|
||||
|
||||
user_updates: dict[str, list[int]] = defaultdict(list)
|
||||
session_updates: dict[str, list[int]] = defaultdict(list)
|
||||
for history_id in history_ids:
|
||||
entry = self._history_owner_cache.get(history_id)
|
||||
if entry is None:
|
||||
continue
|
||||
user_id, session_ids = entry
|
||||
if user_id is not None:
|
||||
user_updates[str(user_id)].append(history_id)
|
||||
else:
|
||||
for session_id in session_ids:
|
||||
session_updates[str(session_id)].append(history_id)
|
||||
|
||||
if not user_updates and not session_updates:
|
||||
return
|
||||
|
||||
self._dispatcher.history_update(
|
||||
user_updates=dict(user_updates),
|
||||
session_updates=dict(session_updates) if session_updates else None,
|
||||
)
|
||||
|
||||
def _refresh_owner_cache(self, history_ids: set[int]) -> None:
|
||||
"""Look up ownership for given history_ids and update the bounded cache.
|
||||
|
||||
Registered-owned histories resolve with just ``History.user_id``. For
|
||||
histories where ``user_id IS NULL`` we additionally fetch associated
|
||||
``galaxy_session.id`` values from ``GalaxySessionToHistoryAssociation``
|
||||
so the anonymous SSE dispatch path can target the right browser.
|
||||
"""
|
||||
try:
|
||||
with self._model.new_session() as session:
|
||||
stmt = sa_select(History.id, History.user_id).where(History.id.in_(history_ids))
|
||||
anon_history_ids: set[int] = set()
|
||||
for row in session.execute(stmt):
|
||||
hid, uid = row[0], row[1]
|
||||
self._history_owner_cache[hid] = (uid, ())
|
||||
self._history_owner_cache.move_to_end(hid)
|
||||
if uid is None:
|
||||
anon_history_ids.add(hid)
|
||||
|
||||
if anon_history_ids:
|
||||
assoc_stmt = sa_select(
|
||||
GalaxySessionToHistoryAssociation.history_id,
|
||||
GalaxySessionToHistoryAssociation.session_id,
|
||||
).where(GalaxySessionToHistoryAssociation.history_id.in_(anon_history_ids))
|
||||
sessions_by_history: dict[int, list[int]] = defaultdict(list)
|
||||
for row in session.execute(assoc_stmt):
|
||||
hid, sid = row[0], row[1]
|
||||
if sid is not None:
|
||||
sessions_by_history[hid].append(sid)
|
||||
for hid, sids in sessions_by_history.items():
|
||||
self._history_owner_cache[hid] = (None, tuple(sids))
|
||||
|
||||
while len(self._history_owner_cache) > OWNER_CACHE_MAX:
|
||||
self._history_owner_cache.popitem(last=False)
|
||||
except Exception:
|
||||
log.debug("Failed to refresh history owner cache", exc_info=True)
|
||||
@@ -6,6 +6,7 @@ from collections.abc import (
|
||||
)
|
||||
from typing import (
|
||||
Any,
|
||||
Optional,
|
||||
TYPE_CHECKING,
|
||||
Union,
|
||||
)
|
||||
@@ -28,6 +29,7 @@ from sqlalchemy import (
|
||||
)
|
||||
|
||||
from galaxy import exceptions
|
||||
from galaxy.managers.sse_dispatch import SSEEventDispatcher
|
||||
from galaxy.model import (
|
||||
InteractiveToolEntryPoint,
|
||||
Job,
|
||||
@@ -147,7 +149,11 @@ class InteractiveToolManager:
|
||||
Manager for dealing with InteractiveTools
|
||||
"""
|
||||
|
||||
def __init__(self, app: "MinimalManagerApp") -> None:
|
||||
def __init__(
|
||||
self,
|
||||
app: "MinimalManagerApp",
|
||||
dispatcher: Optional[SSEEventDispatcher] = None,
|
||||
) -> None:
|
||||
self.app = app
|
||||
self.security = app.security
|
||||
self.sa_session = app.model.context
|
||||
@@ -157,6 +163,12 @@ class InteractiveToolManager:
|
||||
app.config.interactivetoolsproxy_map or app.config.interactivetools_map,
|
||||
self.encoder.encode_id,
|
||||
)
|
||||
# Lagom can't auto-inject ``SSEEventDispatcher`` here because the
|
||||
# ``app: "MinimalManagerApp"`` hint is only a forward reference
|
||||
# (TYPE_CHECKING import), so ``get_type_hints`` on this signature
|
||||
# fails. Resolve through the container explicitly — ``resolve_or_none``
|
||||
# returns ``None`` for mocks/test apps that never registered one.
|
||||
self.dispatcher = dispatcher if dispatcher is not None else app.resolve_or_none(SSEEventDispatcher)
|
||||
|
||||
def create_entry_points(
|
||||
self, job: Job, tool: "Tool", entry_points=Union[Iterable[dict[str, Any]], None], flush: bool = True
|
||||
@@ -198,6 +210,17 @@ class InteractiveToolManager:
|
||||
configured.append(ep)
|
||||
if configured:
|
||||
self.sa_session.commit()
|
||||
# Fan out an SSE push so the user's browser can refresh the entry
|
||||
# point list immediately instead of waiting for the 10 s poll.
|
||||
# Anonymous jobs fall back to polling — ``push_to_user`` keys on
|
||||
# user_id, and anonymous clients sit in the broadcast-only set.
|
||||
if self.dispatcher is not None and job.user_id is not None:
|
||||
try:
|
||||
self.dispatcher.entry_point_update(user_id=job.user_id)
|
||||
except Exception:
|
||||
# The DB commit is authoritative; the SSE event is best
|
||||
# effort. Never let a dispatch failure poison the caller.
|
||||
log.exception("Failed to dispatch entry_point_update SSE event for job %s", job.id)
|
||||
return dict(not_configured=not_configured, configured=configured)
|
||||
|
||||
def save_entry_point(self, entry_point: InteractiveToolEntryPoint) -> None:
|
||||
|
||||
@@ -5,6 +5,7 @@ from typing import (
|
||||
cast,
|
||||
NamedTuple,
|
||||
Optional,
|
||||
Union,
|
||||
)
|
||||
from urllib.parse import urlparse
|
||||
|
||||
@@ -28,6 +29,7 @@ from sqlalchemy.sql import Select
|
||||
from typing_extensions import Protocol
|
||||
|
||||
from galaxy import util
|
||||
from galaxy.celery.helpers import async_task_summary
|
||||
from galaxy.config import (
|
||||
GalaxyAppConfiguration,
|
||||
templates,
|
||||
@@ -37,6 +39,7 @@ from galaxy.exceptions import (
|
||||
ObjectNotFound,
|
||||
)
|
||||
from galaxy.managers.markdown_util import to_html
|
||||
from galaxy.managers.sse_dispatch import SSEEventDispatcher
|
||||
from galaxy.model import (
|
||||
GroupRoleAssociation,
|
||||
Notification,
|
||||
@@ -49,6 +52,7 @@ from galaxy.model.scoped_session import galaxy_scoped_session
|
||||
from galaxy.schema.notifications import (
|
||||
AnyNotificationContent,
|
||||
BroadcastNotificationCreateRequest,
|
||||
BroadcastNotificationResponse,
|
||||
MandatoryNotificationCategory,
|
||||
MessageNotificationContent,
|
||||
NewSharedItemNotificationContent,
|
||||
@@ -56,14 +60,17 @@ from galaxy.schema.notifications import (
|
||||
NotificationCategorySettings,
|
||||
NotificationChannelSettings,
|
||||
NotificationCreateData,
|
||||
NotificationCreatedResponse,
|
||||
NotificationCreateRequest,
|
||||
NotificationRecipients,
|
||||
NotificationResponse,
|
||||
NotificationVariant,
|
||||
PersonalNotificationCategory,
|
||||
UpdateUserNotificationPreferencesRequest,
|
||||
UserNotificationPreferences,
|
||||
UserNotificationUpdateRequest,
|
||||
)
|
||||
from galaxy.schema.schema import AsyncTaskResultSummary
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@@ -94,9 +101,15 @@ class NotificationChannelPlugin(Protocol):
|
||||
class NotificationManager:
|
||||
"""Manager class to interact with the database models related with Notifications."""
|
||||
|
||||
def __init__(self, sa_session: galaxy_scoped_session, config: GalaxyAppConfiguration):
|
||||
def __init__(
|
||||
self,
|
||||
sa_session: galaxy_scoped_session,
|
||||
config: GalaxyAppConfiguration,
|
||||
sse_dispatcher: Optional[SSEEventDispatcher] = None,
|
||||
):
|
||||
self.sa_session = sa_session
|
||||
self.config = config
|
||||
self.sse_dispatcher = sse_dispatcher
|
||||
self.recipient_resolver = NotificationRecipientResolver(strategy=DefaultStrategy(sa_session))
|
||||
self.user_notification_columns: list[InstrumentedAttribute] = [
|
||||
Notification.id,
|
||||
@@ -164,8 +177,36 @@ class NotificationManager:
|
||||
notifications_sent = self._create_associations(notification, recipient_users)
|
||||
self.sa_session.commit()
|
||||
|
||||
# Push SSE events to connected users via control queue
|
||||
user_ids = [user.id for user in recipient_users]
|
||||
self._notify_users_via_sse(user_ids, notification)
|
||||
|
||||
return notification, notifications_sent
|
||||
|
||||
def send_notification_internal(
|
||||
self, request: NotificationCreateRequest, force_sync: bool = False
|
||||
) -> Union[NotificationCreatedResponse, AsyncTaskResultSummary]:
|
||||
"""Sends a notification to a list of recipients (users, groups or roles).
|
||||
|
||||
If `force_sync` is set to `True`, the notification recipients will be processed synchronously instead of
|
||||
in a background task.
|
||||
|
||||
Note: This function is meant for internal use from other callers that don't need to check sender permissions.
|
||||
"""
|
||||
if self.can_send_notifications_async and not force_sync:
|
||||
# Local import: galaxy.celery.tasks imports NotificationManager at module load,
|
||||
# so importing it at module level here would be a circular dependency.
|
||||
from galaxy.celery.tasks import send_notification_to_recipients_async
|
||||
|
||||
result = send_notification_to_recipients_async.delay(request)
|
||||
return async_task_summary(result)
|
||||
|
||||
notification, recipient_user_count = self.send_notification_to_recipients(request)
|
||||
return NotificationCreatedResponse(
|
||||
total_notifications_sent=recipient_user_count,
|
||||
notification=NotificationResponse.model_validate(notification),
|
||||
)
|
||||
|
||||
def _create_associations(self, notification: Notification, users: list[User]) -> int:
|
||||
success_count = 0
|
||||
for user in users:
|
||||
@@ -179,6 +220,26 @@ class NotificationManager:
|
||||
continue
|
||||
return success_count
|
||||
|
||||
def _notify_users_via_sse(self, user_ids: list[int], notification: Notification) -> None:
|
||||
"""Broadcast a control task to all workers to push SSE events to connected users."""
|
||||
if not self.sse_dispatcher or not user_ids:
|
||||
return
|
||||
try:
|
||||
payload = NotificationResponse.model_validate(notification).model_dump_json()
|
||||
self.sse_dispatcher.notify_users(user_ids, payload)
|
||||
except Exception:
|
||||
log.warning("Failed to send SSE notification event", exc_info=True)
|
||||
|
||||
def _notify_broadcast_via_sse(self, notification: Notification) -> None:
|
||||
"""Broadcast a control task to all workers to push SSE broadcast events."""
|
||||
if not self.sse_dispatcher:
|
||||
return
|
||||
try:
|
||||
payload = BroadcastNotificationResponse.model_validate(notification).model_dump_json()
|
||||
self.sse_dispatcher.notify_broadcast(payload)
|
||||
except Exception:
|
||||
log.warning("Failed to send SSE broadcast event", exc_info=True)
|
||||
|
||||
def dispatch_pending_notifications_via_channels(self) -> int:
|
||||
"""
|
||||
Dispatches all pending notifications to the users depending on the configured channels.
|
||||
@@ -273,6 +334,7 @@ class NotificationManager:
|
||||
notification = self._create_notification_model(request)
|
||||
self.sa_session.add(notification)
|
||||
self.sa_session.commit()
|
||||
self._notify_broadcast_via_sse(notification)
|
||||
return notification
|
||||
|
||||
def get_user_notification(self, user: User, notification_id: int, active_only: Optional[bool] = True):
|
||||
@@ -353,7 +415,10 @@ class NotificationManager:
|
||||
return result
|
||||
|
||||
def update_user_notifications(
|
||||
self, user: User, notification_ids: set[int], request: UserNotificationUpdateRequest
|
||||
self,
|
||||
user: User,
|
||||
notification_ids: set[int],
|
||||
request: UserNotificationUpdateRequest,
|
||||
) -> int:
|
||||
"""Updates a batch of notifications associated with the user using the requested values."""
|
||||
updated_row_count = 0
|
||||
@@ -447,7 +512,8 @@ class NotificationManager:
|
||||
UserNotificationAssociation.notification_id.in_(expired_notifications_stmt)
|
||||
)
|
||||
result = cast(
|
||||
CursorResult, self.sa_session.execute(delete_stmt, execution_options={"synchronize_session": False})
|
||||
CursorResult,
|
||||
self.sa_session.execute(delete_stmt, execution_options={"synchronize_session": False}),
|
||||
)
|
||||
deleted_associations_count = result.rowcount
|
||||
|
||||
@@ -474,7 +540,10 @@ class NotificationManager:
|
||||
return notification
|
||||
|
||||
def _user_notifications_query(
|
||||
self, user: User, since: Optional[datetime] = None, active_only: Optional[bool] = True
|
||||
self,
|
||||
user: User,
|
||||
since: Optional[datetime] = None,
|
||||
active_only: Optional[bool] = True,
|
||||
):
|
||||
stmt = (
|
||||
select(*self.user_notification_columns)
|
||||
@@ -552,7 +621,7 @@ class DefaultStrategy(NotificationRecipientResolverStrategy):
|
||||
user_ids_from_roles_stmt = self._get_all_user_ids_from_roles_query(all_role_ids)
|
||||
|
||||
union_stmt = union(user_ids_from_groups_stmt, user_ids_from_roles_stmt)
|
||||
user_ids_from_groups_and_roles = {id for id, in self.sa_session.execute(union_stmt)}
|
||||
user_ids_from_groups_and_roles = {id for (id,) in self.sa_session.execute(union_stmt)}
|
||||
unique_user_ids.update(user_ids_from_groups_and_roles)
|
||||
|
||||
stmt = select(User).where(User.id.in_(unique_user_ids))
|
||||
@@ -591,7 +660,7 @@ class DefaultStrategy(NotificationRecipientResolverStrategy):
|
||||
.where(GroupRoleAssociation.role_id.in_(role_ids))
|
||||
.distinct()
|
||||
)
|
||||
group_ids_from_roles = {id for id, in self.sa_session.execute(stmt) if id is not None}
|
||||
group_ids_from_roles = {id for (id,) in self.sa_session.execute(stmt) if id is not None}
|
||||
new_group_ids = group_ids_from_roles - processed_group_ids
|
||||
|
||||
# Get role IDs associated with any of the given group IDs
|
||||
@@ -601,7 +670,7 @@ class DefaultStrategy(NotificationRecipientResolverStrategy):
|
||||
.where(GroupRoleAssociation.group_id.in_(group_ids))
|
||||
.distinct()
|
||||
)
|
||||
role_ids_from_groups = {id for id, in self.sa_session.execute(stmt) if id is not None}
|
||||
role_ids_from_groups = {id for (id,) in self.sa_session.execute(stmt) if id is not None}
|
||||
new_role_ids = role_ids_from_groups - processed_role_ids
|
||||
|
||||
# Stop if there are no new group or role IDs to process
|
||||
@@ -713,7 +782,6 @@ class EmailNotificationTemplateBuilder(Protocol):
|
||||
|
||||
|
||||
class MessageEmailNotificationTemplateBuilder(EmailNotificationTemplateBuilder):
|
||||
|
||||
markdown_to = {
|
||||
TemplateFormats.HTML: to_html,
|
||||
TemplateFormats.TXT: lambda x: x, # TODO: strip markdown?
|
||||
@@ -730,9 +798,10 @@ class MessageEmailNotificationTemplateBuilder(EmailNotificationTemplateBuilder):
|
||||
|
||||
|
||||
class NewSharedItemEmailNotificationTemplateBuilder(EmailNotificationTemplateBuilder):
|
||||
|
||||
def get_content(self, template_format: TemplateFormats) -> AnyNotificationContent:
|
||||
content = NewSharedItemNotificationContent.model_construct(**self.notification.content) # type: ignore[arg-type]
|
||||
content = NewSharedItemNotificationContent.model_construct(
|
||||
**self.notification.content
|
||||
) # type: ignore[arg-type]
|
||||
return content
|
||||
|
||||
def get_subject(self) -> str:
|
||||
@@ -741,7 +810,6 @@ class NewSharedItemEmailNotificationTemplateBuilder(EmailNotificationTemplateBui
|
||||
|
||||
|
||||
class EmailNotificationChannelPlugin(NotificationChannelPlugin):
|
||||
|
||||
# Register the supported email templates here
|
||||
email_templates_by_category: dict[PersonalNotificationCategory, type[EmailNotificationTemplateBuilder]] = {
|
||||
PersonalNotificationCategory.message: MessageEmailNotificationTemplateBuilder,
|
||||
|
||||
@@ -0,0 +1,164 @@
|
||||
"""Periodic gauge emitter for control-queue depth and SSE-connection counts.
|
||||
|
||||
Scheduled by Celery beat (see ``galaxy.celery.__init__.setup_periodic_tasks``)
|
||||
at a fixed cadence. Opens a short-lived kombu connection, iterates the control
|
||||
queues returned by ``all_control_queues_for_declare`` and samples each queue's
|
||||
message-count via a passive declare. Also samples in-memory connection counts
|
||||
from ``SSEConnectionManager`` and the active-``WorkerProcess`` count from the
|
||||
database.
|
||||
|
||||
All instrumentation no-ops when ``statsd_client`` is ``None`` — i.e. statsd
|
||||
isn't configured.
|
||||
|
||||
The sub-emitters take narrow, typed collaborators (a kombu connection, an
|
||||
application stack, a model mapping, the statsd client, an SSE manager) rather
|
||||
than the whole ``StructuredApp``. The Celery task in
|
||||
``galaxy.celery.tasks.emit_queue_metrics_task`` is the composition root that
|
||||
resolves those narrow deps from the app and passes them in.
|
||||
"""
|
||||
|
||||
import datetime
|
||||
import logging
|
||||
from collections import defaultdict
|
||||
from collections.abc import Callable
|
||||
from typing import (
|
||||
Optional,
|
||||
TYPE_CHECKING,
|
||||
)
|
||||
|
||||
from sqlalchemy import (
|
||||
func,
|
||||
select,
|
||||
)
|
||||
|
||||
from galaxy.managers.sse import SSEConnectionManager
|
||||
from galaxy.model import WorkerProcess
|
||||
from galaxy.model.mapping import GalaxyModelMapping
|
||||
from galaxy.model.orm.now import now
|
||||
from galaxy.queues import (
|
||||
all_control_queues_for_declare,
|
||||
DEFAULT_ACTIVE_PROCESS_WINDOW_SECONDS,
|
||||
)
|
||||
from galaxy.web_stack import ApplicationStack
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from kombu import Connection
|
||||
|
||||
from galaxy.web.statsd_client import VanillaGalaxyStatsdClient
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def emit_sse_connection_gauges(
|
||||
statsd_client: "VanillaGalaxyStatsdClient",
|
||||
sse_manager: SSEConnectionManager,
|
||||
) -> None:
|
||||
"""Emit ``galaxy.sse.connections.active`` gauges by connection kind."""
|
||||
statsd_client.timing(
|
||||
"galaxy.sse.connections.active",
|
||||
sse_manager.total_broadcast_connections,
|
||||
tags={"kind": "broadcast"},
|
||||
)
|
||||
statsd_client.timing(
|
||||
"galaxy.sse.connections.active",
|
||||
sse_manager.total_per_user_connections,
|
||||
tags={"kind": "per_user"},
|
||||
)
|
||||
|
||||
|
||||
def emit_control_queue_depth(
|
||||
statsd_client: "VanillaGalaxyStatsdClient",
|
||||
connection: "Optional[Connection]",
|
||||
application_stack: ApplicationStack,
|
||||
) -> None:
|
||||
"""Emit ``galaxy.control_queue.depth`` per active webapp/handler queue.
|
||||
|
||||
A per-queue passive declare can fail on transports that don't implement it
|
||||
(e.g. the sqlalchemy kombu transport) or for queues that don't yet exist on
|
||||
the broker. Those are expected and quiet — logged at DEBUG, no metric, move
|
||||
on. Errors at the broker-connection layer propagate up so the caller can
|
||||
surface them.
|
||||
"""
|
||||
if connection is None:
|
||||
return
|
||||
queues = all_control_queues_for_declare(application_stack)
|
||||
if not queues:
|
||||
return
|
||||
with connection.clone() as conn:
|
||||
channel = conn.channel()
|
||||
try:
|
||||
for queue in queues:
|
||||
try:
|
||||
declared = queue.bind(channel).queue_declare(passive=True)
|
||||
except Exception:
|
||||
log.debug(
|
||||
"queue_metrics: passive declare failed for %s",
|
||||
queue.name,
|
||||
exc_info=True,
|
||||
)
|
||||
continue
|
||||
statsd_client.timing(
|
||||
"galaxy.control_queue.depth",
|
||||
declared.message_count,
|
||||
tags={"queue_name": queue.name},
|
||||
)
|
||||
finally:
|
||||
channel.close()
|
||||
|
||||
|
||||
def emit_worker_process_gauge(
|
||||
statsd_client: "VanillaGalaxyStatsdClient",
|
||||
model: GalaxyModelMapping,
|
||||
) -> None:
|
||||
"""Emit ``galaxy.worker_process.active`` gauge grouped by ``app_type``."""
|
||||
cutoff = now() - datetime.timedelta(seconds=DEFAULT_ACTIVE_PROCESS_WINDOW_SECONDS)
|
||||
stmt = (
|
||||
select(WorkerProcess.app_type, func.count(WorkerProcess.id))
|
||||
.where(WorkerProcess.update_time > cutoff)
|
||||
.group_by(WorkerProcess.app_type)
|
||||
)
|
||||
counts: dict[str, int] = defaultdict(int)
|
||||
with model.new_session() as session:
|
||||
for app_type, count in session.execute(stmt):
|
||||
counts[app_type or "unknown"] = int(count)
|
||||
for app_type, count in counts.items():
|
||||
statsd_client.timing(
|
||||
"galaxy.worker_process.active",
|
||||
count,
|
||||
tags={"app_type": app_type},
|
||||
)
|
||||
|
||||
|
||||
def _run(name: str, statsd_client: "VanillaGalaxyStatsdClient", fn: Callable[[], None]) -> None:
|
||||
"""Run a sub-emitter, isolating its failures.
|
||||
|
||||
A broken sub-emitter logs once at WARNING and increments
|
||||
``galaxy.queue_metrics.error`` (tagged by emitter name) so the failure is
|
||||
observable in metrics without the Celery-beat wrapper logging on every
|
||||
tick. The other sub-emitters continue to run on this tick.
|
||||
"""
|
||||
try:
|
||||
fn()
|
||||
except Exception:
|
||||
log.warning("queue_metrics: %s emitter failed", name, exc_info=True)
|
||||
statsd_client.incr("galaxy.queue_metrics.error", tags={"emitter": name})
|
||||
|
||||
|
||||
def emit_queue_metrics(
|
||||
statsd_client: "Optional[VanillaGalaxyStatsdClient]",
|
||||
connection: "Optional[Connection]",
|
||||
application_stack: ApplicationStack,
|
||||
model: GalaxyModelMapping,
|
||||
sse_manager: Optional[SSEConnectionManager],
|
||||
) -> None:
|
||||
"""Periodic entry-point — no-ops when statsd isn't configured."""
|
||||
if statsd_client is None:
|
||||
return
|
||||
if sse_manager is not None:
|
||||
_run("sse_connections", statsd_client, lambda: emit_sse_connection_gauges(statsd_client, sse_manager))
|
||||
_run(
|
||||
"control_queue_depth",
|
||||
statsd_client,
|
||||
lambda: emit_control_queue_depth(statsd_client, connection, application_stack),
|
||||
)
|
||||
_run("worker_process", statsd_client, lambda: emit_worker_process_gauge(statsd_client, model))
|
||||
@@ -0,0 +1,242 @@
|
||||
"""Server-Sent Events (SSE) connection manager for real-time notifications.
|
||||
|
||||
Manages per-worker in-memory mapping of user IDs to asyncio.Queue instances,
|
||||
enabling push of events from any thread (e.g. Kombu control queue worker)
|
||||
to async SSE endpoint handlers running in the uvicorn event loop.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from collections import defaultdict
|
||||
from collections.abc import (
|
||||
AsyncIterator,
|
||||
Awaitable,
|
||||
Callable,
|
||||
)
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
from typing import (
|
||||
Optional,
|
||||
)
|
||||
|
||||
from galaxy.model.orm.now import now
|
||||
from galaxy.web.statsd_client import VanillaGalaxyStatsdClient
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def make_event_id() -> str:
|
||||
"""Return an SSE ``id`` string for Last-Event-ID replay.
|
||||
|
||||
Uses ``galaxy.model.orm.now`` so the timestamp format matches the rest of
|
||||
Galaxy's database-backed timestamps (timezone-naive UTC). Kept in one place
|
||||
so producers and the parse path cannot drift.
|
||||
"""
|
||||
return now().isoformat()
|
||||
|
||||
|
||||
def parse_event_id(event_id: str) -> Optional[datetime]:
|
||||
"""Inverse of :func:`make_event_id`. Returns ``None`` if unparseable."""
|
||||
try:
|
||||
return datetime.fromisoformat(event_id)
|
||||
except (ValueError, TypeError):
|
||||
return None
|
||||
|
||||
|
||||
#: Async callable returning True when the client has disconnected. The SSE
|
||||
#: stream loop polls this each iteration so managers don't depend on starlette.
|
||||
IsDisconnected = Callable[[], Awaitable[bool]]
|
||||
|
||||
|
||||
@dataclass
|
||||
class SSEEvent:
|
||||
"""An event to be sent to an SSE client."""
|
||||
|
||||
event: str # e.g. "notification_update", "broadcast_update", "notification_status"
|
||||
data: str # JSON payload
|
||||
id: Optional[str] = None # ISO timestamp, used by EventSource as Last-Event-ID on reconnect
|
||||
|
||||
def to_wire(self) -> str:
|
||||
"""Serialize this event to the SSE wire format (``event:…\\ndata:…\\n[id:…\\n]\\n``)."""
|
||||
frame = f"event: {self.event}\ndata: {self.data}\n"
|
||||
if self.id:
|
||||
frame += f"id: {self.id}\n"
|
||||
return frame + "\n"
|
||||
|
||||
|
||||
class SSEConnectionManager:
|
||||
"""Per-worker manager for SSE connections.
|
||||
|
||||
Maps user_ids to sets of asyncio.Queue instances. Each SSE connection
|
||||
gets its own queue. The manager is thread-safe for push operations
|
||||
via ``loop.call_soon_threadsafe``.
|
||||
|
||||
Lifecycle:
|
||||
- Instantiated once per Galaxy worker process (on app object).
|
||||
- ``connect()`` is called from the SSE async endpoint (event loop thread).
|
||||
- ``disconnect()`` is called from the SSE endpoint's ``finally`` block.
|
||||
- ``push_to_user()`` / ``push_broadcast()`` are called from ANY thread
|
||||
(typically the Kombu daemon thread via control task handlers).
|
||||
"""
|
||||
|
||||
def __init__(self, statsd_client: Optional[VanillaGalaxyStatsdClient] = None) -> None:
|
||||
self._connections: dict[int, set[asyncio.Queue]] = defaultdict(set)
|
||||
self._session_connections: dict[int, set[asyncio.Queue]] = defaultdict(set)
|
||||
self._broadcast_connections: set[asyncio.Queue] = set()
|
||||
self._loop: Optional[asyncio.AbstractEventLoop] = None
|
||||
self._statsd_client = statsd_client
|
||||
|
||||
def _ensure_loop(self) -> None:
|
||||
"""Capture the running asyncio event loop. Must be called from async context."""
|
||||
if self._loop is None or self._loop.is_closed():
|
||||
self._loop = asyncio.get_running_loop()
|
||||
|
||||
# -- Called from ASYNC context (uvicorn event loop thread) --
|
||||
|
||||
def connect(self, user_id: Optional[int], galaxy_session_id: Optional[int] = None) -> asyncio.Queue:
|
||||
"""Register a new SSE connection. Returns a queue to await events from.
|
||||
|
||||
Called from the SSE endpoint handler (async context). A ``ready`` event is
|
||||
enqueued immediately so that clients (and tests) can synchronize on the
|
||||
server-side subscription rather than the underlying socket open event.
|
||||
|
||||
``galaxy_session_id`` is the dispatch key for events that target a
|
||||
specific browser session (e.g. history updates for anonymous users,
|
||||
whose ``user_id`` is ``None``).
|
||||
"""
|
||||
self._ensure_loop()
|
||||
queue: asyncio.Queue = asyncio.Queue(maxsize=64)
|
||||
if user_id is not None:
|
||||
self._connections[user_id].add(queue)
|
||||
if galaxy_session_id is not None:
|
||||
self._session_connections[galaxy_session_id].add(queue)
|
||||
self._broadcast_connections.add(queue)
|
||||
queue.put_nowait(SSEEvent(event="ready", data=""))
|
||||
log.debug(
|
||||
"SSE connection opened for user_id=%s session_id=%s (total=%d)",
|
||||
user_id,
|
||||
galaxy_session_id,
|
||||
len(self._broadcast_connections),
|
||||
)
|
||||
return queue
|
||||
|
||||
def disconnect(
|
||||
self,
|
||||
user_id: Optional[int],
|
||||
queue: asyncio.Queue,
|
||||
galaxy_session_id: Optional[int] = None,
|
||||
) -> None:
|
||||
"""Unregister an SSE connection.
|
||||
|
||||
Called from the SSE endpoint's ``finally`` block (async context).
|
||||
"""
|
||||
if user_id is not None:
|
||||
self._connections[user_id].discard(queue)
|
||||
if not self._connections[user_id]:
|
||||
del self._connections[user_id]
|
||||
if galaxy_session_id is not None:
|
||||
self._session_connections[galaxy_session_id].discard(queue)
|
||||
if not self._session_connections[galaxy_session_id]:
|
||||
del self._session_connections[galaxy_session_id]
|
||||
self._broadcast_connections.discard(queue)
|
||||
log.debug(
|
||||
"SSE connection closed for user_id=%s session_id=%s (total=%d)",
|
||||
user_id,
|
||||
galaxy_session_id,
|
||||
len(self._broadcast_connections),
|
||||
)
|
||||
|
||||
# -- Called from ANY thread (Kombu thread or async) --
|
||||
|
||||
def push_to_user(self, user_id: int, event: SSEEvent) -> None:
|
||||
"""Thread-safe. Push an event to all SSE connections for a specific user."""
|
||||
for queue in list(self._connections.get(user_id, [])):
|
||||
self._safe_put(queue, event)
|
||||
|
||||
def push_to_session(self, galaxy_session_id: int, event: SSEEvent) -> None:
|
||||
"""Thread-safe. Push an event to all SSE connections for a specific galaxy_session.
|
||||
|
||||
Used to route per-browser events (e.g. history updates for anonymous
|
||||
histories) when there is no registered ``user_id`` to key on.
|
||||
"""
|
||||
for queue in list(self._session_connections.get(galaxy_session_id, [])):
|
||||
self._safe_put(queue, event)
|
||||
|
||||
def push_broadcast(self, event: SSEEvent) -> None:
|
||||
"""Thread-safe. Push an event to ALL connected SSE clients."""
|
||||
for queue in list(self._broadcast_connections):
|
||||
self._safe_put(queue, event)
|
||||
|
||||
def _safe_put(self, queue: asyncio.Queue, event: SSEEvent) -> None:
|
||||
"""Cross the thread boundary safely using ``call_soon_threadsafe``."""
|
||||
if self._loop is None or self._loop.is_closed():
|
||||
return
|
||||
try:
|
||||
self._loop.call_soon_threadsafe(self._do_put, queue, event)
|
||||
except RuntimeError:
|
||||
# Event loop is closed or shutting down
|
||||
pass
|
||||
|
||||
def _do_put(self, queue: asyncio.Queue, event: SSEEvent) -> None:
|
||||
"""Runs ON the event loop thread. Safe to touch asyncio.Queue here."""
|
||||
try:
|
||||
queue.put_nowait(event)
|
||||
except asyncio.QueueFull:
|
||||
log.warning("SSE queue full, dropping event: %s", event.event)
|
||||
if self._statsd_client is not None:
|
||||
self._statsd_client.incr("galaxy.sse.connections.dropped")
|
||||
|
||||
@property
|
||||
def connected_user_ids(self) -> set[int]:
|
||||
return set(self._connections.keys())
|
||||
|
||||
@property
|
||||
def total_connections(self) -> int:
|
||||
return len(self._broadcast_connections)
|
||||
|
||||
@property
|
||||
def total_broadcast_connections(self) -> int:
|
||||
"""Number of all active SSE connections (includes anonymous).
|
||||
|
||||
Every connection is added to ``_broadcast_connections``; this is the
|
||||
most accurate "SSE clients currently connected" gauge.
|
||||
"""
|
||||
return len(self._broadcast_connections)
|
||||
|
||||
@property
|
||||
def total_per_user_connections(self) -> int:
|
||||
"""Number of active SSE connections bound to a specific user_id."""
|
||||
return sum(len(queues) for queues in self._connections.values())
|
||||
|
||||
# -- High-level streaming helper --
|
||||
|
||||
async def stream(
|
||||
self,
|
||||
is_disconnected: IsDisconnected,
|
||||
user_id: Optional[int],
|
||||
catch_up: Optional[SSEEvent] = None,
|
||||
keepalive: float = 30.0,
|
||||
galaxy_session_id: Optional[int] = None,
|
||||
) -> AsyncIterator[str]:
|
||||
"""Yield SSE-framed strings for one connected client.
|
||||
|
||||
Handles ``connect``, optional catch-up event priming, the main event
|
||||
loop with a keepalive comment on timeout, disconnect detection, and
|
||||
``disconnect`` in ``finally``. The ``is_disconnected`` callable is
|
||||
what the service passes in (typically ``request.is_disconnected`` from
|
||||
starlette) so the manager stays framework-agnostic.
|
||||
"""
|
||||
queue = self.connect(user_id, galaxy_session_id)
|
||||
if catch_up is not None:
|
||||
await queue.put(catch_up)
|
||||
try:
|
||||
while True:
|
||||
if await is_disconnected():
|
||||
break
|
||||
try:
|
||||
event: SSEEvent = await asyncio.wait_for(queue.get(), timeout=keepalive)
|
||||
yield event.to_wire()
|
||||
except asyncio.TimeoutError:
|
||||
yield ": keepalive\n\n"
|
||||
finally:
|
||||
self.disconnect(user_id, queue, galaxy_session_id)
|
||||
@@ -0,0 +1,166 @@
|
||||
"""Producer-side SSE helper: fans events out across Galaxy processes.
|
||||
|
||||
Kept in its own module (separate from ``galaxy.managers.sse``) because the
|
||||
dispatcher depends on ``galaxy.queue_worker`` and ``galaxy.queues``, while
|
||||
``queue_worker`` in turn depends on the connection types in ``sse``. Splitting
|
||||
the producer (``SSEEventDispatcher``) from the connection state
|
||||
(``SSEConnectionManager`` / ``SSEEvent``) breaks that cycle without requiring
|
||||
inline imports in the hot path.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import threading
|
||||
import time
|
||||
from collections.abc import Callable
|
||||
from typing import (
|
||||
Any,
|
||||
Optional,
|
||||
)
|
||||
|
||||
from cachetools import TTLCache
|
||||
from kombu import Queue
|
||||
|
||||
from galaxy.managers.sse import make_event_id
|
||||
from galaxy.queue_worker import (
|
||||
ControlTask,
|
||||
GalaxyQueueWorker,
|
||||
)
|
||||
from galaxy.queues import all_control_queues_for_declare
|
||||
from galaxy.web.statsd_client import VanillaGalaxyStatsdClient
|
||||
from galaxy.web_stack import ApplicationStack
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class SSEEventDispatcher:
|
||||
"""Fans out SSE events across all Galaxy worker processes via the control queue.
|
||||
|
||||
Dependencies are injected individually so the dispatcher can be unit-tested
|
||||
without a full ``app``. ``queue_worker`` is ``Optional`` because unit-test
|
||||
mock apps and Galaxy configurations without AMQP don't construct one — the
|
||||
dispatcher silently no-ops in that case.
|
||||
|
||||
``statsd_client`` is optional — if ``None`` (statsd not configured), all
|
||||
instrumentation becomes a cheap attribute-lookup no-op.
|
||||
"""
|
||||
|
||||
# TTL for the active-worker declare-queue cache. The WorkerProcess heartbeat
|
||||
# writes every 60 s and ``all_control_queues_for_declare`` filters on a 120 s
|
||||
# window, so a 30 s cache cannot produce a result that wasn't also valid in
|
||||
# the non-cached call. Surfaced as a class constant so tests can monkey-patch.
|
||||
_DECLARE_QUEUES_TTL_SECONDS = 30
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
queue_worker: Optional[GalaxyQueueWorker],
|
||||
application_stack: ApplicationStack,
|
||||
statsd_client: Optional[VanillaGalaxyStatsdClient] = None,
|
||||
clock: Callable[[], float] = time.monotonic,
|
||||
# Factory return is typed ``Any`` so ``ControlTask`` itself and test-only
|
||||
# duck-typed doubles (FakeControlTask/BoomControlTask/NoopControlTask)
|
||||
# all satisfy the signature under mypy.
|
||||
control_task_factory: Callable[[GalaxyQueueWorker], Any] = ControlTask,
|
||||
queues_provider: Optional[Callable[[], list[Queue]]] = None,
|
||||
) -> None:
|
||||
self._queue_worker = queue_worker
|
||||
self._application_stack = application_stack
|
||||
self._statsd_client = statsd_client
|
||||
self._clock = clock
|
||||
self._control_task_factory = control_task_factory
|
||||
# Default provider closes over application_stack so tests can pass a
|
||||
# plain ``lambda: [...]`` without needing a stack.
|
||||
self._queues_provider = queues_provider or (
|
||||
lambda: all_control_queues_for_declare(application_stack, webapp_only=True)
|
||||
)
|
||||
self._declare_queues_cache: TTLCache = TTLCache(maxsize=1, ttl=self._DECLARE_QUEUES_TTL_SECONDS, timer=clock)
|
||||
self._declare_queues_lock = threading.RLock()
|
||||
|
||||
def _get_declare_queues(self) -> list[Queue]:
|
||||
# Empty results (startup before DatabaseHeartbeat registers this process,
|
||||
# or a transient DB error swallowed by the provider) must not be pinned
|
||||
# for the full TTL — they'd silently drop every SSE event until the next
|
||||
# expiry. Only cache non-empty results.
|
||||
with self._declare_queues_lock:
|
||||
try:
|
||||
return self._declare_queues_cache["webapp"]
|
||||
except KeyError:
|
||||
queues = self._queues_provider()
|
||||
if queues:
|
||||
self._declare_queues_cache["webapp"] = queues
|
||||
return queues
|
||||
|
||||
def _send(self, task: str, kwargs: dict[str, Any]) -> None:
|
||||
if self._queue_worker is None:
|
||||
# AMQP not configured at all (e.g. unit-test mock app). Skip silently.
|
||||
log.debug("SSE dispatch skipped: no queue_worker configured (task=%s)", task)
|
||||
if self._statsd_client is not None:
|
||||
self._statsd_client.incr("galaxy.sse.dispatch.skipped_no_qw")
|
||||
return
|
||||
if self._statsd_client is not None:
|
||||
self._statsd_client.incr("galaxy.sse.dispatch.count", tags={"task": task})
|
||||
# Only fan out to webapp processes — job handlers and workflow schedulers
|
||||
# don't have browser SSE connections to push to.
|
||||
declare_queues = self._get_declare_queues()
|
||||
control_task = self._control_task_factory(self._queue_worker)
|
||||
start_time = time.perf_counter() if self._statsd_client is not None else 0.0
|
||||
try:
|
||||
control_task.send_task(
|
||||
payload={"task": task, "kwargs": kwargs},
|
||||
routing_key="control.*",
|
||||
expiration=10,
|
||||
declare_queues=declare_queues,
|
||||
)
|
||||
finally:
|
||||
if self._statsd_client is not None:
|
||||
dt_ms = int((time.perf_counter() - start_time) * 1000)
|
||||
self._statsd_client.timing("galaxy.sse.dispatch.latency_ms", dt_ms, tags={"task": task})
|
||||
|
||||
def notify_users(self, user_ids: list[int], payload: str, event_id: Optional[str] = None) -> None:
|
||||
self._send(
|
||||
"notify_users",
|
||||
{
|
||||
"user_ids": user_ids,
|
||||
"payload": payload,
|
||||
"event_id": event_id or make_event_id(),
|
||||
},
|
||||
)
|
||||
|
||||
def notify_broadcast(self, payload: str, event_id: Optional[str] = None) -> None:
|
||||
self._send(
|
||||
"notify_broadcast",
|
||||
{
|
||||
"payload": payload,
|
||||
"event_id": event_id or make_event_id(),
|
||||
},
|
||||
)
|
||||
|
||||
def history_update(
|
||||
self,
|
||||
user_updates: dict[str, list[int]],
|
||||
event_id: Optional[str] = None,
|
||||
session_updates: Optional[dict[str, list[int]]] = None,
|
||||
) -> None:
|
||||
kwargs: dict[str, Any] = {
|
||||
"user_updates": user_updates,
|
||||
"event_id": event_id or make_event_id(),
|
||||
}
|
||||
if session_updates:
|
||||
# Only include when non-empty: anonymous histories are uncommon on
|
||||
# most deployments, and an empty dict is wasted wire payload.
|
||||
kwargs["session_updates"] = session_updates
|
||||
self._send("history_update", kwargs)
|
||||
|
||||
def entry_point_update(self, user_id: int, event_id: Optional[str] = None) -> None:
|
||||
"""Fan out a wake-up ``entry_point_update`` event for one user.
|
||||
|
||||
The client always refetches the canonical entry-point list on receipt,
|
||||
so no IDs are sent — keeping the payload small and the dispatch path
|
||||
free of per-event encoding work.
|
||||
"""
|
||||
self._send(
|
||||
"entry_point_update",
|
||||
{
|
||||
"user_id": user_id,
|
||||
"event_id": event_id or make_event_id(),
|
||||
},
|
||||
)
|
||||
@@ -17,6 +17,8 @@ from galaxy.model.orm.now import now
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
WEBAPP = "webapp" # WorkerProcess.app_type for web apps.
|
||||
SSE_MONITOR = "sse_monitor" # WorkerProcess.app_type for the standalone SSE monitor process.
|
||||
SSE_MONITOR_SERVER_PREFIX = "sse_monitor." # server_name prefix used by the standalone SSE monitor process.
|
||||
|
||||
|
||||
class DatabaseHeartbeat:
|
||||
@@ -27,7 +29,9 @@ class DatabaseHeartbeat:
|
||||
self.hostname = socket.gethostname()
|
||||
self._engine = application_stack.app.model.engine
|
||||
self._is_config_watcher = False
|
||||
self._is_history_audit_monitor = False
|
||||
self._observers = []
|
||||
self._audit_monitor_observers = []
|
||||
self.exit = threading.Event()
|
||||
self.thread = None
|
||||
self.active = False
|
||||
@@ -72,6 +76,9 @@ class DatabaseHeartbeat:
|
||||
def add_change_callback(self, callback):
|
||||
self._observers.append(callback)
|
||||
|
||||
def add_audit_monitor_change_callback(self, callback):
|
||||
self._audit_monitor_observers.append(callback)
|
||||
|
||||
@property
|
||||
def is_config_watcher(self):
|
||||
return self._is_config_watcher
|
||||
@@ -83,6 +90,28 @@ class DatabaseHeartbeat:
|
||||
for callback in self._observers:
|
||||
callback(self._is_config_watcher)
|
||||
|
||||
@property
|
||||
def is_history_audit_monitor(self):
|
||||
return self._is_history_audit_monitor
|
||||
|
||||
@is_history_audit_monitor.setter
|
||||
def is_history_audit_monitor(self, value):
|
||||
self._is_history_audit_monitor = value
|
||||
log.debug(
|
||||
"%s %s history audit monitor",
|
||||
self.server_name,
|
||||
"is" if self._is_history_audit_monitor else "is not",
|
||||
)
|
||||
for callback in self._audit_monitor_observers:
|
||||
callback(self._is_history_audit_monitor)
|
||||
|
||||
def _app_type(self):
|
||||
if self.application_stack.app.is_webapp:
|
||||
return WEBAPP
|
||||
if self.server_name.startswith(SSE_MONITOR_SERVER_PREFIX):
|
||||
return SSE_MONITOR
|
||||
return None
|
||||
|
||||
def update_watcher_designation(self):
|
||||
expression = self._worker_process_identifying_clause()
|
||||
stmt = select(WorkerProcess).with_for_update(of=WorkerProcess).where(expression)
|
||||
@@ -90,19 +119,37 @@ class DatabaseHeartbeat:
|
||||
worker_process = session.scalars(stmt).first()
|
||||
if not worker_process:
|
||||
worker_process = WorkerProcess(server_name=self.server_name, hostname=self.hostname)
|
||||
if self.application_stack.app.is_webapp:
|
||||
worker_process.app_type = WEBAPP
|
||||
app_type = self._app_type()
|
||||
if app_type is not None:
|
||||
worker_process.app_type = app_type
|
||||
worker_process.update_time = now()
|
||||
worker_process.pid = self.pid
|
||||
session.add(worker_process)
|
||||
active = list(self.get_active_processes(self.heartbeat_interval + 1))
|
||||
# We only want a single process watching the various config files on the file system.
|
||||
# We just pick the max server name for simplicity
|
||||
webapp_servers = [
|
||||
p.server_name for p in self.get_active_processes(self.heartbeat_interval + 1) if p.app_type == WEBAPP
|
||||
]
|
||||
webapp_servers = [p.server_name for p in active if p.app_type == WEBAPP]
|
||||
is_config_watcher = bool(webapp_servers) and self.server_name == max(webapp_servers)
|
||||
if is_config_watcher != self.is_config_watcher:
|
||||
self.is_config_watcher = is_config_watcher
|
||||
# The history-audit monitor is a single elected process too, but preference
|
||||
# goes to a standalone sse_monitor daemon when one is running so the
|
||||
# monitor's postgres LISTEN isn't blocked by webapp GIL pauses. If no
|
||||
# dedicated process is registered we fall back to a webapp (same
|
||||
# max-server_name tiebreaker as config_watcher).
|
||||
audit_leader = self._elect_audit_leader(active, webapp_servers)
|
||||
is_history_audit_monitor = audit_leader is not None and self.server_name == audit_leader
|
||||
if is_history_audit_monitor != self.is_history_audit_monitor:
|
||||
self.is_history_audit_monitor = is_history_audit_monitor
|
||||
|
||||
@staticmethod
|
||||
def _elect_audit_leader(active, webapp_servers):
|
||||
monitor_servers = [p.server_name for p in active if p.app_type == SSE_MONITOR]
|
||||
if monitor_servers:
|
||||
return min(monitor_servers)
|
||||
if webapp_servers:
|
||||
return max(webapp_servers)
|
||||
return None
|
||||
|
||||
def send_database_heartbeat(self):
|
||||
if self.active:
|
||||
|
||||
+62
@@ -0,0 +1,62 @@
|
||||
"""Add pg_notify to history audit triggers
|
||||
|
||||
Revision ID: b8d5e2f9a1c7
|
||||
Revises: f5e9e4bca542
|
||||
Create Date: 2026-04-17 14:30:00.000000
|
||||
|
||||
The SSE-based history update pipeline (see `managers/history_audit_monitor.py`)
|
||||
depends on a PostgreSQL LISTEN on the `galaxy_history_update` channel. For
|
||||
existing installations, trigger functions installed by earlier migrations (most
|
||||
recently `c716ee82337b_replace_triggers`) do not emit the corresponding
|
||||
`pg_notify`, so the monitor wakes up only from the poll-timeout fallback and
|
||||
per-history events are never dispatched in real time. This revision replaces
|
||||
both audit trigger functions with versions that emit `pg_notify` for each
|
||||
affected history id, matching `model/triggers/update_audit_table.py` used on
|
||||
fresh installs.
|
||||
|
||||
The STATEMENT-vs-ROW decision must match the trigger DEFINITION installed by
|
||||
`c716ee82337b` so the function body references the right context (`new_table`
|
||||
vs `NEW`); both use `version > 10` (and treat offline mode as STATEMENT).
|
||||
|
||||
SQLite installations use the poll-only path and require no change.
|
||||
"""
|
||||
|
||||
from alembic import op
|
||||
|
||||
from galaxy.model.migrations.util import (
|
||||
_is_sqlite,
|
||||
transaction,
|
||||
)
|
||||
from galaxy.model.triggers.update_audit_table import (
|
||||
build_trigger_fn,
|
||||
fn_prefix,
|
||||
use_statement_trigger,
|
||||
)
|
||||
|
||||
revision = "b8d5e2f9a1c7"
|
||||
down_revision = "f5e9e4bca542"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade():
|
||||
if _is_sqlite():
|
||||
return
|
||||
with transaction():
|
||||
_install_functions(with_notify=True)
|
||||
|
||||
|
||||
def downgrade():
|
||||
if _is_sqlite():
|
||||
return
|
||||
with transaction():
|
||||
_install_functions(with_notify=False)
|
||||
|
||||
|
||||
def _install_functions(with_notify: bool) -> None:
|
||||
version_info = op.get_bind().engine.dialect.server_version_info
|
||||
# Offline mode (no live connection) matches c716ee82337b: assume STATEMENT.
|
||||
statement = version_info is None or use_statement_trigger(version_info[0])
|
||||
for id_field in ("history_id", "id"):
|
||||
fn_name = f"{fn_prefix}_{id_field}"
|
||||
op.execute(build_trigger_fn(fn_name, id_field, use_statement=statement, with_notify=with_notify))
|
||||
@@ -3,6 +3,9 @@ from sqlalchemy import DDL
|
||||
# function name prefix
|
||||
fn_prefix = "fn_audit_history_by"
|
||||
|
||||
# channel used by pg_notify so HistoryAuditMonitor can LISTEN for updates
|
||||
NOTIFY_CHANNEL = "galaxy_history_update"
|
||||
|
||||
# map between source table and associated incoming id field
|
||||
trigger_config = {
|
||||
"history_dataset_association": "history_id",
|
||||
@@ -11,6 +14,68 @@ trigger_config = {
|
||||
}
|
||||
|
||||
|
||||
def use_statement_trigger(version: int) -> bool:
|
||||
"""Return True when the postgres version supports the STATEMENT variant.
|
||||
|
||||
Fresh installs and the pg_notify migration share this predicate to ensure
|
||||
the trigger function body (STATEMENT references new_table, ROW references NEW)
|
||||
matches the trigger definition installed at that version.
|
||||
"""
|
||||
return version > 10
|
||||
|
||||
|
||||
def build_trigger_fn(function_name: str, id_field: str, *, use_statement: bool, with_notify: bool = True) -> str:
|
||||
"""Build the plpgsql CREATE OR REPLACE FUNCTION body for an audit trigger.
|
||||
|
||||
Shared between runtime install (update_audit_table.install) and alembic
|
||||
migrations so the two cannot drift.
|
||||
"""
|
||||
if use_statement:
|
||||
notify_block = (
|
||||
f"""
|
||||
FOR _history_id IN SELECT DISTINCT {id_field} FROM new_table WHERE {id_field} IS NOT NULL
|
||||
LOOP
|
||||
PERFORM pg_notify('{NOTIFY_CHANNEL}', _history_id::text);
|
||||
END LOOP;
|
||||
"""
|
||||
if with_notify
|
||||
else ""
|
||||
)
|
||||
declare_block = "DECLARE _history_id integer;" if with_notify else ""
|
||||
return f"""
|
||||
CREATE OR REPLACE FUNCTION {function_name}()
|
||||
RETURNS TRIGGER
|
||||
LANGUAGE 'plpgsql'
|
||||
AS $BODY$
|
||||
{declare_block}
|
||||
BEGIN
|
||||
INSERT INTO history_audit (history_id, update_time)
|
||||
SELECT DISTINCT {id_field}, clock_timestamp() AT TIME ZONE 'UTC'
|
||||
FROM new_table
|
||||
WHERE {id_field} IS NOT NULL
|
||||
ON CONFLICT DO NOTHING;
|
||||
{notify_block}
|
||||
RETURN NULL;
|
||||
END;
|
||||
$BODY$
|
||||
"""
|
||||
notify_stmt = f"PERFORM pg_notify('{NOTIFY_CHANNEL}', NEW.{id_field}::text);" if with_notify else ""
|
||||
return f"""
|
||||
CREATE OR REPLACE FUNCTION {function_name}()
|
||||
RETURNS TRIGGER
|
||||
LANGUAGE 'plpgsql'
|
||||
AS $BODY$
|
||||
BEGIN
|
||||
INSERT INTO history_audit (history_id, update_time)
|
||||
VALUES (NEW.{id_field}, clock_timestamp() AT TIME ZONE 'UTC')
|
||||
ON CONFLICT DO NOTHING;
|
||||
{notify_stmt}
|
||||
RETURN NULL;
|
||||
END;
|
||||
$BODY$
|
||||
"""
|
||||
|
||||
|
||||
def install(engine):
|
||||
"""Install history audit table triggers"""
|
||||
sql = _postgres_install(engine) if "postgres" in engine.name else _sqlite_install()
|
||||
@@ -41,47 +106,6 @@ def _postgres_install(engine):
|
||||
|
||||
sql = []
|
||||
|
||||
# PostgreSQL trigger function template
|
||||
# need to make separate functions purely because the incoming history_id field name will be
|
||||
# different for different source tables. There may be a fancier way to dynamically choose
|
||||
# between incoming fields, but having 2 triggers fns seems straightforward
|
||||
|
||||
def statement_trigger_fn(id_field):
|
||||
fn = f"{fn_prefix}_{id_field}"
|
||||
|
||||
return f"""
|
||||
CREATE OR REPLACE FUNCTION {fn}()
|
||||
RETURNS TRIGGER
|
||||
LANGUAGE 'plpgsql'
|
||||
AS $BODY$
|
||||
BEGIN
|
||||
INSERT INTO history_audit (history_id, update_time)
|
||||
SELECT DISTINCT {id_field}, clock_timestamp() AT TIME ZONE 'UTC'
|
||||
FROM new_table
|
||||
WHERE {id_field} IS NOT NULL
|
||||
ON CONFLICT DO NOTHING;
|
||||
RETURN NULL;
|
||||
END;
|
||||
$BODY$
|
||||
"""
|
||||
|
||||
def row_trigger_fn(id_field):
|
||||
fn = f"{fn_prefix}_{id_field}"
|
||||
|
||||
return f"""
|
||||
CREATE OR REPLACE FUNCTION {fn}()
|
||||
RETURNS TRIGGER
|
||||
LANGUAGE 'plpgsql'
|
||||
AS $BODY$
|
||||
BEGIN
|
||||
INSERT INTO history_audit (history_id, update_time)
|
||||
VALUES (NEW.{id_field}, clock_timestamp() AT TIME ZONE 'UTC')
|
||||
ON CONFLICT DO NOTHING;
|
||||
RETURN NULL;
|
||||
END;
|
||||
$BODY$
|
||||
"""
|
||||
|
||||
def trigger_def(source_table: str, id_field: str, operation: str, version: int, when: str = "AFTER") -> str:
|
||||
fn = f"{fn_prefix}_{id_field}"
|
||||
# PostgreSQL supports many triggers per operation/table so the label can
|
||||
@@ -93,7 +117,7 @@ def _postgres_install(engine):
|
||||
# The use of the keyword PROCEDURE here is historical and deprecated (https://www.postgresql.org/docs/11/sql-createtrigger.html).
|
||||
function_keyword = "FUNCTION" if version >= 11 else "PROCEDURE"
|
||||
create_or_replace = "CREATE OR REPLACE" if version >= 14 else "CREATE"
|
||||
if version >= 10 and when == "AFTER":
|
||||
if use_statement_trigger(version) and when == "AFTER":
|
||||
return f"""
|
||||
{create_or_replace} TRIGGER {trigger_name}
|
||||
AFTER {operation}
|
||||
@@ -114,10 +138,11 @@ def _postgres_install(engine):
|
||||
|
||||
# pick row or statement triggers depending on postgres version
|
||||
version = engine.dialect.server_version_info[0]
|
||||
trigger_fn = statement_trigger_fn if version >= 10 else row_trigger_fn
|
||||
statement = use_statement_trigger(version)
|
||||
|
||||
for id_field in ["history_id", "id"]:
|
||||
sql.append(trigger_fn(id_field))
|
||||
fn_name = f"{fn_prefix}_{id_field}"
|
||||
sql.append(build_trigger_fn(fn_name, id_field, use_statement=statement, with_notify=True))
|
||||
|
||||
for source_table, id_field in trigger_config.items():
|
||||
for operation in ["UPDATE", "INSERT"]:
|
||||
|
||||
@@ -9,6 +9,7 @@ galaxy-data.
|
||||
import os
|
||||
import shutil
|
||||
import tempfile
|
||||
from types import SimpleNamespace
|
||||
from typing import Optional
|
||||
|
||||
from galaxy import (
|
||||
@@ -105,6 +106,11 @@ class GalaxyDataTestApp:
|
||||
model.setup_global_object_store_for_models(self.object_store)
|
||||
self.security_agent = self.model.security_agent
|
||||
self.tag_handler = GalaxyTagHandler(self.model.session)
|
||||
# statsd/observability plumbing — stubbed out so paths that read
|
||||
# ``app.execution_timer_factory.galaxy_statsd_client`` (e.g. the
|
||||
# queue-worker instrumentation) degrade to a no-op if ever exercised
|
||||
# against this mock instead of raising ``AttributeError``.
|
||||
self.execution_timer_factory = SimpleNamespace(galaxy_statsd_client=None)
|
||||
self.init_datatypes()
|
||||
|
||||
def init_datatypes(self):
|
||||
|
||||
@@ -4,6 +4,7 @@ reloading the toolbox, etc., across multiple processes.
|
||||
"""
|
||||
|
||||
import importlib
|
||||
import json
|
||||
import logging
|
||||
import math
|
||||
import socket
|
||||
@@ -12,8 +13,11 @@ import threading
|
||||
import time
|
||||
from inspect import ismodule
|
||||
from typing import (
|
||||
Any,
|
||||
cast,
|
||||
Optional,
|
||||
TYPE_CHECKING,
|
||||
TypedDict,
|
||||
)
|
||||
|
||||
from kombu import (
|
||||
@@ -27,6 +31,10 @@ from kombu.pools import producers
|
||||
import galaxy.queues
|
||||
from galaxy import util
|
||||
from galaxy.config import reload_config_options
|
||||
from galaxy.managers.sse import (
|
||||
SSEConnectionManager,
|
||||
SSEEvent,
|
||||
)
|
||||
from galaxy.model import User
|
||||
from galaxy.tools import ToolBox
|
||||
from galaxy.tools.data_manager.manager import DataManagers
|
||||
@@ -43,7 +51,49 @@ if TYPE_CHECKING:
|
||||
)
|
||||
|
||||
|
||||
def send_local_control_task(app: "StructuredApp", task: str, get_response: bool = False, kwargs: Optional[dict] = None):
|
||||
class NotifyUsersPayload(TypedDict, total=False):
|
||||
"""Wire contract for the ``notify_users`` control-task kwargs."""
|
||||
|
||||
user_ids: list[int]
|
||||
payload: str
|
||||
event_id: Optional[str]
|
||||
|
||||
|
||||
class NotifyBroadcastPayload(TypedDict, total=False):
|
||||
"""Wire contract for the ``notify_broadcast`` control-task kwargs."""
|
||||
|
||||
payload: str
|
||||
event_id: Optional[str]
|
||||
|
||||
|
||||
class HistoryUpdatePayload(TypedDict, total=False):
|
||||
"""Wire contract for the ``history_update`` control-task kwargs.
|
||||
|
||||
``user_updates`` maps stringified user IDs to lists of (unencoded) history IDs.
|
||||
``session_updates`` is the parallel route for anonymous-owned histories,
|
||||
keyed by stringified ``galaxy_session.id`` (the dispatch key never leaves
|
||||
the server — browsers never see it). Stringified because AMQP JSON
|
||||
serialization coerces dict keys to strings.
|
||||
"""
|
||||
|
||||
user_updates: dict[str, list[int]]
|
||||
session_updates: dict[str, list[int]]
|
||||
event_id: Optional[str]
|
||||
|
||||
|
||||
class EntryPointUpdatePayload(TypedDict, total=False):
|
||||
"""Wire contract for the ``entry_point_update`` control-task kwargs."""
|
||||
|
||||
user_id: int
|
||||
event_id: Optional[str]
|
||||
|
||||
|
||||
def send_local_control_task(
|
||||
app: "StructuredApp",
|
||||
task: str,
|
||||
get_response: bool = False,
|
||||
kwargs: Optional[dict] = None,
|
||||
) -> Any:
|
||||
"""
|
||||
This sends a message to the process-local control worker, which is useful
|
||||
for one-time asynchronous tasks like recalculating user disk usage.
|
||||
@@ -57,7 +107,16 @@ def send_local_control_task(app: "StructuredApp", task: str, get_response: bool
|
||||
return control_task.send_task(payload, routing_key, local=True, get_response=get_response)
|
||||
|
||||
|
||||
def send_control_task(app, task, noop_self=False, get_response=False, routing_key="control.*", kwargs=None):
|
||||
def send_control_task(
|
||||
app: "StructuredApp",
|
||||
task: str,
|
||||
noop_self: bool = False,
|
||||
get_response: bool = False,
|
||||
routing_key: str = "control.*",
|
||||
kwargs: Optional[dict] = None,
|
||||
expiration: Optional[int] = None,
|
||||
declare_queues: Optional[list[Queue]] = None,
|
||||
) -> Any:
|
||||
"""
|
||||
This sends a control task out to all processes, useful for things like
|
||||
reloading a data table, which needs to happen individually in all
|
||||
@@ -65,6 +124,9 @@ def send_control_task(app, task, noop_self=False, get_response=False, routing_ke
|
||||
Set noop_self to True to not run task for current process.
|
||||
Set get_response to True to wait for and return the task results
|
||||
as a list.
|
||||
Set expiration to a number of seconds for message TTL.
|
||||
Pass ``declare_queues`` to override the default active-processes list —
|
||||
e.g. the SSE dispatcher uses this to restrict fan-out to webapp processes.
|
||||
"""
|
||||
if kwargs is None:
|
||||
kwargs = {}
|
||||
@@ -73,7 +135,13 @@ def send_control_task(app, task, noop_self=False, get_response=False, routing_ke
|
||||
if noop_self:
|
||||
payload["noop"] = app.config.server_name
|
||||
control_task = ControlTask(app.queue_worker)
|
||||
return control_task.send_task(payload=payload, routing_key=routing_key, get_response=get_response)
|
||||
return control_task.send_task(
|
||||
payload=payload,
|
||||
routing_key=routing_key,
|
||||
get_response=get_response,
|
||||
expiration=expiration,
|
||||
declare_queues=declare_queues,
|
||||
)
|
||||
|
||||
|
||||
class ControlTask:
|
||||
@@ -107,10 +175,19 @@ class ControlTask:
|
||||
if message.properties["correlation_id"] == self.correlation_id:
|
||||
self.response = message.payload["result"]
|
||||
|
||||
def send_task(self, payload, routing_key, local=False, get_response=False, timeout=10):
|
||||
def send_task(
|
||||
self,
|
||||
payload: dict,
|
||||
routing_key: str,
|
||||
local: bool = False,
|
||||
get_response: bool = False,
|
||||
timeout: int = 10,
|
||||
expiration: Optional[int] = None,
|
||||
declare_queues: Optional[list[Queue]] = None,
|
||||
):
|
||||
if local:
|
||||
declare_queues = self.control_queues
|
||||
else:
|
||||
elif declare_queues is None:
|
||||
declare_queues = self.declare_queues
|
||||
reply_to = None
|
||||
callback_queue = []
|
||||
@@ -129,14 +206,24 @@ class ControlTask:
|
||||
correlation_id=self.correlation_id,
|
||||
retry=True,
|
||||
headers={"epoch": time.time()},
|
||||
expiration=expiration,
|
||||
)
|
||||
if get_response:
|
||||
with Consumer(self.connection, on_message=self.on_response, queues=callback_queue, no_ack=True):
|
||||
with Consumer(
|
||||
self.connection,
|
||||
on_message=self.on_response,
|
||||
queues=callback_queue,
|
||||
no_ack=True,
|
||||
):
|
||||
while self.response is self._response:
|
||||
self.connection.drain_events(timeout=timeout)
|
||||
return self.response
|
||||
except TimeoutError:
|
||||
log.exception("Error waiting for task: '%s' sent with routing key '%s'", payload, routing_key)
|
||||
log.exception(
|
||||
"Error waiting for task: '%s' sent with routing key '%s'",
|
||||
payload,
|
||||
routing_key,
|
||||
)
|
||||
except Exception:
|
||||
log.exception("Error queueing async task: '%s'. for %s", payload, routing_key)
|
||||
|
||||
@@ -189,7 +276,10 @@ def _get_new_toolbox(app: "UniverseApplication", save_integrated_tool_panel: boo
|
||||
tool_configs = app.config.tool_configs
|
||||
|
||||
new_toolbox = ToolBox(
|
||||
tool_configs, app.config.tool_path, app, save_integrated_tool_panel=save_integrated_tool_panel
|
||||
tool_configs,
|
||||
app.config.tool_path,
|
||||
app,
|
||||
save_integrated_tool_panel=save_integrated_tool_panel,
|
||||
)
|
||||
new_toolbox.data_manager_tools = app.toolbox.data_manager_tools
|
||||
app.datatypes_registry.load_datatype_converters(new_toolbox, use_cached=True)
|
||||
@@ -309,6 +399,71 @@ def admin_job_lock(app, **kwargs):
|
||||
log.info(f"Administrative Job Lock is now set to {job_lock}. Jobs will {'not' if job_lock else 'now'} dispatch.")
|
||||
|
||||
|
||||
def notify_users(app: "MinimalManagerApp", **kwargs) -> None:
|
||||
"""Push SSE events to connected users on this worker process."""
|
||||
payload = cast(NotifyUsersPayload, kwargs)
|
||||
sse_manager = app[SSEConnectionManager]
|
||||
event = SSEEvent(
|
||||
event="notification_update",
|
||||
data=payload.get("payload", "{}"),
|
||||
id=payload.get("event_id"),
|
||||
)
|
||||
for user_id in payload.get("user_ids", []):
|
||||
sse_manager.push_to_user(user_id, event)
|
||||
|
||||
|
||||
def notify_broadcast(app: "MinimalManagerApp", **kwargs) -> None:
|
||||
"""Push SSE broadcast events to all connected clients on this worker process."""
|
||||
payload = cast(NotifyBroadcastPayload, kwargs)
|
||||
sse_manager = app[SSEConnectionManager]
|
||||
event = SSEEvent(
|
||||
event="broadcast_update",
|
||||
data=payload.get("payload", "{}"),
|
||||
id=payload.get("event_id"),
|
||||
)
|
||||
sse_manager.push_broadcast(event)
|
||||
|
||||
|
||||
def history_update(app: "MinimalManagerApp", **kwargs) -> None:
|
||||
"""Push SSE history update events to connected users on this worker process.
|
||||
|
||||
Encodes integer history IDs here (not in the monitor) so the manager layer
|
||||
stays free of presentation/security concerns. Handles both user-keyed
|
||||
routing (registered users) and galaxy_session-keyed routing (anonymous
|
||||
histories, which have ``user_id IS NULL``).
|
||||
"""
|
||||
payload = cast(HistoryUpdatePayload, kwargs)
|
||||
sse_manager = app[SSEConnectionManager]
|
||||
event_id = payload.get("event_id")
|
||||
encode = app.security.encode_id
|
||||
for user_id_str, history_ids in payload.get("user_updates", {}).items():
|
||||
user_id = int(user_id_str)
|
||||
encoded_ids = [encode(hid) for hid in history_ids]
|
||||
data = json.dumps({"history_ids": encoded_ids})
|
||||
event = SSEEvent(event="history_update", data=data, id=event_id)
|
||||
sse_manager.push_to_user(user_id, event)
|
||||
for session_id_str, history_ids in payload.get("session_updates", {}).items():
|
||||
session_id = int(session_id_str)
|
||||
encoded_ids = [encode(hid) for hid in history_ids]
|
||||
data = json.dumps({"history_ids": encoded_ids})
|
||||
event = SSEEvent(event="history_update", data=data, id=event_id)
|
||||
sse_manager.push_to_session(session_id, event)
|
||||
|
||||
|
||||
def entry_point_update(app: "MinimalManagerApp", **kwargs) -> None:
|
||||
"""Push a wake-up SSE event to a single connected user.
|
||||
|
||||
The payload is empty by design: the client refetches ``/api/entry_points``
|
||||
(the canonical source) on receipt, so there's nothing to narrow or merge.
|
||||
Dropping the IDs from the payload also avoids per-event ``encode_id`` work
|
||||
at 1000+ events/s.
|
||||
"""
|
||||
payload = cast(EntryPointUpdatePayload, kwargs)
|
||||
sse_manager = app[SSEConnectionManager]
|
||||
event = SSEEvent(event="entry_point_update", data="{}", id=payload.get("event_id"))
|
||||
sse_manager.push_to_user(int(payload["user_id"]), event)
|
||||
|
||||
|
||||
control_message_to_task = {
|
||||
"create_panel_section": create_panel_section,
|
||||
"reload_tool": reload_tool,
|
||||
@@ -324,6 +479,10 @@ control_message_to_task = {
|
||||
"reconfigure_watcher": reconfigure_watcher,
|
||||
"reload_tour": reload_tour,
|
||||
"reload_core_config": reload_core_config,
|
||||
"notify_users": notify_users,
|
||||
"notify_broadcast": notify_broadcast,
|
||||
"history_update": history_update,
|
||||
"entry_point_update": entry_point_update,
|
||||
}
|
||||
|
||||
|
||||
@@ -354,7 +513,14 @@ class GalaxyQueueWorker(ConsumerProducerMixin, threading.Thread):
|
||||
self.control_queues = []
|
||||
self.epoch = 0
|
||||
|
||||
def send_control_task(self, task, noop_self=False, get_response=False, routing_key="control.*", kwargs=None):
|
||||
def send_control_task(
|
||||
self,
|
||||
task,
|
||||
noop_self=False,
|
||||
get_response=False,
|
||||
routing_key="control.*",
|
||||
kwargs=None,
|
||||
):
|
||||
return send_control_task(
|
||||
app=self.app,
|
||||
task=task,
|
||||
@@ -369,14 +535,32 @@ class GalaxyQueueWorker(ConsumerProducerMixin, threading.Thread):
|
||||
|
||||
@property
|
||||
def declare_queues(self):
|
||||
# dynamically produce queues, allows addressing all known processes at a given time
|
||||
# Dynamically produce queues, allows addressing all known processes at a given time.
|
||||
return galaxy.queues.all_control_queues_for_declare(self.app.application_stack)
|
||||
|
||||
def bind_publisher(self):
|
||||
"""Set up the queues needed to PUBLISH control tasks (no consumer thread).
|
||||
|
||||
Safe to call from any process that needs to produce control messages — notably
|
||||
Celery workers, which want to fan out SSE events to web workers but must not
|
||||
start a consumer themselves.
|
||||
|
||||
Always (re)binds. A prefork call in ``GalaxyManagerApplication.__init__`` binds
|
||||
using the parent's ``config.server_name``; under gunicorn with ``--preload``
|
||||
the child's ``set_postfork_server_name`` mutates ``server_name`` to e.g.
|
||||
``main.1`` after fork. ``bind_and_start`` calls back into this so the
|
||||
consumer's queues match what post-fork producers declare.
|
||||
"""
|
||||
self.exchange_queue, self.direct_queue = galaxy.queues.control_queues_from_config(self.app.config)
|
||||
self.control_queues = [self.exchange_queue, self.direct_queue]
|
||||
|
||||
def bind_and_start(self):
|
||||
# This is post-forking, so we got the correct sever name
|
||||
log.info("Binding and starting galaxy control worker for %s", self.app.config.server_name)
|
||||
self.exchange_queue, self.direct_queue = galaxy.queues.control_queues_from_config(self.app.config)
|
||||
self.control_queues = [self.exchange_queue, self.direct_queue]
|
||||
log.info(
|
||||
"Binding and starting galaxy control worker for %s",
|
||||
self.app.config.server_name,
|
||||
)
|
||||
self.bind_publisher()
|
||||
self.epoch = time.time()
|
||||
self.start()
|
||||
|
||||
@@ -388,8 +572,14 @@ class GalaxyQueueWorker(ConsumerProducerMixin, threading.Thread):
|
||||
|
||||
def process_task(self, body, message):
|
||||
result = "NO_RESULT"
|
||||
task_name = body.get("task")
|
||||
statsd_client = self.app.execution_timer_factory.galaxy_statsd_client
|
||||
if statsd_client is not None and task_name is not None:
|
||||
statsd_client.incr("galaxy.control_queue.task.count", tags={"task": task_name})
|
||||
if body["task"] in self.task_mapping:
|
||||
if body.get("noop", None) != self.app.config.server_name:
|
||||
outcome = "ok"
|
||||
handler_start = time.perf_counter() if statsd_client is not None else 0.0
|
||||
try:
|
||||
f = self.task_mapping[body["task"]]
|
||||
if message.headers.get("epoch", math.inf) > self.epoch:
|
||||
@@ -410,7 +600,16 @@ class GalaxyQueueWorker(ConsumerProducerMixin, threading.Thread):
|
||||
result = "NO_OP"
|
||||
except Exception:
|
||||
# this shouldn't ever throw an exception, but...
|
||||
outcome = "error"
|
||||
log.exception("Error running control task type: %s", body["task"])
|
||||
finally:
|
||||
if statsd_client is not None and task_name is not None:
|
||||
dt_ms = int((time.perf_counter() - handler_start) * 1000)
|
||||
statsd_client.timing(
|
||||
"galaxy.control_queue.task.latency_ms",
|
||||
dt_ms,
|
||||
tags={"task": task_name, "outcome": outcome},
|
||||
)
|
||||
else:
|
||||
result = "NO_OP"
|
||||
else:
|
||||
|
||||
@@ -4,29 +4,67 @@ All message queues used by Galaxy
|
||||
|
||||
"""
|
||||
|
||||
import datetime
|
||||
import logging
|
||||
import socket
|
||||
from typing import Optional
|
||||
from typing import (
|
||||
Optional,
|
||||
TYPE_CHECKING,
|
||||
)
|
||||
|
||||
from kombu import (
|
||||
Connection,
|
||||
Exchange,
|
||||
Queue,
|
||||
)
|
||||
from sqlalchemy import select
|
||||
|
||||
from galaxy.model import WorkerProcess
|
||||
from galaxy.model.orm.now import now
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from galaxy.web_stack import ApplicationStack
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
ALL_CONTROL = "control.*"
|
||||
galaxy_exchange = Exchange("galaxy_core_exchange", type="topic")
|
||||
|
||||
DEFAULT_ACTIVE_PROCESS_WINDOW_SECONDS = 120
|
||||
# Matches WorkerProcess.app_type set by DatabaseHeartbeat for webapp processes.
|
||||
WEBAPP_APP_TYPE = "webapp"
|
||||
|
||||
def all_control_queues_for_declare(application_stack):
|
||||
|
||||
def all_control_queues_for_declare(application_stack: "ApplicationStack", webapp_only: bool = False) -> list[Queue]:
|
||||
"""
|
||||
For in-memory routing (used by sqlalchemy-based transports), we need to be able to
|
||||
build the entire routing table in producers.
|
||||
|
||||
Queries ``WorkerProcess`` directly rather than going through
|
||||
``DatabaseHeartbeat`` so this works from Celery workers too — they have a
|
||||
``model`` but no heartbeat thread. Without this, a notification created in
|
||||
a Celery task publishes a ``notify_users`` control task with an empty
|
||||
``declare`` list, so on the sqlalchemy+sqlite kombu transport the message
|
||||
never lands in a web worker's queue.
|
||||
|
||||
When ``webapp_only`` is True, only returns queues for processes that have
|
||||
registered themselves with ``app_type='webapp'``. This is what the SSE
|
||||
dispatcher wants: job handlers and workflow schedulers have no browser
|
||||
connections, so routing SSE events to them is wasted work.
|
||||
"""
|
||||
# Get all active processes and construct queues for each process
|
||||
process_names = (
|
||||
f"{p.server_name}@{p.hostname}" for p in application_stack.app.database_heartbeat.get_active_processes()
|
||||
)
|
||||
return [Queue(f"control.{server_name}", galaxy_exchange, routing_key="control.*") for server_name in process_names]
|
||||
app = application_stack.app
|
||||
try:
|
||||
stmt = select(WorkerProcess).where(
|
||||
WorkerProcess.update_time > now() - datetime.timedelta(seconds=DEFAULT_ACTIVE_PROCESS_WINDOW_SECONDS)
|
||||
)
|
||||
if webapp_only:
|
||||
stmt = stmt.where(WorkerProcess.app_type == WEBAPP_APP_TYPE)
|
||||
with app.model.new_session() as session:
|
||||
processes = session.scalars(stmt).all()
|
||||
except Exception:
|
||||
log.debug("Failed to look up active processes for control-queue declare", exc_info=True)
|
||||
return []
|
||||
return [Queue(f"control.{p.server_name}@{p.hostname}", galaxy_exchange, routing_key="control.*") for p in processes]
|
||||
|
||||
|
||||
def control_queues_from_config(config):
|
||||
|
||||
@@ -112,6 +112,7 @@ See Also
|
||||
"""
|
||||
|
||||
import abc
|
||||
import logging
|
||||
from contextlib import contextmanager
|
||||
from typing import (
|
||||
Any,
|
||||
@@ -157,6 +158,8 @@ from .playwright_element import PlaywrightElement
|
||||
from .wait_methods_mixin import WaitMethodsMixin
|
||||
from .web_element_protocol import WebElementProtocol
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
UNSPECIFIED_TIMEOUT = object()
|
||||
|
||||
|
||||
@@ -1153,9 +1156,23 @@ class HasPlaywrightDriver(TimeoutMessageMixin, WaitMethodsMixin, Generic[WaitTyp
|
||||
|
||||
This closes all windows/tabs and releases all system resources.
|
||||
The driver cannot be used after calling this method.
|
||||
|
||||
``browser.close()`` can raise — in CI we've seen it time out or hit
|
||||
target-detached errors — and if the exception escapes before
|
||||
``playwright.stop()`` runs, the per-instance asyncio loop is left
|
||||
registered as "running" on the main thread. Every subsequent test's
|
||||
``sync_playwright().__enter__`` then refuses to start with "Playwright
|
||||
Sync API inside the asyncio loop", cascading the whole shard into
|
||||
errors. Always tear down the Playwright instance even if the browser
|
||||
close failed.
|
||||
"""
|
||||
self.close()
|
||||
self._playwright_resources.playwright.stop()
|
||||
try:
|
||||
self.close()
|
||||
finally:
|
||||
try:
|
||||
self._playwright_resources.playwright.stop()
|
||||
except Exception:
|
||||
logger.exception("Error stopping Playwright instance during quit()")
|
||||
|
||||
|
||||
__all__ = (
|
||||
|
||||
@@ -0,0 +1,30 @@
|
||||
"""Standalone SSE monitor process.
|
||||
|
||||
Runs the :class:`~galaxy.managers.history_audit_monitor.HistoryAuditMonitor`
|
||||
(and, in principle, any future SSE event producer) in its own OS process so a
|
||||
webapp's GIL pauses or fork-lifecycle events can never block history-update
|
||||
dispatch. Webapps still consume SSE events normally; only the *producer* moves.
|
||||
|
||||
The process registers itself via :class:`~galaxy.model.database_heartbeat.DatabaseHeartbeat`
|
||||
with a server_name beginning with ``sse_monitor.``. The heartbeat's
|
||||
``is_history_audit_monitor`` election prefers any such process, so the monitor
|
||||
automatically migrates here when this daemon is running. If it stops, a webapp
|
||||
is re-elected within one heartbeat interval (~60s).
|
||||
|
||||
Starting the daemon
|
||||
-------------------
|
||||
|
||||
Installed::
|
||||
|
||||
GALAXY_CONFIG_FILE=/etc/galaxy/galaxy.yml galaxy-sse-monitor
|
||||
|
||||
From a source checkout::
|
||||
|
||||
GALAXY_CONFIG_FILE=config/galaxy.yml python -m galaxy.sse_monitor
|
||||
|
||||
For production deployments, run under systemd / supervisord alongside the
|
||||
webapp. A native ``gravity`` (``galaxyctl``) service type for this daemon is
|
||||
tracked as a follow-up against the ``gravity`` package — until that lands,
|
||||
either start the process via your process manager of choice, or omit it and
|
||||
let a webapp fall back to the monitor role.
|
||||
"""
|
||||
@@ -0,0 +1,99 @@
|
||||
"""Entry point for the standalone ``galaxy-sse-monitor`` daemon.
|
||||
|
||||
Loads a minimal :class:`GalaxyManagerApplication` (same shape the Celery
|
||||
workers use), forces the server_name to ``sse_monitor.<host>.<pid>`` so
|
||||
DatabaseHeartbeat's election picks this process as the history-audit monitor,
|
||||
and blocks on SIGINT/SIGTERM.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import os
|
||||
import signal
|
||||
import socket
|
||||
import sys
|
||||
import threading
|
||||
|
||||
from galaxy.celery import get_app_properties
|
||||
from galaxy.model.database_heartbeat import DatabaseHeartbeat
|
||||
|
||||
log = logging.getLogger("galaxy.sse_monitor")
|
||||
|
||||
|
||||
def _build_server_name() -> str:
|
||||
return f"sse_monitor.{socket.gethostname()}.{os.getpid()}"
|
||||
|
||||
|
||||
def _build_app(server_name: str):
|
||||
kwargs = get_app_properties() or {}
|
||||
if not kwargs:
|
||||
raise RuntimeError(
|
||||
"GALAXY_CONFIG_FILE (or GALAXY_ROOT_DIR with an on-disk Galaxy config) is required "
|
||||
"to start galaxy-sse-monitor"
|
||||
)
|
||||
kwargs = dict(kwargs)
|
||||
kwargs["check_migrate_databases"] = False
|
||||
kwargs["use_display_applications"] = False
|
||||
kwargs["use_converters"] = False
|
||||
kwargs["server_name"] = server_name
|
||||
|
||||
import galaxy.app
|
||||
|
||||
return galaxy.app.GalaxyManagerApplication(configure_logging=True, **kwargs)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s %(message)s")
|
||||
|
||||
server_name = _build_server_name()
|
||||
log.info("Starting galaxy-sse-monitor as %s", server_name)
|
||||
|
||||
app = _build_app(server_name)
|
||||
|
||||
# GalaxyManagerApplication doesn't wire a DatabaseHeartbeat (that lives on
|
||||
# UniverseApplication, which pulls in the webapp stack we don't need). We
|
||||
# spin up our own and register the audit-monitor callback so election
|
||||
# transitions start/stop the producer cleanly.
|
||||
heartbeat = DatabaseHeartbeat(application_stack=app.application_stack)
|
||||
|
||||
monitor = None
|
||||
if app.config.enable_sse_updates:
|
||||
from galaxy.managers.history_audit_monitor import HistoryAuditMonitor
|
||||
|
||||
monitor = app[HistoryAuditMonitor]
|
||||
heartbeat.add_audit_monitor_change_callback(monitor.on_role_change)
|
||||
else:
|
||||
log.warning("enable_sse_updates is False — galaxy-sse-monitor will idle with no producers")
|
||||
|
||||
heartbeat.start()
|
||||
|
||||
shutdown = threading.Event()
|
||||
|
||||
def _handle_signal(signum, _frame):
|
||||
log.info("Received signal %s, shutting down galaxy-sse-monitor", signum)
|
||||
shutdown.set()
|
||||
|
||||
signal.signal(signal.SIGINT, _handle_signal)
|
||||
signal.signal(signal.SIGTERM, _handle_signal)
|
||||
|
||||
try:
|
||||
shutdown.wait()
|
||||
finally:
|
||||
if monitor is not None:
|
||||
try:
|
||||
monitor.shutdown()
|
||||
except Exception:
|
||||
log.exception("Error shutting down HistoryAuditMonitor")
|
||||
try:
|
||||
heartbeat.shutdown()
|
||||
except Exception:
|
||||
log.exception("Error shutting down database heartbeat")
|
||||
try:
|
||||
app.shutdown()
|
||||
except Exception:
|
||||
log.exception("Error shutting down GalaxyManagerApplication")
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -41,6 +41,7 @@ from galaxy.webhooks import WebhooksRegistry
|
||||
from galaxy.workflow.trs_proxy import TrsProxy
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from galaxy.app import ExecutionTimerFactory
|
||||
from galaxy.config_watchers import ConfigWatchers
|
||||
from galaxy.jobs import JobConfiguration
|
||||
from galaxy.jobs.manager import JobManager
|
||||
@@ -57,7 +58,9 @@ if TYPE_CHECKING:
|
||||
WorkflowsManager,
|
||||
)
|
||||
from galaxy.tool_shed.galaxy_install.client import DataManagersInterface
|
||||
from galaxy.tool_shed.galaxy_install.installed_repository_manager import InstalledRepositoryManager
|
||||
from galaxy.tool_shed.galaxy_install.installed_repository_manager import (
|
||||
InstalledRepositoryManager,
|
||||
)
|
||||
from galaxy.tool_util.data import ToolDataTableManager
|
||||
from galaxy.tools import ToolBox
|
||||
from galaxy.tools.cache import ToolCache
|
||||
@@ -174,6 +177,7 @@ class StructuredApp(MinimalManagerApp):
|
||||
vault: Vault
|
||||
webhooks_registry: WebhooksRegistry
|
||||
queue_worker: Any # 'galaxy.queue_worker.GalaxyQueueWorker'
|
||||
execution_timer_factory: "ExecutionTimerFactory"
|
||||
data_provider_registry: Any # 'galaxy.visualization.data_providers.registry.DataProviderRegistry'
|
||||
tool_cache: "ToolCache"
|
||||
tool_shed_repository_cache: Optional[ToolShedRepositoryCache]
|
||||
|
||||
@@ -0,0 +1,62 @@
|
||||
"""
|
||||
API endpoint for Server-Sent Events (SSE) stream.
|
||||
|
||||
Provides a unified event stream for all real-time push events (notifications,
|
||||
history updates, etc.) independent of the notification system configuration.
|
||||
"""
|
||||
|
||||
import logging
|
||||
from typing import Optional
|
||||
|
||||
from fastapi import (
|
||||
Header,
|
||||
Request,
|
||||
)
|
||||
from starlette.responses import StreamingResponse
|
||||
|
||||
from galaxy.managers.context import ProvidesUserContext
|
||||
from galaxy.webapps.galaxy.services.events import EventsService
|
||||
from . import (
|
||||
depends,
|
||||
DependsOnTrans,
|
||||
Router,
|
||||
)
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
router = Router(tags=["events"])
|
||||
|
||||
|
||||
@router.cbv
|
||||
class FastAPIEvents:
|
||||
service: EventsService = depends(EventsService)
|
||||
|
||||
@router.get(
|
||||
"/api/events/stream",
|
||||
summary="Server-Sent Events stream for real-time updates.",
|
||||
response_class=StreamingResponse,
|
||||
)
|
||||
async def stream_events(
|
||||
self,
|
||||
request: Request,
|
||||
trans: ProvidesUserContext = DependsOnTrans,
|
||||
last_event_id: Optional[str] = Header(None, alias="Last-Event-ID"),
|
||||
) -> StreamingResponse:
|
||||
"""Opens a Server-Sent Events (SSE) connection that pushes real-time
|
||||
updates for notifications, history changes, and other events.
|
||||
|
||||
On reconnect, the browser sends the ``Last-Event-ID`` header automatically.
|
||||
If the notification system is enabled, any notifications created since that
|
||||
timestamp are delivered as a catch-up ``notification_status`` event.
|
||||
|
||||
Anonymous users receive only broadcast events.
|
||||
"""
|
||||
return StreamingResponse(
|
||||
self.service.open_stream(trans, last_event_id, request.is_disconnected),
|
||||
media_type="text/event-stream",
|
||||
headers={
|
||||
"Cache-Control": "no-cache",
|
||||
"Connection": "keep-alive",
|
||||
"X-Accel-Buffering": "no",
|
||||
},
|
||||
)
|
||||
@@ -11,6 +11,7 @@ from pydantic import (
|
||||
Field,
|
||||
)
|
||||
|
||||
from galaxy.celery.helpers import async_task_summary
|
||||
from galaxy.celery.tasks import import_data_bundle
|
||||
from galaxy.managers.context import ProvidesUserContext
|
||||
from galaxy.managers.tool_data import ToolDataManager
|
||||
@@ -25,7 +26,6 @@ from galaxy.tool_util.data._schema import (
|
||||
ToolDataItem,
|
||||
)
|
||||
from galaxy.webapps.base.api import GalaxyFileResponse
|
||||
from galaxy.webapps.galaxy.services.base import async_task_summary
|
||||
from . import (
|
||||
depends,
|
||||
DependsOnTrans,
|
||||
|
||||
@@ -7,8 +7,6 @@ from typing import (
|
||||
Optional,
|
||||
)
|
||||
|
||||
from celery.result import AsyncResult
|
||||
|
||||
from galaxy.exceptions import (
|
||||
AuthenticationRequired,
|
||||
ConfigDoesNotAllowException,
|
||||
@@ -32,7 +30,6 @@ from galaxy.model.store import (
|
||||
)
|
||||
from galaxy.schema.fields import EncodedDatabaseIdField
|
||||
from galaxy.schema.schema import (
|
||||
AsyncTaskResultSummary,
|
||||
ToolRequestDetailedModel,
|
||||
ToolRequestModel,
|
||||
)
|
||||
@@ -185,28 +182,6 @@ class ConsumesModelStores:
|
||||
)
|
||||
|
||||
|
||||
def async_task_summary(async_result: AsyncResult) -> AsyncTaskResultSummary:
|
||||
name = None
|
||||
try:
|
||||
name = async_result.name
|
||||
except AttributeError:
|
||||
# if backend is disabled, we won't have this
|
||||
pass
|
||||
queue = None
|
||||
try:
|
||||
queue = async_result.queue
|
||||
except AttributeError:
|
||||
# if backend is disabled, we won't have this
|
||||
pass
|
||||
|
||||
return AsyncTaskResultSummary(
|
||||
id=str(async_result.id),
|
||||
ignored=async_result.ignored,
|
||||
name=name,
|
||||
queue=queue,
|
||||
)
|
||||
|
||||
|
||||
def _encode_tool_request(tool_request: ToolRequest, security: IdEncodingHelper) -> dict[str, Any]:
|
||||
"""Encode request IDs using strongly-typed parameter walking."""
|
||||
tool_source_model = tool_request.tool_source
|
||||
|
||||
@@ -24,6 +24,7 @@ from galaxy import (
|
||||
util,
|
||||
web,
|
||||
)
|
||||
from galaxy.celery.helpers import async_task_summary
|
||||
from galaxy.celery.tasks import compute_dataset_hash
|
||||
from galaxy.datatypes.binary import Binary
|
||||
from galaxy.datatypes.dataproviders.exceptions import NoProviderAvailable
|
||||
@@ -88,10 +89,7 @@ from galaxy.visualization.data_providers.genome import (
|
||||
)
|
||||
from galaxy.visualization.data_providers.registry import DataProviderRegistry
|
||||
from galaxy.webapps.base.controller import UsesVisualizationMixin
|
||||
from galaxy.webapps.galaxy.services.base import (
|
||||
async_task_summary,
|
||||
ServiceBase,
|
||||
)
|
||||
from galaxy.webapps.galaxy.services.base import ServiceBase
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@@ -0,0 +1,44 @@
|
||||
"""Service layer for the unified SSE events endpoint.
|
||||
|
||||
Unlike :class:`NotificationService.open_stream`, this service does **not**
|
||||
require the notification system to be enabled — ``/api/events/stream`` also
|
||||
serves history updates and other event types independent of the notification
|
||||
configuration. When notifications are disabled the catch-up event is simply
|
||||
skipped; the stream still delivers other push events.
|
||||
"""
|
||||
|
||||
from collections.abc import AsyncIterator
|
||||
from typing import (
|
||||
Optional,
|
||||
)
|
||||
|
||||
from galaxy.managers.context import ProvidesUserContext
|
||||
from galaxy.managers.sse import (
|
||||
IsDisconnected,
|
||||
SSEConnectionManager,
|
||||
)
|
||||
from galaxy.webapps.galaxy.services.base import ServiceBase
|
||||
from galaxy.webapps.galaxy.services.notifications import NotificationService
|
||||
|
||||
|
||||
class EventsService(ServiceBase):
|
||||
def __init__(self, sse_manager: SSEConnectionManager, notifications: NotificationService) -> None:
|
||||
self.sse_manager = sse_manager
|
||||
self.notifications = notifications
|
||||
|
||||
def open_stream(
|
||||
self,
|
||||
user_context: ProvidesUserContext,
|
||||
last_event_id: Optional[str],
|
||||
is_disconnected: IsDisconnected,
|
||||
) -> AsyncIterator[str]:
|
||||
"""Open an SSE events stream.
|
||||
|
||||
Anonymous users still register under their ``galaxy_session.id`` so the
|
||||
server can route per-session events (e.g. ``history_update`` for
|
||||
anonymous-owned histories) even when ``user_id`` is ``None``.
|
||||
"""
|
||||
user_id = user_context.user.id if not user_context.anonymous else None
|
||||
session_id = user_context.galaxy_session.id if user_context.galaxy_session else None
|
||||
catch_up = self.notifications.build_status_catchup(user_context, last_event_id)
|
||||
return self.sse_manager.stream(is_disconnected, user_id, catch_up=catch_up, galaxy_session_id=session_id)
|
||||
@@ -24,6 +24,7 @@ from galaxy import (
|
||||
exceptions as glx_exceptions,
|
||||
model,
|
||||
)
|
||||
from galaxy.celery.helpers import async_task_summary
|
||||
from galaxy.celery.tasks import (
|
||||
import_model_store,
|
||||
prepare_history_download,
|
||||
@@ -87,7 +88,6 @@ from galaxy.security.idencoding import IdEncodingHelper
|
||||
from galaxy.short_term_storage import ShortTermStorageAllocator
|
||||
from galaxy.util import restore_text
|
||||
from galaxy.webapps.galaxy.services.base import (
|
||||
async_task_summary,
|
||||
ConsumesModelStores,
|
||||
model_store_storage_target,
|
||||
ServesExportStores,
|
||||
|
||||
@@ -21,6 +21,7 @@ from typing_extensions import (
|
||||
)
|
||||
|
||||
from galaxy import exceptions
|
||||
from galaxy.celery.helpers import async_task_summary
|
||||
from galaxy.celery.tasks import (
|
||||
change_datatype,
|
||||
materialize as materialize_task,
|
||||
@@ -118,7 +119,6 @@ from galaxy.security.idencoding import IdEncodingHelper
|
||||
from galaxy.short_term_storage import ShortTermStorageAllocator
|
||||
from galaxy.util.zipstream import ZipstreamWrapper
|
||||
from galaxy.webapps.galaxy.services.base import (
|
||||
async_task_summary,
|
||||
ConsumesModelStores,
|
||||
ensure_celery_tasks_enabled,
|
||||
model_store_storage_target,
|
||||
|
||||
@@ -6,6 +6,7 @@ from typing import (
|
||||
|
||||
from pydantic import Field
|
||||
|
||||
from galaxy.celery.helpers import async_task_summary
|
||||
from galaxy.celery.tasks import (
|
||||
prepare_invocation_download,
|
||||
write_invocation_to,
|
||||
@@ -61,7 +62,6 @@ from galaxy.schema.tasks import (
|
||||
from galaxy.security.idencoding import IdEncodingHelper
|
||||
from galaxy.short_term_storage import ShortTermStorageAllocator
|
||||
from galaxy.webapps.galaxy.services.base import (
|
||||
async_task_summary,
|
||||
ConsumesModelStores,
|
||||
ensure_celery_tasks_enabled,
|
||||
model_store_storage_target,
|
||||
|
||||
@@ -16,6 +16,7 @@ from galaxy import (
|
||||
exceptions,
|
||||
model,
|
||||
)
|
||||
from galaxy.celery.helpers import async_task_summary
|
||||
from galaxy.celery.tasks import queue_jobs
|
||||
from galaxy.managers import hdas
|
||||
from galaxy.managers.base import security_check
|
||||
@@ -60,7 +61,6 @@ from galaxy.tool_util.parameters import (
|
||||
ToolParameterBundleModel,
|
||||
)
|
||||
from galaxy.webapps.galaxy.services.base import (
|
||||
async_task_summary,
|
||||
ServiceBase,
|
||||
)
|
||||
from .tools import validate_tool_for_running
|
||||
|
||||
@@ -5,7 +5,6 @@ from typing import (
|
||||
Union,
|
||||
)
|
||||
|
||||
from galaxy.celery.tasks import send_notification_to_recipients_async
|
||||
from galaxy.exceptions import (
|
||||
AdminRequiredException,
|
||||
AuthenticationRequired,
|
||||
@@ -14,6 +13,12 @@ from galaxy.exceptions import (
|
||||
)
|
||||
from galaxy.managers.context import ProvidesUserContext
|
||||
from galaxy.managers.notification import NotificationManager
|
||||
from galaxy.managers.sse import (
|
||||
make_event_id,
|
||||
parse_event_id,
|
||||
SSEConnectionManager,
|
||||
SSEEvent,
|
||||
)
|
||||
from galaxy.model import User
|
||||
from galaxy.schema.fields import Security
|
||||
from galaxy.schema.notifications import (
|
||||
@@ -35,15 +40,27 @@ from galaxy.schema.notifications import (
|
||||
UserNotificationUpdateRequest,
|
||||
)
|
||||
from galaxy.schema.schema import AsyncTaskResultSummary
|
||||
from galaxy.webapps.galaxy.services.base import (
|
||||
async_task_summary,
|
||||
ServiceBase,
|
||||
)
|
||||
from galaxy.webapps.galaxy.services.base import ServiceBase
|
||||
|
||||
|
||||
class NotificationService(ServiceBase):
|
||||
def __init__(self, notification_manager: NotificationManager):
|
||||
def __init__(self, notification_manager: NotificationManager, sse_manager: SSEConnectionManager):
|
||||
self.notification_manager = notification_manager
|
||||
self.sse_manager = sse_manager
|
||||
|
||||
@property
|
||||
def notifications_enabled(self) -> bool:
|
||||
return self.notification_manager.notifications_enabled
|
||||
|
||||
def send_internal_notification(
|
||||
self, request: NotificationCreateRequest, force_sync: bool = False
|
||||
) -> Union[NotificationCreatedResponse, AsyncTaskResultSummary]:
|
||||
"""Send a system-emitted notification on behalf of internal callers (e.g. share flows).
|
||||
|
||||
Unlike :meth:`send_notification`, this skips admin/permission checks because the
|
||||
caller has already resolved the recipient set and is not acting on user input.
|
||||
"""
|
||||
return self.notification_manager.send_notification_internal(request, force_sync=force_sync)
|
||||
|
||||
def send_notification(
|
||||
self, sender_context: ProvidesUserContext, payload: NotificationCreateRequestBody
|
||||
@@ -62,28 +79,7 @@ class NotificationService(ServiceBase):
|
||||
recipients=payload.recipients,
|
||||
galaxy_url=galaxy_url,
|
||||
)
|
||||
return self.send_notification_internal(request)
|
||||
|
||||
def send_notification_internal(
|
||||
self, request: NotificationCreateRequest, force_sync: bool = False
|
||||
) -> Union[NotificationCreatedResponse, AsyncTaskResultSummary]:
|
||||
"""Sends a notification to a list of recipients (users, groups or roles).
|
||||
|
||||
If `force_sync` is set to `True`, the notification recipients will be processed synchronously instead of
|
||||
in a background task.
|
||||
|
||||
Note: This function is meant for internal use from other services that don't need to check sender permissions.
|
||||
"""
|
||||
if self.notification_manager.can_send_notifications_async and not force_sync:
|
||||
result = send_notification_to_recipients_async.delay(request)
|
||||
summary = async_task_summary(result)
|
||||
return summary
|
||||
|
||||
notification, recipient_user_count = self.notification_manager.send_notification_to_recipients(request)
|
||||
return NotificationCreatedResponse(
|
||||
total_notifications_sent=recipient_user_count,
|
||||
notification=NotificationResponse.model_validate(notification),
|
||||
)
|
||||
return self.notification_manager.send_notification_internal(request)
|
||||
|
||||
def broadcast(
|
||||
self, sender_context: ProvidesUserContext, payload: BroadcastNotificationCreateRequest
|
||||
@@ -99,6 +95,27 @@ class NotificationService(ServiceBase):
|
||||
total_notifications_sent=1, notification=NotificationResponse.model_validate(notification)
|
||||
)
|
||||
|
||||
def build_status_catchup(
|
||||
self, user_context: ProvidesUserContext, last_event_id: Optional[str]
|
||||
) -> Optional[SSEEvent]:
|
||||
"""Build a ``notification_status`` SSE event covering everything since ``last_event_id``.
|
||||
|
||||
Returns ``None`` when catch-up isn't possible (no ``Last-Event-ID``,
|
||||
unparseable timestamp, or notifications disabled) so callers can simply
|
||||
pass the result to ``SSEConnectionManager.stream`` without extra guards.
|
||||
"""
|
||||
if not last_event_id or not self.notification_manager.notifications_enabled:
|
||||
return None
|
||||
since = parse_event_id(last_event_id)
|
||||
if since is None:
|
||||
return None
|
||||
catchup = self.get_notifications_status(user_context, since)
|
||||
return SSEEvent(
|
||||
event="notification_status",
|
||||
data=catchup.model_dump_json(),
|
||||
id=make_event_id(),
|
||||
)
|
||||
|
||||
def get_notifications_status(self, user_context: ProvidesUserContext, since: datetime) -> NotificationStatusSummary:
|
||||
"""Returns the status of (unread or updated) notifications received by the user **since** a particular date and time.
|
||||
|
||||
|
||||
@@ -4,6 +4,7 @@ from typing import (
|
||||
)
|
||||
|
||||
from galaxy import exceptions
|
||||
from galaxy.celery.helpers import async_task_summary
|
||||
from galaxy.celery.tasks import prepare_pdf_download
|
||||
from galaxy.managers import base
|
||||
from galaxy.managers.markdown_util import (
|
||||
@@ -34,7 +35,6 @@ from galaxy.security.idencoding import IdEncodingHelper
|
||||
from galaxy.short_term_storage import ShortTermStorageAllocator
|
||||
from galaxy.webapps.galaxy.api.common import PageIdPathParam
|
||||
from galaxy.webapps.galaxy.services.base import (
|
||||
async_task_summary,
|
||||
ensure_celery_tasks_enabled,
|
||||
ServiceBase,
|
||||
)
|
||||
|
||||
@@ -177,17 +177,13 @@ class ShareableService:
|
||||
def _send_notification_to_users(
|
||||
self, users_to_notify: set[User], item: SharableItem, status: ShareWithStatus, galaxy_url: Optional[str] = None
|
||||
):
|
||||
if (
|
||||
self.notification_service.notification_manager.notifications_enabled
|
||||
and not status.errors
|
||||
and users_to_notify
|
||||
):
|
||||
if self.notification_service.notifications_enabled and not status.errors and users_to_notify:
|
||||
request = SharedItemNotificationFactory.build_notification_request(
|
||||
item, users_to_notify, status, galaxy_url
|
||||
)
|
||||
# We can set force_sync=True here because we already have the set of users to notify
|
||||
# and there is no need to resolve them asynchronously as no groups or roles are involved.
|
||||
self.notification_service.send_notification_internal(request, force_sync=True)
|
||||
self.notification_service.send_internal_notification(request, force_sync=True)
|
||||
|
||||
|
||||
class SharedItemNotificationFactory:
|
||||
|
||||
@@ -9,6 +9,7 @@ from galaxy import (
|
||||
exceptions as glx_exceptions,
|
||||
util,
|
||||
)
|
||||
from galaxy.celery.helpers import async_task_summary
|
||||
from galaxy.managers import api_keys
|
||||
from galaxy.managers.context import (
|
||||
ProvidesHistoryContext,
|
||||
@@ -34,10 +35,7 @@ from galaxy.schema.schema import (
|
||||
UserModel,
|
||||
)
|
||||
from galaxy.security.idencoding import IdEncodingHelper
|
||||
from galaxy.webapps.galaxy.services.base import (
|
||||
async_task_summary,
|
||||
ServiceBase,
|
||||
)
|
||||
from galaxy.webapps.galaxy.services.base import ServiceBase
|
||||
from galaxy.webapps.galaxy.services.roles import role_to_model
|
||||
|
||||
if TYPE_CHECKING:
|
||||
|
||||
@@ -0,0 +1,155 @@
|
||||
"""Shared helpers for SSE integration tests.
|
||||
|
||||
The stream layer emits a ``ready`` event as the first frame on every connection
|
||||
so tests can synchronize on the server-side subscription rather than the
|
||||
underlying TCP socket. ``SSELineListener`` waits for that ``ready`` event before
|
||||
``start()`` returns, and propagates listener-thread exceptions back to the main
|
||||
thread instead of silently swallowing them.
|
||||
"""
|
||||
|
||||
import queue
|
||||
import threading
|
||||
from collections.abc import Callable
|
||||
from typing import Optional
|
||||
|
||||
import requests
|
||||
|
||||
from galaxy.util.wait import wait_on
|
||||
|
||||
CONNECT_TIMEOUT = 15
|
||||
DEFAULT_WAIT_TIMEOUT = 15
|
||||
|
||||
|
||||
def parse_sse_events(raw: str) -> list[dict]:
|
||||
"""Parse raw SSE text into a list of event dicts with ``event``, ``data``, and ``id`` keys."""
|
||||
events: list[dict] = []
|
||||
current: dict[str, str] = {}
|
||||
for line in raw.split("\n"):
|
||||
if line.startswith(":"):
|
||||
continue # comment / keepalive
|
||||
if line == "":
|
||||
if current:
|
||||
events.append(current)
|
||||
current = {}
|
||||
continue
|
||||
if ": " in line:
|
||||
field, _, value = line.partition(": ")
|
||||
else:
|
||||
field, value = line.rstrip(":"), ""
|
||||
if field in ("event", "data", "id"):
|
||||
current[field] = value
|
||||
if current:
|
||||
events.append(current)
|
||||
return events
|
||||
|
||||
|
||||
class SSEListenerError(Exception):
|
||||
"""Wraps an exception raised inside the listener thread."""
|
||||
|
||||
|
||||
class SSELineListener:
|
||||
"""Runs an SSE connection on a background thread and collects raw chunks.
|
||||
|
||||
``start()`` blocks until the server-side ``ready`` event has been received,
|
||||
guaranteeing that any event *posted after* ``start()`` returns will be seen
|
||||
by this listener. Failures in the background thread are surfaced via
|
||||
``wait_for_event`` instead of silently timing out.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
url: str,
|
||||
api_key: str,
|
||||
headers: Optional[dict] = None,
|
||||
timeout: int = 30,
|
||||
) -> None:
|
||||
self.url = url
|
||||
self.api_key = api_key
|
||||
self.headers = headers or {}
|
||||
self.timeout = timeout
|
||||
self._collected: list[str] = []
|
||||
self._stop = threading.Event()
|
||||
self._ready = threading.Event()
|
||||
self._errors: queue.Queue[BaseException] = queue.Queue()
|
||||
self._thread = threading.Thread(target=self._listen, daemon=True)
|
||||
|
||||
def start(self) -> None:
|
||||
self._thread.start()
|
||||
wait_on(
|
||||
lambda: True if self._ready.is_set() else None,
|
||||
"SSE `ready` event",
|
||||
timeout=CONNECT_TIMEOUT,
|
||||
)
|
||||
self._raise_if_errored()
|
||||
|
||||
def stop(self) -> None:
|
||||
self._stop.set()
|
||||
self._thread.join(timeout=5)
|
||||
|
||||
def wait_for_event(self, event_type: str, timeout: int = DEFAULT_WAIT_TIMEOUT) -> list[dict]:
|
||||
"""Block until at least one event of ``event_type`` has been observed, then return all such events."""
|
||||
|
||||
def _check():
|
||||
self._raise_if_errored()
|
||||
events = self.get_events(event_type)
|
||||
return events if events else None
|
||||
|
||||
return wait_on(_check, f"SSE {event_type} event", timeout=timeout)
|
||||
|
||||
def wait_for_event_where(
|
||||
self,
|
||||
event_type: str,
|
||||
predicate: Callable[[dict], bool],
|
||||
timeout: int = DEFAULT_WAIT_TIMEOUT,
|
||||
) -> list[dict]:
|
||||
"""Block until at least one ``event_type`` event matches ``predicate``.
|
||||
|
||||
Returns every ``event_type`` event observed so far, not just matches, so
|
||||
callers can still inspect the surrounding stream (e.g. assert what else
|
||||
did or didn't appear) after the wait resolves.
|
||||
"""
|
||||
|
||||
def _check():
|
||||
self._raise_if_errored()
|
||||
events = self.get_events(event_type)
|
||||
return events if any(predicate(e) for e in events) else None
|
||||
|
||||
return wait_on(_check, f"SSE {event_type} matching predicate", timeout=timeout)
|
||||
|
||||
def get_events(self, event_type: Optional[str] = None) -> list[dict]:
|
||||
"""Return all collected events so far, optionally filtered by type."""
|
||||
all_events = parse_sse_events("".join(self._collected))
|
||||
if event_type is None:
|
||||
return all_events
|
||||
return [e for e in all_events if e.get("event") == event_type]
|
||||
|
||||
def _raise_if_errored(self) -> None:
|
||||
try:
|
||||
err = self._errors.get_nowait()
|
||||
except queue.Empty:
|
||||
return
|
||||
raise SSEListenerError(f"SSE listener thread failed: {err!r}") from err
|
||||
|
||||
def _listen(self) -> None:
|
||||
try:
|
||||
resp = requests.get(
|
||||
self.url,
|
||||
params={"key": self.api_key},
|
||||
headers=self.headers,
|
||||
stream=True,
|
||||
timeout=self.timeout,
|
||||
)
|
||||
if resp.status_code != 200:
|
||||
raise RuntimeError(f"SSE connect returned HTTP {resp.status_code}: {resp.text[:200]}")
|
||||
for chunk in resp.iter_content(chunk_size=None, decode_unicode=True):
|
||||
if chunk:
|
||||
self._collected.append(chunk)
|
||||
if not self._ready.is_set() and "event: ready" in "".join(self._collected):
|
||||
self._ready.set()
|
||||
if self._stop.is_set():
|
||||
break
|
||||
resp.close()
|
||||
except Exception as exc:
|
||||
self._errors.put(exc)
|
||||
# Ensure start() doesn't hang forever on connection failure.
|
||||
self._ready.set()
|
||||
@@ -22,8 +22,10 @@ from unittest import (
|
||||
skip,
|
||||
SkipTest,
|
||||
)
|
||||
from urllib.parse import urljoin
|
||||
|
||||
import pytest
|
||||
import requests
|
||||
|
||||
from galaxy.app import UniverseApplication
|
||||
from galaxy.tool_util.verify.test_data import TestDataResolver
|
||||
@@ -238,6 +240,16 @@ class IntegrationInstance(UsesApiTestCaseMixin, UsesCeleryTasks):
|
||||
if not self._app.config.database_connection.startswith("post"):
|
||||
raise SkipTest("Test only valid for postgres")
|
||||
|
||||
def _decode_id(self, encoded_id: str) -> int:
|
||||
"""Decode an encoded API id to its raw int via the live app's security helper."""
|
||||
return self._app.security.decode_id(encoded_id)
|
||||
|
||||
def _user_id_for_api_key(self, api_key: str) -> int:
|
||||
"""Return the raw integer ``User.id`` for the user owning ``api_key``."""
|
||||
response = requests.get(urljoin(self.url, "api/users/current"), params={"key": api_key})
|
||||
response.raise_for_status()
|
||||
return self._decode_id(response.json()["id"])
|
||||
|
||||
def _run_tool_test(self, *args, **kwargs):
|
||||
return self._test_driver.run_tool_test(*args, **kwargs)
|
||||
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
import datetime
|
||||
import errno
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import traceback
|
||||
import unittest
|
||||
@@ -76,6 +77,8 @@ try:
|
||||
except ImportError:
|
||||
GalaxyTestDriver = None # type: ignore[assignment, misc, unused-ignore]
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _load_config_file() -> None:
|
||||
"""
|
||||
@@ -433,10 +436,22 @@ class TestWithSeleniumMixin(GalaxyTestSeleniumContext, UsesApiTestCaseMixin, Use
|
||||
self.target_url_from_selenium = self._target_url_from_selenium()
|
||||
self.snapshots = []
|
||||
self.setup_driver_and_session()
|
||||
if self.run_as_admin and GALAXY_TEST_SELENIUM_ADMIN_USER_EMAIL == DEFAULT_ADMIN_USER:
|
||||
self._setup_interactor()
|
||||
self._setup_user(GALAXY_TEST_SELENIUM_ADMIN_USER_EMAIL)
|
||||
self._try_setup_with_driver()
|
||||
# Once the driver is allocated, any subsequent failure must still
|
||||
# tear it down: pytest does not call tearDown when setUp raises, so
|
||||
# without this the Playwright asyncio loop would stay registered as
|
||||
# "running" on the main thread and cascade every subsequent test's
|
||||
# setUp with "Sync API inside the asyncio loop".
|
||||
try:
|
||||
if self.run_as_admin and GALAXY_TEST_SELENIUM_ADMIN_USER_EMAIL == DEFAULT_ADMIN_USER:
|
||||
self._setup_interactor()
|
||||
self._setup_user(GALAXY_TEST_SELENIUM_ADMIN_USER_EMAIL)
|
||||
self._try_setup_with_driver()
|
||||
except Exception:
|
||||
try:
|
||||
self.tear_down_driver()
|
||||
except Exception:
|
||||
logger.exception("Error tearing down driver after setup_selenium failure")
|
||||
raise
|
||||
|
||||
def _try_setup_with_driver(self):
|
||||
try:
|
||||
|
||||
@@ -101,6 +101,7 @@ test =
|
||||
console_scripts =
|
||||
galaxy-main = galaxy.main:main
|
||||
galaxy-dependencies = galaxy.dependencies.script:main
|
||||
galaxy-sse-monitor = galaxy.sse_monitor.__main__:main
|
||||
|
||||
[options.packages.find]
|
||||
where = src
|
||||
|
||||
+1
@@ -0,0 +1 @@
|
||||
../../../../lib/galaxy/sse_monitor
|
||||
@@ -176,6 +176,7 @@ typecheck = [
|
||||
"pydantic>=2.7.4", # for pydantic.mypy plugin
|
||||
"types-bleach",
|
||||
"types-boto",
|
||||
"types-cachetools",
|
||||
"types-contextvars",
|
||||
"types-dataclasses",
|
||||
"types-docutils",
|
||||
|
||||
@@ -0,0 +1,146 @@
|
||||
"""Integration tests for SSE-based interactive-tool entry-point update notifications.
|
||||
|
||||
Mirrors ``test_history_sse.py``. Rather than spin up a real containerized
|
||||
interactive tool (which would require docker), these tests exercise the
|
||||
dispatch path by creating a ``Job`` and ``InteractiveToolEntryPoint`` rows
|
||||
directly via the live app's SQLAlchemy session and invoking
|
||||
``InteractiveToolManager.configure_entry_points`` with a stub ``ports_dict``.
|
||||
This is exactly the moment the event fires in production — the job runner's
|
||||
port-routing hook is the only upstream caller, and the SSE dispatch happens
|
||||
after the DB commit regardless of how the ports were obtained.
|
||||
"""
|
||||
|
||||
import time
|
||||
from urllib.parse import urljoin
|
||||
from uuid import uuid4
|
||||
|
||||
from galaxy.model import (
|
||||
InteractiveToolEntryPoint,
|
||||
Job,
|
||||
)
|
||||
from galaxy_test.base.populators import DatasetPopulator
|
||||
from galaxy_test.base.sse import SSELineListener
|
||||
from galaxy_test.driver.integration_util import IntegrationTestCase
|
||||
|
||||
|
||||
def _make_ports_dict(tool_port: int) -> dict:
|
||||
"""Stub the runner's port-routing payload — one tool_port, fake host/proto."""
|
||||
return {
|
||||
str(tool_port): {
|
||||
"host": "host.invalid",
|
||||
"port": 12345,
|
||||
"protocol": "http",
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
class TestEntryPointSSEIntegration(IntegrationTestCase):
|
||||
dataset_populator: DatasetPopulator
|
||||
framework_tool_and_types = True
|
||||
|
||||
@classmethod
|
||||
def handle_galaxy_config_kwds(cls, config):
|
||||
super().handle_galaxy_config_kwds(config)
|
||||
config["enable_celery_tasks"] = False
|
||||
|
||||
def setUp(self):
|
||||
super().setUp()
|
||||
self.dataset_populator = DatasetPopulator(self.galaxy_interactor)
|
||||
|
||||
def _events_stream_url(self) -> str:
|
||||
return urljoin(self.url, "api/events/stream")
|
||||
|
||||
def _create_it_job_with_entry_point(self, user_id: int, tool_port: int = 8888) -> tuple[int, int]:
|
||||
"""Create a minimal Job + unconfigured InteractiveToolEntryPoint row pair.
|
||||
|
||||
Returns ``(job_id, entry_point_id)``. The session used is the live
|
||||
app's; the rows are real and survive the call.
|
||||
"""
|
||||
sa_session = self._app.model.context
|
||||
job = Job()
|
||||
job.user_id = user_id
|
||||
job.tool_id = "interactivetool_simple"
|
||||
job.state = Job.states.RUNNING
|
||||
sa_session.add(job)
|
||||
sa_session.flush()
|
||||
ep = InteractiveToolEntryPoint(
|
||||
job=job,
|
||||
tool_port=tool_port,
|
||||
entry_url="/",
|
||||
name="test entry point",
|
||||
label="test",
|
||||
requires_domain=True,
|
||||
requires_path_in_url=False,
|
||||
requires_path_in_header_named=None,
|
||||
)
|
||||
sa_session.add(ep)
|
||||
sa_session.commit()
|
||||
return job.id, ep.id
|
||||
|
||||
def test_entry_point_update_event_fires_on_configure(self):
|
||||
"""configure_entry_points should fire an ``entry_point_update`` wake-up event."""
|
||||
api_key = self.galaxy_interactor.api_key
|
||||
assert api_key is not None
|
||||
user_id = self._user_id_for_api_key(api_key)
|
||||
job_id, _ = self._create_it_job_with_entry_point(user_id)
|
||||
|
||||
listener = SSELineListener(self._events_stream_url(), api_key)
|
||||
listener.start()
|
||||
try:
|
||||
sa_session = self._app.model.context
|
||||
job = sa_session.get(Job, job_id)
|
||||
assert job is not None
|
||||
self._app.interactivetool_manager.configure_entry_points(job, _make_ports_dict(8888))
|
||||
|
||||
entry_point_events = listener.wait_for_event("entry_point_update")
|
||||
finally:
|
||||
listener.stop()
|
||||
|
||||
# The event carries no payload: the client refetches ``/api/entry_points``
|
||||
# (the canonical source) on receipt, so the event just needs to arrive.
|
||||
assert len(entry_point_events) >= 1, f"Expected entry_point_update wake-up, got: {entry_point_events}"
|
||||
assert entry_point_events[0]["event"] == "entry_point_update"
|
||||
|
||||
def test_entry_point_update_is_scoped_to_owning_user(self):
|
||||
"""User A must not see entry_point_update events for user B's jobs.
|
||||
|
||||
The event has no payload to cross-check with, so we assert on event
|
||||
count: user A's stream should receive exactly one event for its own
|
||||
``configure_entry_points`` call and none for user B's.
|
||||
"""
|
||||
user_b = self._setup_user(f"{uuid4()}@galaxy.test")
|
||||
_, user_b_api_key = self._setup_user_get_key(user_b["email"])
|
||||
user_b_id = self._user_id_for_api_key(user_b_api_key)
|
||||
|
||||
user_a_api_key = self.galaxy_interactor.api_key
|
||||
assert user_a_api_key is not None
|
||||
user_a_id = self._user_id_for_api_key(user_a_api_key)
|
||||
|
||||
job_a_id, _ = self._create_it_job_with_entry_point(user_a_id, tool_port=7001)
|
||||
job_b_id, _ = self._create_it_job_with_entry_point(user_b_id, tool_port=7002)
|
||||
|
||||
listener = SSELineListener(self._events_stream_url(), user_a_api_key)
|
||||
listener.start()
|
||||
try:
|
||||
sa_session = self._app.model.context
|
||||
job_b = sa_session.get(Job, job_b_id)
|
||||
assert job_b is not None
|
||||
# User B's job — user A must NOT see this.
|
||||
self._app.interactivetool_manager.configure_entry_points(job_b, _make_ports_dict(7002))
|
||||
# Give the broker a moment so a leaked event (if any) would arrive
|
||||
# before we fire user A's event — the assertion would then catch
|
||||
# more than one event on user A's stream.
|
||||
time.sleep(0.5)
|
||||
|
||||
job_a = sa_session.get(Job, job_a_id)
|
||||
assert job_a is not None
|
||||
# User A's own job — this is what A's stream must observe.
|
||||
self._app.interactivetool_manager.configure_entry_points(job_a, _make_ports_dict(7001))
|
||||
|
||||
entry_point_events = listener.wait_for_event("entry_point_update")
|
||||
finally:
|
||||
listener.stop()
|
||||
|
||||
assert (
|
||||
len(entry_point_events) == 1
|
||||
), f"User A expected exactly one entry_point_update (own job); saw {len(entry_point_events)}: {entry_point_events}"
|
||||
@@ -0,0 +1,104 @@
|
||||
"""Integration tests for SSE-based history update notifications."""
|
||||
|
||||
import json
|
||||
from urllib.parse import urljoin
|
||||
from uuid import uuid4
|
||||
|
||||
import requests
|
||||
|
||||
from galaxy_test.base.populators import DatasetPopulator
|
||||
from galaxy_test.base.sse import SSELineListener
|
||||
from galaxy_test.driver.integration_util import IntegrationTestCase
|
||||
|
||||
|
||||
class TestHistorySSEIntegration(IntegrationTestCase):
|
||||
dataset_populator: DatasetPopulator
|
||||
framework_tool_and_types = True
|
||||
|
||||
@classmethod
|
||||
def handle_galaxy_config_kwds(cls, config):
|
||||
super().handle_galaxy_config_kwds(config)
|
||||
config["enable_celery_tasks"] = False
|
||||
config["enable_sse_updates"] = True
|
||||
|
||||
def setUp(self):
|
||||
super().setUp()
|
||||
self.dataset_populator = DatasetPopulator(self.galaxy_interactor)
|
||||
|
||||
def _events_stream_url(self) -> str:
|
||||
return urljoin(self.url, "api/events/stream")
|
||||
|
||||
def _create_history(self, name=None) -> str:
|
||||
name = name or f"test_history_{uuid4()}"
|
||||
response = self._post("histories", data={"name": name}, json=True)
|
||||
self._assert_status_code_is_ok(response)
|
||||
return response.json()["id"]
|
||||
|
||||
def test_history_update_contains_current_history_id(self):
|
||||
"""The history_update event should contain the history's encoded ID."""
|
||||
history_id = self._create_history()
|
||||
|
||||
api_key = self.galaxy_interactor.api_key
|
||||
assert api_key is not None
|
||||
listener = SSELineListener(self._events_stream_url(), api_key)
|
||||
listener.start()
|
||||
try:
|
||||
self.dataset_populator.new_dataset(history_id, wait=False)
|
||||
history_events = listener.wait_for_event_where(
|
||||
"history_update",
|
||||
lambda e: history_id in json.loads(e["data"]).get("history_ids", []),
|
||||
)
|
||||
found = any(history_id in json.loads(e["data"]).get("history_ids", []) for e in history_events)
|
||||
assert found, f"Expected history_id '{history_id}' in history_update events, got: {history_events}"
|
||||
finally:
|
||||
listener.stop()
|
||||
|
||||
def test_history_update_is_scoped_to_owning_user(self):
|
||||
"""User A must only see history_update events for their own histories.
|
||||
|
||||
Inverted positive assertion: after user B's upload, user A uploads to
|
||||
their own history and we assert A's stream contains A's encoded id and
|
||||
not B's. This avoids a sleep-based "no events" test that was prone to
|
||||
flaking under slow CI.
|
||||
"""
|
||||
user_b = self._setup_user(f"{uuid4()}@galaxy.test")
|
||||
_, user_b_api_key = self._setup_user_get_key(user_b["email"])
|
||||
|
||||
user_a_history_id = self._create_history()
|
||||
|
||||
api_key = self.galaxy_interactor.api_key
|
||||
assert api_key is not None
|
||||
listener = SSELineListener(self._events_stream_url(), api_key)
|
||||
listener.start()
|
||||
try:
|
||||
# User B creates a history and uploads to it. User A must NOT see this.
|
||||
create_resp = requests.post(
|
||||
urljoin(self.url, "api/histories"),
|
||||
params={"key": user_b_api_key},
|
||||
json={"name": "User B History"},
|
||||
)
|
||||
assert create_resp.status_code == 200
|
||||
user_b_history_id = create_resp.json()["id"]
|
||||
|
||||
requests.post(
|
||||
urljoin(self.url, f"api/histories/{user_b_history_id}/contents"),
|
||||
params={"key": user_b_api_key},
|
||||
json={"from_hda_id": None, "source": "pasted", "content": "user b content"},
|
||||
)
|
||||
|
||||
# User A uploads to their own history — this is what A's stream must observe.
|
||||
self.dataset_populator.new_dataset(user_a_history_id, wait=False)
|
||||
history_events = listener.wait_for_event_where(
|
||||
"history_update",
|
||||
lambda e: user_a_history_id in json.loads(e["data"]).get("history_ids", []),
|
||||
)
|
||||
finally:
|
||||
listener.stop()
|
||||
|
||||
seen_ids: set[str] = set()
|
||||
for event in history_events:
|
||||
seen_ids.update(json.loads(event["data"]).get("history_ids", []))
|
||||
assert user_a_history_id in seen_ids, f"User A missed its own history_update: {history_events}"
|
||||
assert (
|
||||
user_b_history_id not in seen_ids
|
||||
), f"User A received history_update for user B's history ({user_b_history_id}): {history_events}"
|
||||
@@ -0,0 +1,173 @@
|
||||
"""Integration tests for the notification SSE (Server-Sent Events) endpoint."""
|
||||
|
||||
import json
|
||||
from typing import Optional
|
||||
from urllib.parse import urljoin
|
||||
from uuid import uuid4
|
||||
|
||||
from galaxy_test.base.populators import DatasetPopulator
|
||||
from galaxy_test.base.sse import SSELineListener
|
||||
from galaxy_test.driver.integration_util import IntegrationTestCase
|
||||
|
||||
|
||||
def notification_test_data(subject: Optional[str] = None, message: Optional[str] = None) -> dict:
|
||||
return {
|
||||
"source": "integration_tests",
|
||||
"variant": "info",
|
||||
"category": "message",
|
||||
"content": {
|
||||
"category": "message",
|
||||
"subject": subject or "Testing Subject",
|
||||
"message": message or "Testing Message",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def notification_broadcast_test_data(subject: Optional[str] = None, message: Optional[str] = None) -> dict:
|
||||
return {
|
||||
"source": "integration_tests",
|
||||
"variant": "info",
|
||||
"category": "broadcast",
|
||||
"content": {
|
||||
"category": "broadcast",
|
||||
"subject": subject or "Testing Broadcast Subject",
|
||||
"message": message or "Testing Broadcast Message",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _notification_subjects(events: list[dict]) -> list[str]:
|
||||
"""Extract ``content.subject`` from each SSE ``data`` payload.
|
||||
|
||||
Verifies JSON shape rather than substring-matching raw ``data`` strings — a
|
||||
regression in the envelope (missing id, wrong serializer, content key
|
||||
renamed) fails here instead of silently passing. Each ``data`` payload is
|
||||
a ``NotificationResponse`` dump with a top-level ``content.subject``.
|
||||
"""
|
||||
subjects = []
|
||||
for event in events:
|
||||
payload = json.loads(event["data"])
|
||||
subjects.append(payload["content"]["subject"])
|
||||
return subjects
|
||||
|
||||
|
||||
class TestNotificationSSEIntegration(IntegrationTestCase):
|
||||
dataset_populator: DatasetPopulator
|
||||
framework_tool_and_types = False
|
||||
|
||||
@classmethod
|
||||
def handle_galaxy_config_kwds(cls, config):
|
||||
super().handle_galaxy_config_kwds(config)
|
||||
config["enable_celery_tasks"] = False
|
||||
config["enable_notification_system"] = True
|
||||
|
||||
def setUp(self):
|
||||
super().setUp()
|
||||
self.dataset_populator = DatasetPopulator(self.galaxy_interactor)
|
||||
|
||||
def _stream_url(self) -> str:
|
||||
return urljoin(self.url, "api/events/stream")
|
||||
|
||||
def test_sse_receives_notification_events(self):
|
||||
"""When a notification is created, the SSE stream should receive it."""
|
||||
user = self._setup_user(f"{uuid4()}@galaxy.test")
|
||||
_, user_api_key = self._setup_user_get_key(user["email"])
|
||||
|
||||
listener = SSELineListener(self._stream_url(), user_api_key)
|
||||
listener.start()
|
||||
try:
|
||||
subject = f"sse_test_{uuid4()}"
|
||||
request = {
|
||||
"recipients": {"user_ids": [user["id"]]},
|
||||
"notification": notification_test_data(subject=subject, message="SSE test notification"),
|
||||
}
|
||||
response = self._post("notifications", data=request, admin=True, json=True)
|
||||
self._assert_status_code_is_ok(response)
|
||||
|
||||
notification_events = listener.wait_for_event("notification_update")
|
||||
finally:
|
||||
listener.stop()
|
||||
|
||||
assert subject in _notification_subjects(
|
||||
notification_events
|
||||
), f"Expected subject '{subject}' in SSE events, got: {notification_events}"
|
||||
|
||||
def test_sse_receives_broadcast_events(self):
|
||||
"""When a broadcast is created, the SSE stream should receive it."""
|
||||
api_key = self.galaxy_interactor.api_key
|
||||
assert api_key is not None
|
||||
listener = SSELineListener(self._stream_url(), api_key)
|
||||
listener.start()
|
||||
try:
|
||||
subject = f"broadcast_sse_test_{uuid4()}"
|
||||
payload = notification_broadcast_test_data(subject=subject)
|
||||
response = self._post("notifications/broadcast", data=payload, admin=True, json=True)
|
||||
self._assert_status_code_is_ok(response)
|
||||
|
||||
broadcast_events = listener.wait_for_event("broadcast_update")
|
||||
finally:
|
||||
listener.stop()
|
||||
|
||||
# Broadcast events carry a BroadcastNotificationResponse, which shares
|
||||
# the top-level content.subject shape with per-user notifications.
|
||||
broadcast_subjects = [json.loads(e["data"])["content"]["subject"] for e in broadcast_events]
|
||||
assert subject in broadcast_subjects, f"Expected subject '{subject}' in broadcast events: {broadcast_events}"
|
||||
|
||||
def test_sse_catchup_on_reconnect(self):
|
||||
"""Reconnecting with Last-Event-ID should replay a catch-up notification_status event.
|
||||
|
||||
The ``Last-Event-ID`` value is the server-issued ID from a prior event,
|
||||
not a client-side timestamp. This avoids clock-skew flake between the
|
||||
test runner and the app in containerized CI.
|
||||
"""
|
||||
user = self._setup_user(f"{uuid4()}@galaxy.test")
|
||||
_, user_api_key = self._setup_user_get_key(user["email"])
|
||||
|
||||
# First connection: capture the server-issued event id of the first notification.
|
||||
listener_1 = SSELineListener(self._stream_url(), user_api_key)
|
||||
listener_1.start()
|
||||
try:
|
||||
subject_1 = f"first_{uuid4()}"
|
||||
request = {
|
||||
"recipients": {"user_ids": [user["id"]]},
|
||||
"notification": notification_test_data(subject=subject_1),
|
||||
}
|
||||
response = self._post("notifications", data=request, admin=True, json=True)
|
||||
self._assert_status_code_is_ok(response)
|
||||
first_events = listener_1.wait_for_event("notification_update")
|
||||
finally:
|
||||
listener_1.stop()
|
||||
|
||||
last_event_id = next((e["id"] for e in first_events if e.get("id")), None)
|
||||
assert last_event_id, f"No server-issued id on first notification event: {first_events}"
|
||||
|
||||
# Emit a second notification while disconnected; it should appear in the catch-up.
|
||||
subject_2 = f"catchup_{uuid4()}"
|
||||
request = {
|
||||
"recipients": {"user_ids": [user["id"]]},
|
||||
"notification": notification_test_data(subject=subject_2, message="Catch-up test"),
|
||||
}
|
||||
response = self._post("notifications", data=request, admin=True, json=True)
|
||||
self._assert_status_code_is_ok(response)
|
||||
|
||||
# Reconnect with Last-Event-ID = the captured id. The catch-up must include
|
||||
# the notification sent after that id but not the one that produced it.
|
||||
listener_2 = SSELineListener(
|
||||
self._stream_url(),
|
||||
user_api_key,
|
||||
headers={"Last-Event-ID": last_event_id},
|
||||
)
|
||||
listener_2.start()
|
||||
try:
|
||||
status_events = listener_2.wait_for_event("notification_status")
|
||||
finally:
|
||||
listener_2.stop()
|
||||
|
||||
replayed_subjects: list[str] = []
|
||||
for event in status_events:
|
||||
payload = json.loads(event["data"])
|
||||
replayed_subjects.extend(n["content"]["subject"] for n in payload.get("notifications", []))
|
||||
assert subject_2 in replayed_subjects, f"Missed catch-up of '{subject_2}': {status_events}"
|
||||
assert (
|
||||
subject_1 not in replayed_subjects
|
||||
), f"Last-Event-ID did not filter — '{subject_1}' replayed: {status_events}"
|
||||
@@ -0,0 +1,113 @@
|
||||
"""Playwright E2E test for the interactive-tool entry-point SSE pipeline.
|
||||
|
||||
Verifies that when an interactive-tool entry point transitions to ``configured``
|
||||
server-side (the runner's port-routing hook calls ``configure_entry_points``),
|
||||
a logged-in user's browser receives the ``entry_point_update`` SSE event and
|
||||
the entry-point store refetches, without the 10 s polling interval.
|
||||
|
||||
This test stubs the server-side runner callback: it creates the Job and entry
|
||||
point directly and calls ``InteractiveToolManager.configure_entry_points`` on
|
||||
the live app. That invocation is the exact dispatch site in production.
|
||||
"""
|
||||
|
||||
from uuid import uuid4
|
||||
|
||||
from galaxy.model import (
|
||||
InteractiveToolEntryPoint,
|
||||
Job,
|
||||
)
|
||||
from galaxy.util.wait import wait_on
|
||||
from galaxy_test.selenium.framework import (
|
||||
managed_history,
|
||||
selenium_test,
|
||||
)
|
||||
from .framework import SeleniumIntegrationTestCase
|
||||
|
||||
SSE_CONNECT_TIMEOUT_SECONDS = 15
|
||||
SSE_EVENT_TIMEOUT_SECONDS = 15
|
||||
|
||||
|
||||
class TestEntryPointSSESeleniumIntegration(SeleniumIntegrationTestCase):
|
||||
ensure_registered = True
|
||||
|
||||
@classmethod
|
||||
def handle_galaxy_config_kwds(cls, config):
|
||||
super().handle_galaxy_config_kwds(config)
|
||||
config["enable_celery_tasks"] = False
|
||||
# App.vue only calls entryPointStore.startWatchingEntryPoints() when
|
||||
# interactivetools_enable is True, and the store only opens an SSE
|
||||
# connection when enable_sse_updates is True. Without both,
|
||||
# __galaxy_sse_connected never becomes true and the gate below times out.
|
||||
config["interactivetools_enable"] = True
|
||||
config["enable_sse_updates"] = True
|
||||
|
||||
def _wait_for_sse_connected(self) -> None:
|
||||
"""Block until the frontend confirms the SSE pipeline is live."""
|
||||
wait_on(
|
||||
lambda: True if self.execute_script("return window.__galaxy_sse_connected === true") else None,
|
||||
"window.__galaxy_sse_connected === true",
|
||||
timeout=SSE_CONNECT_TIMEOUT_SECONDS,
|
||||
)
|
||||
|
||||
def _last_sse_event_ts(self) -> int:
|
||||
return self.execute_script("return window.__galaxy_sse_last_event_ts || 0") or 0
|
||||
|
||||
def _wait_for_sse_event_after(self, baseline_ts: int) -> None:
|
||||
wait_on(
|
||||
lambda: True if self._last_sse_event_ts() > baseline_ts else None,
|
||||
"window.__galaxy_sse_last_event_ts advanced past baseline",
|
||||
timeout=SSE_EVENT_TIMEOUT_SECONDS,
|
||||
)
|
||||
|
||||
def _create_it_job_with_entry_point(self, tool_port: int = 8888) -> tuple[int, int]:
|
||||
# Use the browser's cookie-authenticated user, not the API interactor's
|
||||
# default: SSE connects under the Selenium-registered user, and the
|
||||
# dispatch's user_id must match or push_to_user finds no queues.
|
||||
user_info = self.api_get("users/current")
|
||||
user_id = self._decode_id(user_info["id"])
|
||||
sa_session = self._app.model.context
|
||||
job = Job()
|
||||
job.user_id = user_id
|
||||
job.tool_id = "interactivetool_simple"
|
||||
job.state = Job.states.RUNNING
|
||||
sa_session.add(job)
|
||||
sa_session.flush()
|
||||
ep = InteractiveToolEntryPoint(
|
||||
job=job,
|
||||
tool_port=tool_port,
|
||||
entry_url="/",
|
||||
name=f"selenium entry {uuid4()}",
|
||||
label="selenium",
|
||||
requires_domain=True,
|
||||
requires_path_in_url=False,
|
||||
requires_path_in_header_named=None,
|
||||
)
|
||||
sa_session.add(ep)
|
||||
sa_session.commit()
|
||||
return job.id, ep.id
|
||||
|
||||
@selenium_test
|
||||
@managed_history
|
||||
def test_entry_point_update_pushed_via_sse(self):
|
||||
"""configure_entry_points should trigger an SSE push the client observes."""
|
||||
# Navigate home so the entry-point store is mounted and subscribed.
|
||||
self.home()
|
||||
self._wait_for_sse_connected()
|
||||
baseline_ts = self._last_sse_event_ts()
|
||||
self.screenshot("entry_point_sse_before")
|
||||
|
||||
job_id, _ep_id = self._create_it_job_with_entry_point(tool_port=8888)
|
||||
|
||||
# Stub the runner hook: call configure_entry_points on the live app.
|
||||
sa_session = self._app.model.context
|
||||
job = sa_session.get(Job, job_id)
|
||||
assert job is not None
|
||||
self._app.interactivetool_manager.configure_entry_points(
|
||||
job,
|
||||
{"8888": {"host": "host.invalid", "port": 12345, "protocol": "http"}},
|
||||
)
|
||||
|
||||
# Prove the update arrived via SSE (not polling): the composable's
|
||||
# event-timestamp hook only advances when useSSE's listener fires.
|
||||
self._wait_for_sse_event_after(baseline_ts)
|
||||
self.screenshot("entry_point_sse_after")
|
||||
@@ -0,0 +1,145 @@
|
||||
"""Playwright E2E test for the notification SSE pipeline.
|
||||
|
||||
Verifies that when an admin creates a notification via the API,
|
||||
a logged-in user sees it appear in the UI in real-time (within seconds)
|
||||
without a page refresh, proving the SSE push pipeline works end-to-end.
|
||||
"""
|
||||
|
||||
from uuid import uuid4
|
||||
|
||||
from galaxy.util.wait import wait_on
|
||||
from galaxy_test.selenium.framework import (
|
||||
managed_history,
|
||||
selenium_test,
|
||||
)
|
||||
from .framework import SeleniumIntegrationTestCase
|
||||
|
||||
SSE_CONNECT_TIMEOUT_SECONDS = 15
|
||||
SSE_EVENT_TIMEOUT_SECONDS = 15
|
||||
|
||||
|
||||
class TestNotificationSSESeleniumIntegration(SeleniumIntegrationTestCase):
|
||||
ensure_registered = True
|
||||
|
||||
@classmethod
|
||||
def handle_galaxy_config_kwds(cls, config):
|
||||
super().handle_galaxy_config_kwds(config)
|
||||
config["enable_notification_system"] = True
|
||||
config["enable_sse_updates"] = True
|
||||
config["enable_celery_tasks"] = False
|
||||
|
||||
def _wait_for_sse_connected(self) -> None:
|
||||
"""Block until the frontend confirms the SSE pipeline is live.
|
||||
|
||||
Without this gate, the 30 s polling fallback silently masks a broken
|
||||
SSE pipeline — the UI would still update, but via polling, and the
|
||||
test would falsely pass.
|
||||
"""
|
||||
wait_on(
|
||||
lambda: True if self.execute_script("return window.__galaxy_sse_connected === true") else None,
|
||||
"window.__galaxy_sse_connected === true",
|
||||
timeout=SSE_CONNECT_TIMEOUT_SECONDS,
|
||||
)
|
||||
|
||||
def _last_sse_event_ts(self) -> int:
|
||||
"""Return the last SSE event timestamp recorded by the composable, or 0."""
|
||||
return self.execute_script("return window.__galaxy_sse_last_event_ts || 0") or 0
|
||||
|
||||
def _wait_for_sse_event_after(self, baseline_ts: int) -> None:
|
||||
"""Block until an SSE event arrives after ``baseline_ts``.
|
||||
|
||||
Guards against a silent regression where the UI update originates from
|
||||
the polling fallback rather than the SSE push: ``__galaxy_sse_last_event_ts``
|
||||
only advances when the composable's event listener fires.
|
||||
"""
|
||||
wait_on(
|
||||
lambda: True if self._last_sse_event_ts() > baseline_ts else None,
|
||||
"window.__galaxy_sse_last_event_ts advanced past baseline",
|
||||
timeout=SSE_EVENT_TIMEOUT_SECONDS,
|
||||
)
|
||||
|
||||
@selenium_test
|
||||
@managed_history
|
||||
def test_notification_appears_via_sse(self):
|
||||
"""Send a notification via the API and verify it appears in the UI without refresh."""
|
||||
# Get the browser-logged-in user's info via the browser's cookie. ``self._get``
|
||||
# uses the API interactor's default-user key, which does not match the Selenium
|
||||
# user created by ``ensure_registered``, so the SSE push would target a
|
||||
# different user than the one watching the stream.
|
||||
user_info = self.api_get("users/current")
|
||||
user_id = user_info["id"]
|
||||
|
||||
# Navigate to notifications page so the store is watching.
|
||||
# ``get()`` uses ``build_url()`` which handles trailing slashes on the
|
||||
# base correctly; concatenating against ``target_url_from_selenium``
|
||||
# can produce a double-slash that Galaxy routes differently.
|
||||
self.get("user/notifications")
|
||||
self._wait_for_sse_connected()
|
||||
baseline_ts = self._last_sse_event_ts()
|
||||
self.screenshot("notification_sse_before")
|
||||
|
||||
# Send a notification to this user via the admin API
|
||||
subject = f"SSE E2E Test {uuid4()}"
|
||||
notification_request = {
|
||||
"recipients": {"user_ids": [user_id]},
|
||||
"notification": {
|
||||
"source": "integration_tests",
|
||||
"variant": "info",
|
||||
"category": "message",
|
||||
"content": {
|
||||
"category": "message",
|
||||
"subject": subject,
|
||||
"message": "This notification was pushed via SSE",
|
||||
},
|
||||
},
|
||||
}
|
||||
response = self._post("notifications", data=notification_request, admin=True, json=True)
|
||||
self._assert_status_code_is_ok(response)
|
||||
|
||||
# Prove the incoming update arrived via SSE: the event-timestamp hook
|
||||
# only advances when useSSE's listener fires. If this times out while
|
||||
# the UI still shows the notification, polling picked it up — a silent
|
||||
# regression this assertion catches.
|
||||
self._wait_for_sse_event_after(baseline_ts)
|
||||
# Use xpath (works on both Selenium and Playwright backends); `text=` is a
|
||||
# Playwright engine prefix that Selenium's css-selector path rejects.
|
||||
self.wait_for_xpath_visible(f'//*[contains(text(), "{subject}")]', timeout=SSE_EVENT_TIMEOUT_SECONDS)
|
||||
self.screenshot("notification_sse_after")
|
||||
|
||||
@selenium_test
|
||||
@managed_history
|
||||
def test_notification_bell_updates_via_sse(self):
|
||||
"""The notification bell indicator should update when a new notification arrives via SSE."""
|
||||
# See ``test_notification_appears_via_sse`` — must use the browser's user,
|
||||
# not the API interactor's default user.
|
||||
user_info = self.api_get("users/current")
|
||||
user_id = user_info["id"]
|
||||
|
||||
# Go to home page (bell is in masthead)
|
||||
self.home()
|
||||
self._wait_for_sse_connected()
|
||||
baseline_ts = self._last_sse_event_ts()
|
||||
|
||||
# Send a notification
|
||||
subject = f"Bell Test {uuid4()}"
|
||||
notification_request = {
|
||||
"recipients": {"user_ids": [user_id]},
|
||||
"notification": {
|
||||
"source": "integration_tests",
|
||||
"variant": "info",
|
||||
"category": "message",
|
||||
"content": {
|
||||
"category": "message",
|
||||
"subject": subject,
|
||||
"message": "Testing bell indicator update via SSE",
|
||||
},
|
||||
},
|
||||
}
|
||||
response = self._post("notifications", data=notification_request, admin=True, json=True)
|
||||
self._assert_status_code_is_ok(response)
|
||||
|
||||
self._wait_for_sse_event_after(baseline_ts)
|
||||
# The activity-bar notifications item renders its unread-count badge as
|
||||
# ``.nav-indicator`` (see ``ActivityItem.vue``) once ``totalUnreadCount > 0``.
|
||||
self.wait_for_selector_visible("#activity-notifications .nav-indicator", timeout=SSE_EVENT_TIMEOUT_SECONDS)
|
||||
self.screenshot("notification_bell_indicator")
|
||||
@@ -0,0 +1,227 @@
|
||||
"""Unit tests for :mod:`galaxy.managers.queue_metrics`.
|
||||
|
||||
The ``SSEConnectionManager`` and the kombu connection are small hand-built
|
||||
fakes — no broker or database is required. Assertions are on the recorded
|
||||
state of the statsd client (counters, timings) rather than on mock call-lists
|
||||
so a regression that stops emitting a gauge fails the test for the right
|
||||
reason.
|
||||
|
||||
The failure-isolation test drives real sub-emitters into their error paths by
|
||||
handing them genuinely broken collaborators (a connection whose ``clone()``
|
||||
raises, a model whose ``new_session()`` raises). That way the test exercises
|
||||
the real ``_run`` wrapper rather than asserting a monkey-patched side_effect.
|
||||
"""
|
||||
|
||||
from dataclasses import (
|
||||
dataclass,
|
||||
field,
|
||||
)
|
||||
from types import SimpleNamespace
|
||||
from typing import (
|
||||
cast,
|
||||
Optional,
|
||||
)
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
from galaxy.managers import queue_metrics
|
||||
from galaxy.managers.sse import SSEConnectionManager
|
||||
from galaxy.model.mapping import GalaxyModelMapping
|
||||
from galaxy.web.statsd_client import VanillaGalaxyStatsdClient
|
||||
from galaxy.web_stack import ApplicationStack
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Fakes
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@dataclass
|
||||
class FakeStatsdClient:
|
||||
"""In-memory statsd recorder — see docstring in ``test_sse_dispatch.py``."""
|
||||
|
||||
counters: dict[tuple[str, tuple[tuple[str, str], ...]], int] = field(default_factory=dict)
|
||||
timings: list[tuple[str, float, tuple[tuple[str, str], ...]]] = field(default_factory=list)
|
||||
|
||||
def incr(self, metric: str, tags: Optional[dict[str, str]] = None) -> None:
|
||||
key = (metric, tuple(sorted((tags or {}).items())))
|
||||
self.counters[key] = self.counters.get(key, 0) + 1
|
||||
|
||||
def timing(self, metric: str, value: float, tags: Optional[dict[str, str]] = None) -> None:
|
||||
self.timings.append((metric, value, tuple(sorted((tags or {}).items()))))
|
||||
|
||||
def counter(self, metric: str, tags: Optional[dict[str, str]] = None) -> int:
|
||||
return self.counters.get((metric, tuple(sorted((tags or {}).items()))), 0)
|
||||
|
||||
def timings_for(self, metric: str) -> list[tuple[float, dict[str, str]]]:
|
||||
return [(v, dict(t)) for m, v, t in self.timings if m == metric]
|
||||
|
||||
|
||||
def _fake_sse_manager(broadcast: int = 3, per_user: int = 5):
|
||||
m = MagicMock(spec=SSEConnectionManager)
|
||||
m.total_broadcast_connections = broadcast
|
||||
m.total_per_user_connections = per_user
|
||||
return m
|
||||
|
||||
|
||||
def _make_fake_queue(name: str, count: int):
|
||||
"""Fake kombu Queue exposing ``.bind(channel).queue_declare(passive=True).message_count``."""
|
||||
declared = SimpleNamespace(message_count=count)
|
||||
bound = SimpleNamespace(queue_declare=lambda passive: declared)
|
||||
return SimpleNamespace(name=name, bind=lambda channel: bound)
|
||||
|
||||
|
||||
def _make_fake_connection(channel=None):
|
||||
"""Fake kombu Connection whose ``.clone()`` is usable as a context manager."""
|
||||
channel = channel or MagicMock()
|
||||
conn_cm = MagicMock()
|
||||
conn_cm.channel.return_value = channel
|
||||
cm = MagicMock()
|
||||
cm.__enter__ = lambda self: conn_cm
|
||||
cm.__exit__ = lambda self, *a: False
|
||||
connection = MagicMock()
|
||||
connection.clone.return_value = cm
|
||||
return connection
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Sub-emitter tests — assert on recorded state
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_emit_sse_connection_gauges_emits_both_kinds():
|
||||
statsd = FakeStatsdClient()
|
||||
queue_metrics.emit_sse_connection_gauges(
|
||||
cast(VanillaGalaxyStatsdClient, statsd), _fake_sse_manager(broadcast=4, per_user=7)
|
||||
)
|
||||
|
||||
assert statsd.timings_for("galaxy.sse.connections.active") == [
|
||||
(4, {"kind": "broadcast"}),
|
||||
(7, {"kind": "per_user"}),
|
||||
]
|
||||
|
||||
|
||||
def test_emit_control_queue_depth_emits_per_queue(monkeypatch):
|
||||
statsd = FakeStatsdClient()
|
||||
fake_queues = [
|
||||
_make_fake_queue("control.main@h", 3),
|
||||
_make_fake_queue("control.main.1@h", 0),
|
||||
]
|
||||
monkeypatch.setattr(
|
||||
queue_metrics,
|
||||
"all_control_queues_for_declare",
|
||||
lambda application_stack: fake_queues,
|
||||
)
|
||||
|
||||
queue_metrics.emit_control_queue_depth(
|
||||
cast(VanillaGalaxyStatsdClient, statsd),
|
||||
_make_fake_connection(),
|
||||
cast(ApplicationStack, MagicMock()),
|
||||
)
|
||||
|
||||
assert statsd.timings_for("galaxy.control_queue.depth") == [
|
||||
(3, {"queue_name": "control.main@h"}),
|
||||
(0, {"queue_name": "control.main.1@h"}),
|
||||
]
|
||||
|
||||
|
||||
def test_emit_control_queue_depth_skips_failed_passive_declare(monkeypatch):
|
||||
"""One bad queue → we skip it and keep going for the rest."""
|
||||
statsd = FakeStatsdClient()
|
||||
|
||||
def bad_declare(passive):
|
||||
raise RuntimeError("queue does not exist yet")
|
||||
|
||||
good_queue = _make_fake_queue("control.good@h", 9)
|
||||
bad_queue = SimpleNamespace(
|
||||
name="control.bad@h",
|
||||
bind=lambda channel: SimpleNamespace(queue_declare=bad_declare),
|
||||
)
|
||||
monkeypatch.setattr(queue_metrics, "all_control_queues_for_declare", lambda stack: [good_queue, bad_queue])
|
||||
|
||||
queue_metrics.emit_control_queue_depth(
|
||||
cast(VanillaGalaxyStatsdClient, statsd),
|
||||
_make_fake_connection(),
|
||||
cast(ApplicationStack, MagicMock()),
|
||||
)
|
||||
|
||||
assert statsd.timings_for("galaxy.control_queue.depth") == [
|
||||
(9, {"queue_name": "control.good@h"}),
|
||||
]
|
||||
|
||||
|
||||
def test_emit_control_queue_depth_no_broker_connection_is_noop():
|
||||
statsd = FakeStatsdClient()
|
||||
queue_metrics.emit_control_queue_depth(
|
||||
cast(VanillaGalaxyStatsdClient, statsd), None, cast(ApplicationStack, MagicMock())
|
||||
)
|
||||
assert statsd.timings == []
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# emit_queue_metrics — aggregate entry-point
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_emit_queue_metrics_is_silent_when_statsd_is_none():
|
||||
"""No statsd client → every sub-call is skipped, no DB or broker access."""
|
||||
# Would raise AttributeError if the short-circuit didn't fire before any
|
||||
# real collaborator was touched.
|
||||
queue_metrics.emit_queue_metrics(
|
||||
statsd_client=None,
|
||||
connection=None,
|
||||
application_stack=cast(ApplicationStack, MagicMock()),
|
||||
model=cast(GalaxyModelMapping, MagicMock()),
|
||||
sse_manager=None,
|
||||
)
|
||||
|
||||
|
||||
def test_emit_queue_metrics_isolates_real_subemitter_failures(monkeypatch):
|
||||
"""When real sub-emitters raise, ``_run`` contains the failure and logs an error counter.
|
||||
|
||||
We drive the failures through the actual sub-emitter bodies — not monkey-
|
||||
patched side_effects — by handing in a connection whose ``.clone()`` raises
|
||||
(for ``emit_control_queue_depth``) and a model whose ``.new_session()``
|
||||
raises (for ``emit_worker_process_gauge``). The SSE gauge is given healthy
|
||||
collaborators and should still land.
|
||||
"""
|
||||
statsd = FakeStatsdClient()
|
||||
sse_manager = _fake_sse_manager(broadcast=2, per_user=1)
|
||||
|
||||
broken_connection = MagicMock()
|
||||
broken_connection.clone.side_effect = RuntimeError("broker is gone")
|
||||
|
||||
broken_model = MagicMock()
|
||||
broken_model.new_session.side_effect = RuntimeError("db is gone")
|
||||
|
||||
# Ensure the broker path reaches .clone() rather than short-circuiting on
|
||||
# an empty queue list.
|
||||
monkeypatch.setattr(
|
||||
queue_metrics,
|
||||
"all_control_queues_for_declare",
|
||||
lambda stack: [_make_fake_queue("control.main@h", 0)],
|
||||
)
|
||||
|
||||
# Must not raise — the SSE gauge still lands.
|
||||
queue_metrics.emit_queue_metrics(
|
||||
statsd_client=cast(VanillaGalaxyStatsdClient, statsd),
|
||||
connection=broken_connection,
|
||||
application_stack=cast(ApplicationStack, MagicMock()),
|
||||
model=cast(GalaxyModelMapping, broken_model),
|
||||
sse_manager=sse_manager,
|
||||
)
|
||||
|
||||
# SSE gauge landed despite the other two failing.
|
||||
sse_timings = statsd.timings_for("galaxy.sse.connections.active")
|
||||
assert (2, {"kind": "broadcast"}) in sse_timings
|
||||
assert (1, {"kind": "per_user"}) in sse_timings
|
||||
|
||||
# Each failing sub-emitter bumped its error counter tagged by name.
|
||||
assert statsd.counter("galaxy.queue_metrics.error", {"emitter": "control_queue_depth"}) == 1
|
||||
assert statsd.counter("galaxy.queue_metrics.error", {"emitter": "worker_process"}) == 1
|
||||
# The healthy SSE sub-emitter did not.
|
||||
assert statsd.counter("galaxy.queue_metrics.error", {"emitter": "sse_connections"}) == 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
pytest.main([__file__, "-v"])
|
||||
@@ -0,0 +1,236 @@
|
||||
"""Unit tests for :class:`galaxy.managers.sse_dispatch.SSEEventDispatcher` observability.
|
||||
|
||||
Focus is on the statsd instrumentation contract AND the effect of dispatch:
|
||||
counters/timers fire on the happy path and the ``_queue_worker is None``
|
||||
early-return, payloads reach the broker with the expected task+kwargs, and the
|
||||
dispatcher is a silent no-op when ``statsd_client`` is ``None``.
|
||||
|
||||
These tests use lightweight fakes (``FakeStatsdClient``, ``FakeControlTask``)
|
||||
that record state we can assert against — rather than ``MagicMock`` call-lists —
|
||||
so a regression that silently drops dispatch or stops recording metrics fails
|
||||
the test for the right reason.
|
||||
"""
|
||||
|
||||
from dataclasses import (
|
||||
dataclass,
|
||||
field,
|
||||
)
|
||||
from typing import (
|
||||
Any,
|
||||
Optional,
|
||||
)
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
from galaxy.managers.sse_dispatch import SSEEventDispatcher
|
||||
|
||||
|
||||
@dataclass
|
||||
class FakeStatsdClient:
|
||||
"""In-memory stand-in for ``VanillaGalaxyStatsdClient``.
|
||||
|
||||
Records ``incr`` and ``timing`` calls as plain data so tests assert on
|
||||
observable state (counter totals, recorded timings) instead of mock
|
||||
call-lists.
|
||||
"""
|
||||
|
||||
counters: dict[tuple[str, tuple[tuple[str, str], ...]], int] = field(default_factory=dict)
|
||||
timings: list[tuple[str, float, tuple[tuple[str, str], ...]]] = field(default_factory=list)
|
||||
|
||||
def incr(self, metric: str, tags: Optional[dict[str, str]] = None) -> None:
|
||||
key = (metric, tuple(sorted((tags or {}).items())))
|
||||
self.counters[key] = self.counters.get(key, 0) + 1
|
||||
|
||||
def timing(self, metric: str, value: float, tags: Optional[dict[str, str]] = None) -> None:
|
||||
self.timings.append((metric, value, tuple(sorted((tags or {}).items()))))
|
||||
|
||||
def counter(self, metric: str, tags: Optional[dict[str, str]] = None) -> int:
|
||||
return self.counters.get((metric, tuple(sorted((tags or {}).items()))), 0)
|
||||
|
||||
|
||||
@dataclass
|
||||
class RecordedTask:
|
||||
payload: dict[str, Any]
|
||||
routing_key: str
|
||||
expiration: Optional[int]
|
||||
declare_queues: Any
|
||||
|
||||
|
||||
class FakeControlTask:
|
||||
"""Stand-in for ``ControlTask`` that records dispatches instead of touching AMQP."""
|
||||
|
||||
instances: list["FakeControlTask"] = []
|
||||
|
||||
def __init__(self, queue_worker) -> None:
|
||||
self.queue_worker = queue_worker
|
||||
self.sent: list[RecordedTask] = []
|
||||
FakeControlTask.instances.append(self)
|
||||
|
||||
def send_task(
|
||||
self,
|
||||
payload: dict[str, Any],
|
||||
routing_key: str,
|
||||
expiration: Optional[int] = None,
|
||||
declare_queues: Any = None,
|
||||
**_: Any,
|
||||
) -> None:
|
||||
self.sent.append(
|
||||
RecordedTask(
|
||||
payload=payload,
|
||||
routing_key=routing_key,
|
||||
expiration=expiration,
|
||||
declare_queues=declare_queues,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
class BoomControlTask:
|
||||
"""``ControlTask`` fake whose ``send_task`` always raises — exercises the finally block."""
|
||||
|
||||
def __init__(self, queue_worker) -> None:
|
||||
self.queue_worker = queue_worker
|
||||
|
||||
def send_task(self, **kwargs) -> None:
|
||||
raise RuntimeError("broker down")
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def application_stack():
|
||||
return MagicMock(name="ApplicationStack")
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def queue_worker():
|
||||
return MagicMock(name="GalaxyQueueWorker")
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def statsd() -> FakeStatsdClient:
|
||||
return FakeStatsdClient()
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _reset_fake_control_task_instances():
|
||||
FakeControlTask.instances.clear()
|
||||
yield
|
||||
FakeControlTask.instances.clear()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def fake_control_task() -> type[FakeControlTask]:
|
||||
"""Returns the fake class so tests can pass it as ``control_task_factory``."""
|
||||
return FakeControlTask
|
||||
|
||||
|
||||
def _dispatcher_with_fakes(
|
||||
queue_worker,
|
||||
application_stack,
|
||||
statsd,
|
||||
control_task_factory=FakeControlTask,
|
||||
queues=None,
|
||||
) -> SSEEventDispatcher:
|
||||
"""Build a dispatcher wired to injected fakes — no monkeypatching required."""
|
||||
return SSEEventDispatcher(
|
||||
queue_worker=queue_worker,
|
||||
application_stack=application_stack,
|
||||
statsd_client=statsd,
|
||||
control_task_factory=control_task_factory,
|
||||
queues_provider=lambda: queues if queues is not None else [],
|
||||
)
|
||||
|
||||
|
||||
def test_dispatcher_no_op_when_queue_worker_is_none_and_no_statsd(application_stack):
|
||||
"""No statsd client set and no queue_worker → silent no-op, no AttributeError."""
|
||||
dispatcher = SSEEventDispatcher(
|
||||
queue_worker=None,
|
||||
application_stack=application_stack,
|
||||
statsd_client=None,
|
||||
)
|
||||
# Must not raise, must not attempt to declare queues.
|
||||
dispatcher.notify_users([1, 2], "hello")
|
||||
dispatcher.notify_broadcast("hi")
|
||||
dispatcher.history_update({"1": [42]})
|
||||
|
||||
|
||||
def test_dispatcher_records_skipped_counter_when_queue_worker_is_none(application_stack, statsd):
|
||||
"""Two dispatches with no queue_worker → two skipped_no_qw increments, no timings."""
|
||||
dispatcher = SSEEventDispatcher(
|
||||
queue_worker=None,
|
||||
application_stack=application_stack,
|
||||
statsd_client=statsd,
|
||||
)
|
||||
dispatcher.notify_users([1], "hello")
|
||||
dispatcher.notify_broadcast("world")
|
||||
|
||||
assert statsd.counter("galaxy.sse.dispatch.skipped_no_qw") == 2
|
||||
# No latency timing — we never got as far as the broker call.
|
||||
assert statsd.timings == []
|
||||
|
||||
|
||||
def test_dispatcher_enqueues_payload_and_records_metrics_on_send(
|
||||
application_stack, queue_worker, statsd, fake_control_task
|
||||
):
|
||||
"""Happy path: payload reaches the broker AND counter+timer are recorded."""
|
||||
dispatcher = _dispatcher_with_fakes(
|
||||
queue_worker=queue_worker,
|
||||
application_stack=application_stack,
|
||||
statsd=statsd,
|
||||
control_task_factory=fake_control_task,
|
||||
)
|
||||
dispatcher.notify_users([1, 2], "hello")
|
||||
|
||||
# Exactly one ControlTask constructed, one send_task recorded with the right
|
||||
# payload — asserts the dispatch *effect*, not just that a mock was called.
|
||||
assert len(fake_control_task.instances) == 1
|
||||
sent = fake_control_task.instances[0].sent
|
||||
assert len(sent) == 1
|
||||
assert sent[0].payload["task"] == "notify_users"
|
||||
assert sent[0].payload["kwargs"]["user_ids"] == [1, 2]
|
||||
assert sent[0].payload["kwargs"]["payload"] == "hello"
|
||||
assert "event_id" in sent[0].payload["kwargs"]
|
||||
assert sent[0].routing_key == "control.*"
|
||||
assert sent[0].expiration == 10
|
||||
|
||||
# Counter + timer both recorded with matching task tag.
|
||||
assert statsd.counter("galaxy.sse.dispatch.count", {"task": "notify_users"}) == 1
|
||||
assert len(statsd.timings) == 1
|
||||
metric, _value, tags = statsd.timings[0]
|
||||
assert metric == "galaxy.sse.dispatch.latency_ms"
|
||||
assert dict(tags) == {"task": "notify_users"}
|
||||
|
||||
|
||||
def test_dispatcher_timer_still_fires_on_send_exception(application_stack, queue_worker, statsd):
|
||||
"""Timer lives in ``finally`` — broker errors don't mask the latency metric."""
|
||||
dispatcher = _dispatcher_with_fakes(
|
||||
queue_worker=queue_worker,
|
||||
application_stack=application_stack,
|
||||
statsd=statsd,
|
||||
control_task_factory=BoomControlTask,
|
||||
)
|
||||
with pytest.raises(RuntimeError):
|
||||
dispatcher.history_update({"7": [1]})
|
||||
|
||||
assert statsd.counter("galaxy.sse.dispatch.count", {"task": "history_update"}) == 1
|
||||
assert len(statsd.timings) == 1
|
||||
metric, _value, tags = statsd.timings[0]
|
||||
assert metric == "galaxy.sse.dispatch.latency_ms"
|
||||
assert dict(tags) == {"task": "history_update"}
|
||||
|
||||
|
||||
def test_dispatcher_no_statsd_means_no_instrumentation(application_stack, queue_worker, fake_control_task):
|
||||
"""When ``statsd_client`` is ``None`` instrumentation is bypassed entirely.
|
||||
|
||||
The dispatch still happens — we assert via the ControlTask fake — but there
|
||||
is nothing to observe on the (absent) statsd side.
|
||||
"""
|
||||
dispatcher = _dispatcher_with_fakes(
|
||||
queue_worker=queue_worker,
|
||||
application_stack=application_stack,
|
||||
statsd=None,
|
||||
control_task_factory=fake_control_task,
|
||||
)
|
||||
dispatcher.notify_broadcast("hi")
|
||||
assert len(fake_control_task.instances) == 1
|
||||
assert len(fake_control_task.instances[0].sent) == 1
|
||||
assert fake_control_task.instances[0].sent[0].payload["task"] == "notify_broadcast"
|
||||
@@ -0,0 +1,144 @@
|
||||
"""Tests for the TTL cache on ``SSEEventDispatcher._get_declare_queues``.
|
||||
|
||||
The cache exists because ``_send`` is on a hot path (1000+ events/s at target
|
||||
load) and without it each dispatch fires a ``WorkerProcess`` DB query. The
|
||||
underlying data only changes on a 60 s heartbeat cadence, so a 30 s TTL is safe.
|
||||
"""
|
||||
|
||||
from concurrent.futures import (
|
||||
as_completed,
|
||||
ThreadPoolExecutor,
|
||||
)
|
||||
from typing import Any
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
from galaxy.managers.sse_dispatch import SSEEventDispatcher
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def fake_declare():
|
||||
"""Call-counting provider passed to ``SSEEventDispatcher`` via DI.
|
||||
|
||||
Returns a non-empty list by default so the cache stores a value — empty
|
||||
results are intentionally not cached (see ``test_empty_result_not_cached``).
|
||||
"""
|
||||
calls: dict[str, Any] = {"count": 0, "returns": [MagicMock(name="queue")]}
|
||||
|
||||
def _provider():
|
||||
calls["count"] += 1
|
||||
return calls["returns"]
|
||||
|
||||
calls["provider"] = _provider
|
||||
return calls
|
||||
|
||||
|
||||
class FakeClock:
|
||||
"""Controllable time source passed to ``SSEEventDispatcher``.
|
||||
|
||||
Lets tests advance the dispatcher's TTL cache deterministically without
|
||||
reaching into ``cachetools`` internals.
|
||||
"""
|
||||
|
||||
def __init__(self, start: float = 0.0) -> None:
|
||||
self.now = start
|
||||
|
||||
def __call__(self) -> float:
|
||||
return self.now
|
||||
|
||||
def advance(self, seconds: float) -> None:
|
||||
self.now += seconds
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def clock() -> FakeClock:
|
||||
return FakeClock()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def dispatcher(clock, fake_declare):
|
||||
"""Build a dispatcher with a stub queue_worker, a counting queues_provider,
|
||||
and a no-op ``ControlTask`` factory — no monkey-patching required.
|
||||
"""
|
||||
queue_worker = MagicMock(name="queue_worker")
|
||||
application_stack = MagicMock(name="application_stack")
|
||||
|
||||
class NoopControlTask:
|
||||
def __init__(self, qw):
|
||||
self.qw = qw
|
||||
|
||||
def send_task(self, **kwargs):
|
||||
pass
|
||||
|
||||
return SSEEventDispatcher(
|
||||
queue_worker=queue_worker,
|
||||
application_stack=application_stack,
|
||||
clock=clock,
|
||||
control_task_factory=NoopControlTask,
|
||||
queues_provider=fake_declare["provider"],
|
||||
)
|
||||
|
||||
|
||||
def test_declare_queues_cached_within_ttl(dispatcher, fake_declare):
|
||||
"""Repeated dispatches inside the TTL window only hit the DB once."""
|
||||
for _ in range(10):
|
||||
dispatcher.notify_broadcast("payload")
|
||||
assert fake_declare["count"] == 1
|
||||
|
||||
|
||||
def test_declare_queues_refetched_after_ttl(dispatcher, fake_declare, clock):
|
||||
"""Once the TTL expires, the next call refetches exactly once."""
|
||||
dispatcher.notify_broadcast("payload")
|
||||
assert fake_declare["count"] == 1
|
||||
|
||||
# Advance the injected clock past the TTL so the cache sees the entry as
|
||||
# expired on the next read.
|
||||
clock.advance(dispatcher._DECLARE_QUEUES_TTL_SECONDS + 1)
|
||||
|
||||
dispatcher.notify_broadcast("payload")
|
||||
assert fake_declare["count"] == 2
|
||||
|
||||
# Further dispatches at the advanced time reuse the newly populated entry.
|
||||
dispatcher.notify_broadcast("payload")
|
||||
assert fake_declare["count"] == 2
|
||||
|
||||
|
||||
def test_empty_result_not_cached(dispatcher, fake_declare):
|
||||
"""An empty list must not be pinned in the cache for the full TTL.
|
||||
|
||||
Empty results arise during startup (before ``DatabaseHeartbeat`` writes the
|
||||
row) and on swallowed DB errors. Caching them would silently drop every SSE
|
||||
event until the next TTL expiry.
|
||||
"""
|
||||
fake_declare["returns"] = []
|
||||
dispatcher.notify_broadcast("payload")
|
||||
dispatcher.notify_broadcast("payload")
|
||||
assert fake_declare["count"] == 2
|
||||
|
||||
# Once the upstream starts returning a non-empty result, caching resumes.
|
||||
fake_declare["returns"] = [MagicMock(name="queue")]
|
||||
dispatcher.notify_broadcast("payload")
|
||||
dispatcher.notify_broadcast("payload")
|
||||
assert fake_declare["count"] == 3
|
||||
|
||||
|
||||
def test_declare_queues_thread_safe_single_query_under_load(dispatcher, fake_declare):
|
||||
"""Concurrent ``_send`` from many threads still only triggers one DB query.
|
||||
|
||||
With stampede protection (RLock around the miss) all 500 dispatches should
|
||||
collapse to a single ``all_control_queues_for_declare`` call inside one TTL
|
||||
window. The assertion is exact (== 1), not loose, because the lock
|
||||
serializes the miss.
|
||||
"""
|
||||
iterations = 500
|
||||
|
||||
def work():
|
||||
dispatcher.notify_broadcast("payload")
|
||||
|
||||
with ThreadPoolExecutor(max_workers=16) as pool:
|
||||
futures = [pool.submit(work) for _ in range(iterations)]
|
||||
for future in as_completed(futures):
|
||||
future.result()
|
||||
|
||||
assert fake_declare["count"] == 1
|
||||
@@ -1,6 +1,13 @@
|
||||
import datetime
|
||||
import time
|
||||
from dataclasses import (
|
||||
dataclass,
|
||||
field,
|
||||
)
|
||||
from math import inf
|
||||
from types import SimpleNamespace
|
||||
from typing import Optional
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
@@ -14,6 +21,27 @@ from galaxy.queues import connection_from_config
|
||||
from galaxy.web_stack import application_stack_instance
|
||||
|
||||
|
||||
@dataclass
|
||||
class FakeStatsdClient:
|
||||
"""In-memory statsd recorder — see docstring in ``test_sse_dispatch.py``."""
|
||||
|
||||
counters: dict[tuple[str, tuple[tuple[str, str], ...]], int] = field(default_factory=dict)
|
||||
timings: list[tuple[str, float, tuple[tuple[str, str], ...]]] = field(default_factory=list)
|
||||
|
||||
def incr(self, metric: str, tags: Optional[dict[str, str]] = None) -> None:
|
||||
key = (metric, tuple(sorted((tags or {}).items())))
|
||||
self.counters[key] = self.counters.get(key, 0) + 1
|
||||
|
||||
def timing(self, metric: str, value: float, tags: Optional[dict[str, str]] = None) -> None:
|
||||
self.timings.append((metric, value, tuple(sorted((tags or {}).items()))))
|
||||
|
||||
def counter(self, metric: str, tags: Optional[dict[str, str]] = None) -> int:
|
||||
return self.counters.get((metric, tuple(sorted((tags or {}).items()))), 0)
|
||||
|
||||
def timings_for(self, metric: str) -> list[tuple[float, dict[str, str]]]:
|
||||
return [(v, dict(t)) for m, v, t in self.timings if m == metric]
|
||||
|
||||
|
||||
def bar(app, **kwargs):
|
||||
app.some_var = "bar"
|
||||
app.tasks_executed.append("echo")
|
||||
@@ -119,3 +147,86 @@ def wait_for_var(obj, var, value, tries=10, sleep=0.25):
|
||||
tries -= 1
|
||||
time.sleep(sleep)
|
||||
assert getattr(obj, var) == value
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# process_task observability tests
|
||||
#
|
||||
# These don't need a real broker or DB — we just instantiate GalaxyQueueWorker
|
||||
# via ``__new__`` and drive ``process_task`` directly with fake ``body`` /
|
||||
# ``message`` objects. The statsd client is pulled through
|
||||
# ``app.execution_timer_factory.galaxy_statsd_client``.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _make_fake_worker(task_fn, statsd_client):
|
||||
worker = GalaxyQueueWorker.__new__(GalaxyQueueWorker)
|
||||
worker.app = SimpleNamespace(
|
||||
config=SimpleNamespace(server_name="test.server"),
|
||||
execution_timer_factory=SimpleNamespace(galaxy_statsd_client=statsd_client),
|
||||
)
|
||||
worker.task_mapping = {"echo": task_fn}
|
||||
worker.epoch = 0
|
||||
# ``producer`` is a read-only property on the mixin; we avoid the publisher
|
||||
# path entirely by leaving ``reply_to`` out of the fake message properties.
|
||||
return worker
|
||||
|
||||
|
||||
def _fake_message():
|
||||
message = MagicMock()
|
||||
message.headers = {"epoch": inf} # always greater than worker.epoch
|
||||
message.properties = {} # no reply_to — skip publisher path
|
||||
return message
|
||||
|
||||
|
||||
def test_process_task_emits_counter_and_ok_timer():
|
||||
statsd = FakeStatsdClient()
|
||||
calls: list[dict] = []
|
||||
|
||||
def handler(app, **kwargs):
|
||||
calls.append(kwargs)
|
||||
return "done"
|
||||
|
||||
worker = _make_fake_worker(handler, statsd)
|
||||
worker.process_task({"task": "echo", "kwargs": {"x": 1}}, _fake_message())
|
||||
|
||||
# Handler actually ran — if the worker silently dropped the task, this list
|
||||
# would be empty and the test would fail loudly.
|
||||
assert calls == [{"x": 1}]
|
||||
assert statsd.counter("galaxy.control_queue.task.count", {"task": "echo"}) == 1
|
||||
assert len(statsd.timings_for("galaxy.control_queue.task.latency_ms")) == 1
|
||||
_value, tags = statsd.timings_for("galaxy.control_queue.task.latency_ms")[0]
|
||||
assert tags == {"task": "echo", "outcome": "ok"}
|
||||
|
||||
|
||||
def test_process_task_emits_error_timer_on_handler_exception():
|
||||
statsd = FakeStatsdClient()
|
||||
invocations: list[bool] = []
|
||||
|
||||
def boom(app, **kwargs):
|
||||
invocations.append(True)
|
||||
raise RuntimeError("handler failed")
|
||||
|
||||
worker = _make_fake_worker(boom, statsd)
|
||||
# process_task swallows handler exceptions (logged, not raised).
|
||||
worker.process_task({"task": "echo", "kwargs": {}}, _fake_message())
|
||||
|
||||
# Handler was actually invoked before raising.
|
||||
assert invocations == [True]
|
||||
assert statsd.counter("galaxy.control_queue.task.count", {"task": "echo"}) == 1
|
||||
assert len(statsd.timings_for("galaxy.control_queue.task.latency_ms")) == 1
|
||||
_value, tags = statsd.timings_for("galaxy.control_queue.task.latency_ms")[0]
|
||||
assert tags == {"task": "echo", "outcome": "error"}
|
||||
|
||||
|
||||
def test_process_task_no_statsd_is_silent_no_op():
|
||||
calls: list[dict] = []
|
||||
|
||||
def handler(app, **kwargs):
|
||||
calls.append(kwargs)
|
||||
return "done"
|
||||
|
||||
worker = _make_fake_worker(handler, statsd_client=None)
|
||||
worker.process_task({"task": "echo", "kwargs": {"x": 2}}, _fake_message())
|
||||
# Handler ran — the test guards both "no exception" AND "task not silently dropped".
|
||||
assert calls == [{"x": 2}]
|
||||
|
||||
@@ -9,10 +9,7 @@ from datetime import (
|
||||
)
|
||||
from types import SimpleNamespace
|
||||
from typing import Optional
|
||||
from unittest.mock import (
|
||||
MagicMock,
|
||||
patch,
|
||||
)
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import jwt
|
||||
import pytest
|
||||
@@ -400,15 +397,16 @@ def test_oidc_config_custom_auth_pipeline_and_extra(mock_oidc_config_file, mock_
|
||||
def test_sync_user_profile_skips_when_account_interface_enabled():
|
||||
manager = MagicMock()
|
||||
session = MagicMock()
|
||||
notify = MagicMock()
|
||||
app_config = SimpleNamespace(enable_account_interface=True, enable_notification_system=True)
|
||||
app = SimpleNamespace(config=app_config, user_manager=manager, notification_manager=SimpleNamespace())
|
||||
notification_manager = SimpleNamespace(send_notification_internal=notify)
|
||||
app = SimpleNamespace(config=app_config, user_manager=manager, notification_manager=notification_manager)
|
||||
trans = SimpleNamespace(app=app, sa_session=session)
|
||||
strategy = SimpleNamespace(config={"GALAXY_TRANS": trans, "FIXED_DELEGATED_AUTH": True})
|
||||
user = SimpleNamespace(id=1, preferences={})
|
||||
details = {"email": "new@example.com", "username": "newname"}
|
||||
|
||||
with patch("galaxy.webapps.galaxy.services.notifications.NotificationService.send_notification_internal") as notify:
|
||||
sync_user_profile(strategy=strategy, details=details, user=user)
|
||||
sync_user_profile(strategy=strategy, details=details, user=user)
|
||||
|
||||
manager.update_email.assert_not_called()
|
||||
manager.update_username.assert_not_called()
|
||||
@@ -419,15 +417,16 @@ def test_sync_user_profile_skips_when_account_interface_enabled():
|
||||
def test_sync_user_profile_skips_when_fixed_delegated_auth_disabled():
|
||||
manager = MagicMock()
|
||||
session = MagicMock()
|
||||
notify = MagicMock()
|
||||
app_config = SimpleNamespace(enable_account_interface=False, enable_notification_system=True)
|
||||
app = SimpleNamespace(config=app_config, user_manager=manager, notification_manager=SimpleNamespace())
|
||||
notification_manager = SimpleNamespace(send_notification_internal=notify)
|
||||
app = SimpleNamespace(config=app_config, user_manager=manager, notification_manager=notification_manager)
|
||||
trans = SimpleNamespace(app=app, sa_session=session)
|
||||
strategy = SimpleNamespace(config={"GALAXY_TRANS": trans, "FIXED_DELEGATED_AUTH": False})
|
||||
user = SimpleNamespace(id=2, email="old@example.com", username="oldname", preferences={})
|
||||
details = {"email": "new@example.com", "username": "newname"}
|
||||
|
||||
with patch("galaxy.webapps.galaxy.services.notifications.NotificationService.send_notification_internal") as notify:
|
||||
sync_user_profile(strategy=strategy, details=details, user=user)
|
||||
sync_user_profile(strategy=strategy, details=details, user=user)
|
||||
|
||||
manager.update_email.assert_not_called()
|
||||
manager.update_username.assert_not_called()
|
||||
@@ -438,16 +437,16 @@ def test_sync_user_profile_skips_when_fixed_delegated_auth_disabled():
|
||||
def test_sync_user_profile_updates_when_account_interface_disabled():
|
||||
manager = MagicMock()
|
||||
session = MagicMock()
|
||||
notify = MagicMock()
|
||||
app_config = SimpleNamespace(enable_account_interface=False, enable_notification_system=True)
|
||||
notification_manager = SimpleNamespace(notifications_enabled=True)
|
||||
notification_manager = SimpleNamespace(notifications_enabled=True, send_notification_internal=notify)
|
||||
app = SimpleNamespace(config=app_config, user_manager=manager, notification_manager=notification_manager)
|
||||
trans = SimpleNamespace(app=app, sa_session=session)
|
||||
strategy = SimpleNamespace(config={"GALAXY_TRANS": trans, "FIXED_DELEGATED_AUTH": True})
|
||||
user = SimpleNamespace(id=2, email="old@example.com", username="oldname", preferences={})
|
||||
details = {"email": "new@example.com", "username": "newname"}
|
||||
|
||||
with patch("galaxy.webapps.galaxy.services.notifications.NotificationService.send_notification_internal") as notify:
|
||||
sync_user_profile(strategy=strategy, details=details, user=user)
|
||||
sync_user_profile(strategy=strategy, details=details, user=user)
|
||||
|
||||
manager.update_email.assert_called_once_with(
|
||||
trans, user, "new@example.com", commit=False, send_activation_email=False
|
||||
|
||||
Reference in New Issue
Block a user