Merge pull request #21920 from dannon/fix/keyed-cache-retry-handling

Fix infinite request loops in cached stores with retry-aware error handling

client/src/api/datasetCollections.ts

@@ -9,7 +9,7 @@ import {
     isHDCA,
 } from "@/api";
 import type { components } from "@/api/schema";
-import { rethrowSimple } from "@/utils/simple-error";
+import { ApiError, errorMessageAsString, type GalaxyApiResult, rethrowSimple } from "@/utils/simple-error";
 
 const DEFAULT_LIMIT = 50;
 
@@ -27,15 +27,15 @@ export type SampleSheetColumnValueT = string | number | boolean;
  * Fetches the details of a collection.
  * @param params.id The ID of the collection (HDCA) to fetch.
  */
-export async function fetchCollectionDetails(params: { hdca_id: string }): Promise<HDCADetailed> {
-    const { data, error } = await GalaxyApi().GET("/api/dataset_collections/{hdca_id}", {
+export async function fetchCollectionDetails(params: { hdca_id: string }): Promise<GalaxyApiResult<HDCADetailed>> {
+    const { data, error, response } = await GalaxyApi().GET("/api/dataset_collections/{hdca_id}", {
         params: { path: params },
     });
 
     if (error) {
-        rethrowSimple(error);
+        return { data: undefined, error: new ApiError(errorMessageAsString(error), response.status) };
     }
-    return data as HDCADetailed;
+    return { data: data as HDCADetailed, error: undefined };
 }
 
 /**

client/src/api/datasets.ts

@@ -55,7 +55,7 @@ export async function loadDatasets(options: LoadDatasetsOptions): Promise<LoadDa
 }
 
 export async function fetchDatasetTextContentDetails(params: { id: string }): Promise<DatasetTextContentDetails> {
-    const { data, error } = await GalaxyApi().GET("/api/datasets/{dataset_id}/get_content_as_text", {
+    const { data, error, response } = await GalaxyApi().GET("/api/datasets/{dataset_id}/get_content_as_text", {
         params: {
             path: {
                 dataset_id: params.id,
@@ -64,7 +64,7 @@ export async function fetchDatasetTextContentDetails(params: { id: string }): Pr
     });
 
     if (error) {
-        rethrowSimple(error);
+        rethrowSimpleWithStatus(error, response);
     }
     return data;
 }

client/src/components/Collections/SampleSheetWizard.vue

@@ -138,10 +138,11 @@ watch(targetCollectionId, async () => {
     if (targetCollectionId.value) {
         fetchingCollection.value = true;
         try {
-            // TODO: spinner while loading
-            const details = await fetchCollectionDetails({ hdca_id: targetCollectionId.value });
-            targetCollection.value = details;
-            // Nothing else to do on the page, just skip to the next step.
+            const result = await fetchCollectionDetails({ hdca_id: targetCollectionId.value });
+            if (result.error) {
+                throw result.error;
+            }
+            targetCollection.value = result.data;
             nextTick(() => {
                 wizard.goTo("fill-grid");
             });

client/src/components/Form/Elements/FormRulesEdit.vue

@@ -39,8 +39,11 @@ async function onEdit() {
         try {
             loading.value = true;
             loadError.value = undefined;
-            const collectionDetails = await fetchCollectionDetails({ hdca_id: props.target.id });
-            elements.value = collectionDetails;
+            const result = await fetchCollectionDetails({ hdca_id: props.target.id });
+            if (result.error) {
+                throw result.error;
+            }
+            elements.value = result.data;
             modal.value.show();
         } catch (e) {
             loadError.value = errorMessageAsString(e);

client/src/components/History/Export/HistoryExport.vue

@@ -94,9 +94,16 @@ watch(isExportTaskRunning, (newValue, oldValue) => {
 async function loadHistory() {
     isLoadingHistory.value = true;
     try {
-        history.value =
-            historyStore.getHistoryById(props.historyId, false) ??
-            (await historyStore.loadHistoryById(props.historyId));
+        if (!historyStore.getHistoryById(props.historyId, false)) {
+            await historyStore.loadHistoryById(props.historyId);
+        }
+        history.value = historyStore.getHistoryById(props.historyId, false) ?? undefined;
+        if (!history.value) {
+            const loadError = historyStore.getHistoryLoadError(props.historyId);
+            if (loadError) {
+                throw loadError;
+            }
+        }
         return true;
     } catch (error) {
         errorMessage.value = errorMessageAsString(error);

client/src/components/History/HistoryView.test.js

@@ -72,7 +72,7 @@ function create_datasets(historyId, count) {
 
 async function createWrapper(localVue, currentUserId, history) {
     const pinia = createPinia();
-    getHistoryByIdFromServer.mockResolvedValue(history);
+    getHistoryByIdFromServer.mockResolvedValue({ data: history, error: undefined });
     setCurrentHistoryOnServer.mockResolvedValue(history);
     const history_contents_result = create_datasets(history.id, history.count);
 

client/src/components/providers/DatasetCollectionProvider.js

@@ -4,11 +4,15 @@ import { SingleQueryProvider } from "@/components/providers/SingleQueryProvider"
 // There isn't really a good way to know when to stop polling for HDCA updates,
 // but we know the populated_state should at least be ok.
 export default SingleQueryProvider(
-    (params) => {
+    async (params) => {
         if (params.view && params.view === "collection") {
             return fetchCollectionSummary({ hdca_id: params.id });
         }
-        return fetchCollectionDetails({ hdca_id: params.id });
+        const result = await fetchCollectionDetails({ hdca_id: params.id });
+        if (result.error) {
+            throw result.error;
+        }
+        return result.data;
     },
     (result) => result.populated_state === "ok",
 );

client/src/composables/keyedCache.test.ts

@@ -223,8 +223,8 @@ describe("useKeyedCache", () => {
 
         const { getItemById, getItemLoadError } = useKeyedCache<ItemData>(fetchItem);
 
-        // First attempt + MAX_RETRIES - 1 retries (first call counts as attempt 1)
-        for (let i = 1; i <= 3; i++) {
+        // Initial fetch + MAX_RETRIES retries = 4 total calls
+        for (let i = 1; i <= 4; i++) {
             getItemById.value(id);
             await flushPromises();
             expect(fetchItem).toHaveBeenCalledTimes(i);
@@ -234,7 +234,7 @@ describe("useKeyedCache", () => {
         // Should stop after max retries exhausted
         getItemById.value(id);
         await flushPromises();
-        expect(fetchItem).toHaveBeenCalledTimes(3);
+        expect(fetchItem).toHaveBeenCalledTimes(4);
     });
 
     it("should not retry on permanent errors (403, 404)", async () => {
@@ -292,4 +292,22 @@ describe("useKeyedCache", () => {
         await flushPromises();
         expect(true).toBe(true);
     });
+
+    it("should clear error on successful recovery after transient failure", async () => {
+        const id = "1";
+        const item = { id, name: "Item 1" };
+        fetchItem.mockRejectedValueOnce(new ApiError("service unavailable", 503));
+        fetchItem.mockResolvedValue(item);
+
+        const { getItemById, storedItems, getItemLoadError } = useKeyedCache<ItemData>(fetchItem);
+
+        getItemById.value(id);
+        await flushPromises();
+        expect(getItemLoadError.value(id)).toBeTruthy();
+
+        getItemById.value(id);
+        await flushPromises();
+        expect(storedItems.value[id]).toEqual(item);
+        expect(getItemLoadError.value(id)).toBeNull();
+    });
 });

client/src/composables/keyedCache.ts

@@ -2,7 +2,7 @@ import { type MaybeRefOrGetter, toValue } from "@vueuse/core";
 import { computed, del, type Ref, ref, set, unref } from "vue";
 
 import { LastQueue } from "@/utils/lastQueue";
-import { ApiError } from "@/utils/simple-error";
+import { isRetryableApiError, MAX_RETRIES } from "@/utils/simple-error";
 
 /**
  * Parameters for fetching an item from the server.
@@ -30,16 +30,6 @@ type ShouldFetchHandler<T> = (item?: T) => boolean;
  */
 const fetchIfAbsent = <T>(item?: T) => item === undefined;
 
-const RETRYABLE_STATUSES = new Set([429, 500, 502, 503, 504]);
-const MAX_RETRIES = 3;
-
-function isRetryableError(error: Error): boolean {
-    if (error instanceof ApiError && error.status !== undefined) {
-        return RETRYABLE_STATUSES.has(error.status);
-    }
-    return false;
-}
-
 /**
  * A composable that provides a simple key-value cache for items fetched from the server.
  *
@@ -56,17 +46,19 @@ export function useKeyedCache<T>(
 ) {
     const storedItems = ref<{ [key: string]: T }>({});
     const loadingErrors = ref<{ [key: string]: Error }>({});
-    const retryCounts: { [key: string]: number } = {};
 
     const loadingRequests = ref<{ [key: string]: Promise<T | undefined> }>({});
 
+    const retryCounts: { [key: string]: number } = {};
+
     const fetchQueue = new LastQueue<FetchHandler<T>>();
 
     const getItemById = computed(() => {
         return (id: string) => {
             const item = storedItems.value[id];
             const existingError = loadingErrors.value[id];
-            const canRetry = existingError && isRetryableError(existingError) && (retryCounts[id] ?? 0) < MAX_RETRIES;
+            const canRetry =
+                existingError && isRetryableApiError(existingError) && (retryCounts[id] ?? 0) <= MAX_RETRIES;
             if (shouldFetch(item) && (!existingError || canRetry)) {
                 fetchItemById({ id: id });
             }
@@ -105,6 +97,7 @@ export function useKeyedCache<T>(
                 const fetchItem = unref(fetchItemHandler);
                 const item = await fetchQueue.enqueue(fetchItem, { id: itemId }, itemId);
                 set(storedItems.value, itemId, item);
+                del(loadingErrors.value, itemId);
                 delete retryCounts[itemId];
                 return item;
             } catch (error) {

client/src/composables/uploadQueue.ts

@@ -389,7 +389,8 @@ export function useUploadQueue() {
     async function validateTargetHistory(targetHistoryId: string): Promise<string | null> {
         let history = historyStore.getHistoryById(targetHistoryId, false) ?? null;
         if (!history) {
-            history = (await historyStore.loadHistoryById(targetHistoryId)) ?? null;
+            await historyStore.loadHistoryById(targetHistoryId);
+            history = historyStore.getHistoryById(targetHistoryId, false) ?? null;
         }
 
         // If history still cannot be resolved, treat as no validation error here

client/src/stores/collectionAttributesStore.ts

@@ -2,15 +2,15 @@ import { defineStore } from "pinia";
 
 import { type DatasetCollectionAttributes, GalaxyApi } from "@/api";
 import { type FetchParams, useKeyedCache } from "@/composables/keyedCache";
-import { rethrowSimple } from "@/utils/simple-error";
+import { rethrowSimpleWithStatus } from "@/utils/simple-error";
 
 export const useCollectionAttributesStore = defineStore("collectionAttributesStore", () => {
     async function fetchAttributes(params: FetchParams): Promise<DatasetCollectionAttributes> {
-        const { data, error } = await GalaxyApi().GET("/api/dataset_collections/{hdca_id}/attributes", {
+        const { data, error, response } = await GalaxyApi().GET("/api/dataset_collections/{hdca_id}/attributes", {
             params: { path: { hdca_id: params.id } },
         });
         if (error) {
-            rethrowSimple(error);
+            rethrowSimpleWithStatus(error, response);
         }
         return data;
     }

client/src/stores/collectionElementsStore.ts

@@ -12,6 +12,7 @@ import {
 import { fetchCollectionDetails, fetchElementsFromCollection } from "@/api/datasetCollections";
 import { ensureDefined } from "@/utils/assertions";
 import { ActionSkippedError, LastQueue } from "@/utils/lastQueue";
+import { isRetryableApiError, MAX_RETRIES } from "@/utils/simple-error";
 
 /**
  * Represents an element in a collection that has not been fetched yet.
@@ -46,6 +47,7 @@ export const useCollectionElementsStore = defineStore("collectionElementsStore",
     const loadingCollectionElements = ref<{ [key: string]: boolean }>({});
     const loadingCollectionElementsErrors = ref<{ [key: string]: Error }>({});
     const storedCollectionElements = ref<{ [key: string]: DCEEntry[] }>({});
+    const retryCounts: { [key: string]: number } = {};
 
     /**
      * Returns a key that can be used to store or retrieve the elements of a collection in the store.
@@ -185,9 +187,15 @@ export const useCollectionElementsStore = defineStore("collectionElementsStore",
     /** Returns collection from storedCollections, will load collection if not in store */
     const getCollectionById = computed(() => {
         return (collectionId: string) => {
-            if (!storedCollections.value[collectionId] && !loadingCollectionElementsErrors.value[collectionId]) {
-                // TODO: Try to remove this as it can cause computed side effects (use keyedCache in this store instead?)
-                fetchCollection({ id: collectionId });
+            if (!storedCollections.value[collectionId]) {
+                const existingError = loadingCollectionElementsErrors.value[collectionId];
+                const canRetry =
+                    existingError &&
+                    isRetryableApiError(existingError) &&
+                    (retryCounts[collectionId] ?? 0) <= MAX_RETRIES;
+                if (!existingError || canRetry) {
+                    fetchCollection({ id: collectionId });
+                }
             }
             return storedCollections.value[collectionId] ?? null;
         };
@@ -195,12 +203,15 @@ export const useCollectionElementsStore = defineStore("collectionElementsStore",
 
     const getDetailedCollectionById = computed(() => {
         return (collectionId: string) => {
-            if (
-                !storedCollectionsDetailed.value[collectionId] &&
-                !loadingCollectionElementsErrors.value[collectionId]
-            ) {
-                // TODO: Try to remove this as it can cause computed side effects (use keyedCache in this store instead?)
-                fetchCollection({ id: collectionId });
+            if (!storedCollectionsDetailed.value[collectionId]) {
+                const existingError = loadingCollectionElementsErrors.value[collectionId];
+                const canRetry =
+                    existingError &&
+                    isRetryableApiError(existingError) &&
+                    (retryCounts[collectionId] ?? 0) <= MAX_RETRIES;
+                if (!existingError || canRetry) {
+                    fetchCollection({ id: collectionId });
+                }
             }
             return storedCollectionsDetailed.value[collectionId] ?? null;
         };
@@ -209,11 +220,15 @@ export const useCollectionElementsStore = defineStore("collectionElementsStore",
     async function fetchCollection(params: { id: string }) {
         set(loadingCollectionElements.value, params.id, true);
         try {
-            const collection = await fetchCollectionDetails({ hdca_id: params.id });
-            saveCollection(collection);
-            return collection;
-        } catch (error) {
-            set(loadingCollectionElementsErrors.value, params.id, error);
+            const result = await fetchCollectionDetails({ hdca_id: params.id });
+            if (result.error) {
+                retryCounts[params.id] = (retryCounts[params.id] ?? 0) + 1;
+                set(loadingCollectionElementsErrors.value, params.id, result.error);
+            } else {
+                saveCollection(result.data);
+                del(loadingCollectionElementsErrors.value, params.id);
+                delete retryCounts[params.id];
+            }
         } finally {
             del(loadingCollectionElements.value, params.id);
         }

client/src/stores/datasetExtraFilesStore.ts

@@ -3,15 +3,15 @@ import { defineStore } from "pinia";
 import { GalaxyApi } from "@/api";
 import type { DatasetExtraFiles } from "@/api/datasets";
 import { type FetchParams, useKeyedCache } from "@/composables/keyedCache";
-import { rethrowSimple } from "@/utils/simple-error";
+import { rethrowSimpleWithStatus } from "@/utils/simple-error";
 
 export const useDatasetExtraFilesStore = defineStore("datasetExtraFilesStore", () => {
     async function fetchDatasetExtraFiles(params: FetchParams): Promise<DatasetExtraFiles> {
-        const { data, error } = await GalaxyApi().GET("/api/datasets/{dataset_id}/extra_files", {
+        const { data, error, response } = await GalaxyApi().GET("/api/datasets/{dataset_id}/extra_files", {
             params: { path: { dataset_id: params.id } },
         });
         if (error) {
-            rethrowSimple(error);
+            rethrowSimpleWithStatus(error, response);
         }
         return data;
     }

client/src/stores/historyStore.ts

@@ -25,7 +25,7 @@ import {
     setCurrentHistoryOnServer,
     updateHistoryFields,
 } from "@/stores/services/history.services";
-import { rethrowSimple } from "@/utils/simple-error";
+import { isRetryableApiError, MAX_RETRIES, rethrowSimple } from "@/utils/simple-error";
 import { sortByObjectProp } from "@/utils/sorting";
 import {
     ACTIVE_POLLING_INTERVAL,
@@ -34,7 +34,8 @@ import {
 } from "@/watch/watchHistory";
 
 const PAGINATION_LIMIT = 10;
-const isLoadingHistory = new Set();
+const isLoadingHistory = new Set<string>();
+const retryCounts: { [key: string]: number } = {};
 const CONTENT_STATS_KEYS = ["size", "contents_active", "update_time"] as const;
 
 export const useHistoryStore = defineStore("historyStore", () => {
@@ -45,6 +46,7 @@ export const useHistoryStore = defineStore("historyStore", () => {
     const storedCurrentHistoryId = ref<string | null>(null);
     const storedFilterTexts = ref<{ [key: string]: string }>({});
     const storedHistories = ref<{ [key: string]: AnyHistory }>({});
+    const historyLoadErrors = ref<{ [key: string]: Error }>({});
     const changingCurrentHistory = ref(false);
 
     const histories = computed(() => {
@@ -80,14 +82,24 @@ export const useHistoryStore = defineStore("historyStore", () => {
         }
     });
 
+    const getHistoryLoadError = computed(() => {
+        return (historyId: string) => {
+            return historyLoadErrors.value[historyId] ?? null;
+        };
+    });
+
     /** Returns history from storedHistories, will load history if not in store by default.
      * If shouldFetchIfMissing is false, will return null if history is not in store.
      */
     const getHistoryById = computed(() => {
         return (historyId: string, shouldFetchIfMissing = true) => {
             if (!storedHistories.value[historyId] && shouldFetchIfMissing) {
-                // TODO: Try to remove this as it can cause computed side effects
-                loadHistoryById(historyId);
+                const existingError = historyLoadErrors.value[historyId];
+                const canRetry =
+                    existingError && isRetryableApiError(existingError) && (retryCounts[historyId] ?? 0) <= MAX_RETRIES;
+                if (!existingError || canRetry) {
+                    loadHistoryById(historyId);
+                }
             }
             return storedHistories.value[historyId] ?? null;
         };
@@ -387,15 +399,19 @@ export const useHistoryStore = defineStore("historyStore", () => {
         },
     );
 
-    async function loadHistoryById(historyId: string): Promise<HistorySummaryExtended | undefined> {
+    async function loadHistoryById(historyId: string) {
         if (!isLoadingHistory.has(historyId)) {
             isLoadingHistory.add(historyId);
             try {
-                const history = (await getHistoryByIdFromServer(historyId)) as HistorySummaryExtended;
-                setHistory(history);
-                return history;
-            } catch (error) {
-                rethrowSimple(error);
+                const result = await getHistoryByIdFromServer(historyId);
+                if (result.error) {
+                    retryCounts[historyId] = (retryCounts[historyId] ?? 0) + 1;
+                    set(historyLoadErrors.value, historyId, result.error);
+                } else {
+                    setHistory(result.data);
+                    del(historyLoadErrors.value, historyId);
+                    delete retryCounts[historyId];
+                }
             } finally {
                 isLoadingHistory.delete(historyId);
             }
@@ -489,6 +505,7 @@ export const useHistoryStore = defineStore("historyStore", () => {
         pinnedHistories,
         storedHistories,
         getHistoryById,
+        getHistoryLoadError,
         getHistoryNameById,
         setCurrentHistory,
         setCurrentHistoryId,

client/src/stores/invocationStore.ts

@@ -10,53 +10,53 @@ import type {
     WorkflowInvocationRequest,
 } from "@/api/invocations";
 import { type FetchParams, useKeyedCache } from "@/composables/keyedCache";
-import { rethrowSimple } from "@/utils/simple-error";
+import { rethrowSimple, rethrowSimpleWithStatus } from "@/utils/simple-error";
 
 export const useInvocationStore = defineStore("invocationStore", () => {
     const scrollListScrollTop = ref(0);
 
     async function fetchInvocationDetails(params: FetchParams): Promise<WorkflowInvocation> {
-        const { data, error } = await GalaxyApi().GET("/api/invocations/{invocation_id}", {
+        const { data, error, response } = await GalaxyApi().GET("/api/invocations/{invocation_id}", {
             params: { path: { invocation_id: params.id } },
         });
         if (error) {
-            rethrowSimple(error);
+            rethrowSimpleWithStatus(error, response);
         }
         return data;
     }
 
     async function fetchInvocationJobsSummary(params: FetchParams): Promise<InvocationJobsSummary> {
-        const { data, error } = await GalaxyApi().GET("/api/invocations/{invocation_id}/jobs_summary", {
+        const { data, error, response } = await GalaxyApi().GET("/api/invocations/{invocation_id}/jobs_summary", {
             params: { path: { invocation_id: params.id } },
         });
         if (error) {
-            rethrowSimple(error);
+            rethrowSimpleWithStatus(error, response);
         }
         return data;
     }
 
     async function fetchInvocationStepJobsSummary(params: FetchParams): Promise<StepJobSummary[]> {
-        const { data, error } = await GalaxyApi().GET("/api/invocations/{invocation_id}/step_jobs_summary", {
+        const { data, error, response } = await GalaxyApi().GET("/api/invocations/{invocation_id}/step_jobs_summary", {
             params: { path: { invocation_id: params.id } },
         });
         if (error) {
-            rethrowSimple(error);
+            rethrowSimpleWithStatus(error, response);
         }
         return data;
     }
 
     async function fetchInvocationStep(params: FetchParams): Promise<InvocationStep> {
-        const { data, error } = await GalaxyApi().GET("/api/invocations/steps/{step_id}", {
+        const { data, error, response } = await GalaxyApi().GET("/api/invocations/steps/{step_id}", {
             params: { path: { step_id: params.id } },
         });
         if (error) {
-            rethrowSimple(error);
+            rethrowSimpleWithStatus(error, response);
         }
         return data;
     }
 
     async function fetchInvocationRequest(params: FetchParams): Promise<WorkflowInvocationRequest> {
-        const { data, error } = await GalaxyApi().GET("/api/invocations/{invocation_id}/request", {
+        const { data, error, response } = await GalaxyApi().GET("/api/invocations/{invocation_id}/request", {
             params: {
                 path: {
                     invocation_id: params.id,
@@ -64,17 +64,17 @@ export const useInvocationStore = defineStore("invocationStore", () => {
             },
         });
         if (error) {
-            rethrowSimple(error);
+            rethrowSimpleWithStatus(error, response);
         }
         return data;
     }
 
     async function fetchInvocationCount(params: FetchParams): Promise<number> {
-        const { data, error } = await GalaxyApi().GET("/api/workflows/{workflow_id}/counts", {
+        const { data, error, response } = await GalaxyApi().GET("/api/workflows/{workflow_id}/counts", {
             params: { path: { workflow_id: params.id } },
         });
         if (error) {
-            rethrowSimple(error);
+            rethrowSimpleWithStatus(error, response);
         }
 
         let allCounts = 0;

client/src/stores/jobDestinationParametersStore.ts

@@ -3,17 +3,17 @@ import { defineStore } from "pinia";
 import { GalaxyApi } from "@/api";
 import type { JobDestinationParams } from "@/api/jobs";
 import { type FetchParams, useKeyedCache } from "@/composables/keyedCache";
-import { rethrowSimple } from "@/utils/simple-error";
+import { rethrowSimpleWithStatus } from "@/utils/simple-error";
 
 export const useJobDestinationParametersStore = defineStore("jobDestinationParametersStore", () => {
     async function fetchJobDestinationParams(params: FetchParams): Promise<JobDestinationParams> {
-        const { data, error } = await GalaxyApi().GET("/api/jobs/{job_id}/destination_params", {
+        const { data, error, response } = await GalaxyApi().GET("/api/jobs/{job_id}/destination_params", {
             params: {
                 path: { job_id: params.id },
             },
         });
         if (error) {
-            rethrowSimple(error);
+            rethrowSimpleWithStatus(error, response);
         }
         return data;
     }

client/src/stores/jobStore.ts

@@ -9,18 +9,18 @@ import { ref } from "vue";
 import { GalaxyApi } from "@/api";
 import { type ResponseVal, type ShowFullJobResponse, TERMINAL_STATES } from "@/api/jobs";
 import { type FetchParams, useKeyedCache } from "@/composables/keyedCache";
-import { rethrowSimple } from "@/utils/simple-error";
+import { rethrowSimpleWithStatus } from "@/utils/simple-error";
 
 export const useJobStore = defineStore("jobStore", () => {
     const latestResponse = ref<ResponseVal | null>(null);
 
     async function fetchJobById(params: FetchParams): Promise<ShowFullJobResponse> {
-        const { data, error } = await GalaxyApi().GET("/api/jobs/{job_id}", {
+        const { data, error, response } = await GalaxyApi().GET("/api/jobs/{job_id}", {
             params: { path: { job_id: params.id } },
             query: { full: true },
         });
         if (error) {
-            rethrowSimple(error);
+            rethrowSimpleWithStatus(error, response);
         }
         return data;
     }

client/src/stores/services/history.services.ts

@@ -3,7 +3,7 @@ import axios, { type AxiosResponse } from "axios";
 import type { AnyHistory, HistorySummaryExtended } from "@/api";
 import { GalaxyApi } from "@/api";
 import { prependPath } from "@/utils/redirect";
-import { rethrowSimple } from "@/utils/simple-error";
+import { ApiError, errorMessageAsString, type GalaxyApiResult, rethrowSimple } from "@/utils/simple-error";
 
 /**
  * Generic json getter
@@ -97,8 +97,8 @@ export async function getHistoryList(offset = 0, limit: number | null = null, qu
 }
 
 /** Load one history by id */
-export async function getHistoryByIdFromServer(id: string) {
-    const { data, error } = await GalaxyApi().GET("/api/histories/{history_id}", {
+export async function getHistoryByIdFromServer(id: string): Promise<GalaxyApiResult<HistorySummaryExtended>> {
+    const { data, error, response } = await GalaxyApi().GET("/api/histories/{history_id}", {
         params: {
             path: { history_id: id },
             query: extendedHistoryParams,
@@ -106,12 +106,12 @@ export async function getHistoryByIdFromServer(id: string) {
     });
 
     if (error) {
-        rethrowSimple(error);
+        return { data: undefined, error: new ApiError(errorMessageAsString(error), response.status) };
     }
 
     // We know that the data is a HistorySummaryExtended because we requested it
     // with the extendedHistoryParams
-    return data as HistorySummaryExtended;
+    return { data: data as HistorySummaryExtended, error: undefined };
 }
 
 /**
@@ -127,9 +127,12 @@ export async function secureHistoryOnServer(history: AnyHistory) {
     if (response.status !== 200) {
         throw new Error(response.statusText);
     }
-    const securedHistory = await getHistoryByIdFromServer(id);
+    const result = await getHistoryByIdFromServer(id);
+    if (result.error) {
+        throw result.error;
+    }
     return {
-        securedHistory,
+        securedHistory: result.data,
         message: response.data.message as string,
         sharingStatusChanged: response.data.sharing_status_changed as boolean,
     };

client/src/utils/simple-error.ts

@@ -39,3 +39,15 @@ export function rethrowSimpleWithStatus(e: any, response?: { status: number }):
     }
     throw new ApiError(errorMessageAsString(e), response?.status);
 }
+
+export type GalaxyApiResult<T> = { data: T; error: undefined } | { data: undefined; error: ApiError };
+
+export const MAX_RETRIES = 3;
+const RETRYABLE_STATUSES = new Set([429, 500, 502, 503, 504]);
+
+export function isRetryableApiError(error: Error): boolean {
+    if (error instanceof ApiError && error.status !== undefined) {
+        return RETRYABLE_STATUSES.has(error.status);
+    }
+    return false;
+}