Merge pull request #22606 from davelopez/feature/bulk-storage-migration

Add bulk dataset storage migration
This commit is contained in:
Ahmed Hamid Awan
2026-05-20 19:15:12 -05:00
committed by GitHub
51 changed files with 8356 additions and 26 deletions
+72
View File
@@ -51,6 +51,11 @@ export type PublishedHistory = SharedHistory & {
*/
export type AnyHistoryEntry = MyHistory | SharedHistory | PublishedHistory | ArchivedHistorySummary;
/**
* Represents a reference to a history, containing only the fields necessary for identifying a history in API calls.
*/
export type HistoryReference = Pick<HistorySummary, "id" | "model_class">;
/**
* Represents the options for fetching histories.
*/
@@ -301,6 +306,73 @@ export async function getHistoryCounts(historyId: string): Promise<HistoryCounts
return data as HistoryCounts;
}
export type StorageOperationMode = components["schemas"]["StorageOperationMode"];
export type StorageOperationExecutePolicy = components["schemas"]["StorageOperationExecutePolicy"];
export type StorageOperationPreviewResponse = components["schemas"]["StorageOperationPreviewResponse"];
export type StorageOperationExecuteResponse = components["schemas"]["StorageOperationExecuteResponse"];
export type StorageOperationRunResponse = components["schemas"]["StorageOperationRunResponse"];
export type StorageOperationRunItemStatus = components["schemas"]["StorageOperationRunItemStatus"];
export interface StorageOperationRunItemsOptions {
offset?: number;
limit?: number;
search?: string;
}
export interface StorageOperationRunItemsWithTotal {
data: StorageOperationRunItemStatus[];
totalMatches: number | null;
}
export async function getStorageOperationRunStatus(
history: HistoryReference,
runId: string,
): Promise<StorageOperationRunResponse> {
const { data, error } = await GalaxyApi().GET("/api/histories/{history_id}/contents/bulk/storage/runs/{run_id}", {
params: {
path: { history_id: history.id, run_id: runId },
},
});
if (error) {
rethrowSimple(error);
}
return data;
}
export async function getStorageOperationRunItemsWithTotal(
history: HistoryReference,
runId: string,
options: StorageOperationRunItemsOptions = {},
): Promise<StorageOperationRunItemsWithTotal> {
const { offset = 0, limit = 50, search } = options;
const { response, data, error } = await GalaxyApi().GET(
"/api/histories/{history_id}/contents/bulk/storage/runs/{run_id}/items",
{
params: {
path: { history_id: history.id, run_id: runId },
query: {
offset,
limit,
search,
},
},
},
);
if (error) {
rethrowSimple(error);
}
const totalMatchesHeader = response.headers.get("total_matches");
const totalMatches = totalMatchesHeader !== null ? parseInt(totalMatchesHeader, 10) : null;
return {
data,
totalMatches: Number.isNaN(totalMatches) ? null : totalMatches,
};
}
/**
* Fetches the workflow extraction summary for a specific history entry.
* @param {string} historyId The ID of the history entry to fetch the workflow extraction summary for
+6 -1
View File
@@ -15,6 +15,11 @@ export interface SharedItemNotification extends BaseUserNotification {
content: components["schemas"]["NewSharedItemNotificationContent"];
}
export interface StorageOperationNotification extends BaseUserNotification {
category: "storage_operation";
content: components["schemas"]["StorageOperationNotificationContent"];
}
type NotificationCreateData = components["schemas"]["NotificationCreateData"];
export interface MessageNotificationCreateData extends NotificationCreateData {
@@ -28,7 +33,7 @@ export interface MessageNotificationCreateRequest extends NotificationCreateRequ
notification: MessageNotificationCreateData;
}
export type UserNotification = MessageNotification | SharedItemNotification;
export type UserNotification = MessageNotification | SharedItemNotification | StorageOperationNotification;
export type NotificationChanges = components["schemas"]["UserNotificationUpdateRequest"];
+594 -2
View File
@@ -2359,6 +2359,74 @@ export interface paths {
patch?: never;
trace?: never;
};
"/api/histories/{history_id}/contents/bulk/storage/execute": {
parameters: {
query?: never;
header?: never;
path?: never;
cookie?: never;
};
get?: never;
put?: never;
/** Executes a previously previewed storage bulk operation snapshot. */
post: operations["bulk_storage_operation_execute_api_histories__history_id__contents_bulk_storage_execute_post"];
delete?: never;
options?: never;
head?: never;
patch?: never;
trace?: never;
};
"/api/histories/{history_id}/contents/bulk/storage/preview": {
parameters: {
query?: never;
header?: never;
path?: never;
cookie?: never;
};
get?: never;
put?: never;
/** Previews a storage bulk operation for selected history contents. */
post: operations["bulk_storage_operation_preview_api_histories__history_id__contents_bulk_storage_preview_post"];
delete?: never;
options?: never;
head?: never;
patch?: never;
trace?: never;
};
"/api/histories/{history_id}/contents/bulk/storage/runs/{run_id}": {
parameters: {
query?: never;
header?: never;
path?: never;
cookie?: never;
};
/** Returns run status summary for a storage bulk operation. */
get: operations["bulk_storage_operation_run_api_histories__history_id__contents_bulk_storage_runs__run_id__get"];
put?: never;
post?: never;
delete?: never;
options?: never;
head?: never;
patch?: never;
trace?: never;
};
"/api/histories/{history_id}/contents/bulk/storage/runs/{run_id}/items": {
parameters: {
query?: never;
header?: never;
path?: never;
cookie?: never;
};
/** Returns paginated per-item details for a storage bulk operation run. */
get: operations["bulk_storage_operation_run_items_api_histories__history_id__contents_bulk_storage_runs__run_id__items_get"];
put?: never;
post?: never;
delete?: never;
options?: never;
head?: never;
patch?: never;
trace?: never;
};
"/api/histories/{history_id}/contents/dataset_collections/{hdca_id}/download": {
parameters: {
query?: never;
@@ -11309,6 +11377,22 @@ export interface components {
[key: string]: unknown;
}[];
};
/**
* DatasetStorageOperationFailureReasonCode
* @enum {string}
*/
DatasetStorageOperationFailureReasonCode:
| "dataset_not_found"
| "invalid_target_object_store"
| "missing_source_object_store"
| "already_in_target"
| "target_quota_exceeded"
| "shared_dataset"
| "insufficient_permissions"
| "dataset_in_use"
| "target_expiration_imminent"
| "checksum_verification_failed"
| "execution_error";
/** DatasetTextContentDetails */
DatasetTextContentDetails: {
/**
@@ -19327,6 +19411,7 @@ export interface components {
content:
| components["schemas"]["MessageNotificationContent"]
| components["schemas"]["NewSharedItemNotificationContent"]
| components["schemas"]["StorageOperationNotificationContent"]
| components["schemas"]["BroadcastNotificationContent"];
/**
* Expiration time
@@ -19415,6 +19500,7 @@ export interface components {
content:
| components["schemas"]["MessageNotificationContent"]
| components["schemas"]["NewSharedItemNotificationContent"]
| components["schemas"]["StorageOperationNotificationContent"]
| components["schemas"]["BroadcastNotificationContent"];
/**
* Create time
@@ -20502,7 +20588,7 @@ export interface components {
* displayed in the notification preferences.
* @enum {string}
*/
PersonalNotificationCategory: "message" | "new_shared_item";
PersonalNotificationCategory: "message" | "new_shared_item" | "storage_operation";
/** PluginAspectStatus */
PluginAspectStatus: {
/** Message */
@@ -22578,6 +22664,267 @@ export interface components {
/** Total Item Count */
total_item_count: number;
};
/** StorageOperationEligibilityReasonSummary */
StorageOperationEligibilityReasonSummary: {
/** Count */
count: number;
reason_code: components["schemas"]["DatasetStorageOperationFailureReasonCode"];
};
/** StorageOperationEligibilitySummary */
StorageOperationEligibilitySummary: {
/** Eligible Count */
eligible_count: number;
/** Ineligible Count */
ineligible_count: number;
/** Reasons */
reasons?: components["schemas"]["StorageOperationEligibilityReasonSummary"][];
};
/** StorageOperationEstimateSummary */
StorageOperationEstimateSummary: {
/**
* Bytes To Transfer
* @default 0
*/
bytes_to_transfer: number;
/** Quota Delta Transfers */
quota_delta_transfers?: components["schemas"]["StorageOperationQuotaDeltaTransfer"][];
quota_projection?: components["schemas"]["StorageOperationQuotaProjectionSummary"] | null;
};
/** StorageOperationExecutePolicy */
StorageOperationExecutePolicy: {
/** Max Retries */
max_retries?: number | null;
/**
* Skip Ineligible
* @default true
*/
skip_ineligible: boolean;
};
/** StorageOperationExecuteRequest */
StorageOperationExecuteRequest: {
execution_policy?: components["schemas"]["StorageOperationExecutePolicy"];
/**
* Notify On Completion
* @default true
*/
notify_on_completion: boolean;
/**
* Snapshot Id
* @example 0123456789ABCDEF
*/
snapshot_id: string;
};
/** StorageOperationExecuteResponse */
StorageOperationExecuteResponse: {
run: components["schemas"]["StorageOperationRunSummary"];
};
/**
* StorageOperationMode
* @enum {string}
*/
StorageOperationMode: "move";
/** StorageOperationNotificationContent */
StorageOperationNotificationContent: {
/**
* @description discriminator enum property added by openapi-typescript
* @enum {string}
*/
category: "storage_operation";
/**
* Failed Count
* @description Failed datasets count.
* @default 0
*/
failed_count: number;
/**
* History ID
* @description The encoded history ID.
* @example 0123456789ABCDEF
*/
history_id: string;
/**
* Message
* @description The message of the notification (supports Markdown).
*/
message: string;
/**
* Mode
* @description Storage operation mode.
*/
mode: string;
/**
* Run ID
* @description The encoded storage operation run ID.
* @example 0123456789ABCDEF
*/
run_id: string;
/**
* Run URL
* @description Absolute or relative URL to the storage operation run status view.
*/
run_url: string;
/**
* Skipped Count
* @description Skipped datasets count.
* @default 0
*/
skipped_count: number;
/**
* State
* @description The current state of the storage operation run when this notification was generated.
*/
state: components["schemas"]["StorageOperationRunState"];
/**
* Subject
* @description The subject of the notification.
*/
subject: string;
/**
* Succeeded Count
* @description Succeeded datasets count.
* @default 0
*/
succeeded_count: number;
/**
* Total Count
* @description Total datasets in the run.
*/
total_count: number;
};
/** StorageOperationPreviewRequest */
StorageOperationPreviewRequest: {
/** Items */
items?:
| {
[key: string]: unknown;
}[]
| null;
mode: components["schemas"]["StorageOperationMode"];
/** Target Object Store Id */
target_object_store_id: string;
};
/** StorageOperationPreviewResponse */
StorageOperationPreviewResponse: {
eligibility: components["schemas"]["StorageOperationEligibilitySummary"];
estimates: components["schemas"]["StorageOperationEstimateSummary"];
/**
* Expires At
* Format: date-time
*/
expires_at: string;
selection_counts: components["schemas"]["StorageOperationSelectionCounts"];
/**
* Snapshot Id
* @example 0123456789ABCDEF
*/
snapshot_id: string;
/** Warnings */
warnings?: string[];
};
/** StorageOperationQuotaDeltaTransfer */
StorageOperationQuotaDeltaTransfer: {
/**
* Bytes
* @default 0
*/
bytes: number;
/** Source Object Store Id */
source_object_store_id: string;
/** Target Object Store Id */
target_object_store_id: string;
};
/** StorageOperationQuotaProjectionSummary */
StorageOperationQuotaProjectionSummary: {
/** Projected Usage */
projected_usage: number;
/** Quota Limit */
quota_limit: number;
};
/**
* StorageOperationRunItemState
* @enum {string}
*/
StorageOperationRunItemState: "pending" | "running" | "succeeded" | "failed" | "skipped";
/** StorageOperationRunItemStatus */
StorageOperationRunItemStatus: {
/** Bytes Processed */
bytes_processed: number;
/**
* Create Time
* Format: date-time
* @description The time and date this item was created.
*/
create_time: string;
/**
* Dataset Id
* @example 0123456789ABCDEF
*/
dataset_id: string;
reason_code?: components["schemas"]["DatasetStorageOperationFailureReasonCode"] | null;
state: components["schemas"]["StorageOperationRunItemState"];
/**
* Update Time
* Format: date-time
* @description The last time and date this item was updated.
*/
update_time: string;
};
/** StorageOperationRunResponse */
StorageOperationRunResponse: {
/** Items */
items?: components["schemas"]["StorageOperationRunItemStatus"][];
run: components["schemas"]["StorageOperationRunSummary"];
};
/**
* StorageOperationRunState
* @enum {string}
*/
StorageOperationRunState: "pending" | "running" | "completed" | "failed";
/** StorageOperationRunSummary */
StorageOperationRunSummary: {
/**
* Create Time
* Format: date-time
* @description The time and date this item was created.
*/
create_time: string;
/** Failed Count */
failed_count: number;
mode: components["schemas"]["StorageOperationMode"];
/**
* Run Id
* @example 0123456789ABCDEF
*/
run_id: string;
/** Skipped Count */
skipped_count: number;
state: components["schemas"]["StorageOperationRunState"];
/** Succeeded Count */
succeeded_count: number;
/** Target Object Store Id */
target_object_store_id: string;
/** Task Id */
task_id?: string | null;
/** Total Bytes Processed */
total_bytes_processed: number;
/** Total Count */
total_count: number;
/**
* Update Time
* Format: date-time
* @description The last time and date this item was updated.
*/
update_time: string;
};
/** StorageOperationSelectionCounts */
StorageOperationSelectionCounts: {
/** Expanded Leaf Count */
expanded_leaf_count: number;
/** Selected Items Count */
selected_items_count: number;
/** Unique Dataset Count */
unique_dataset_count: number;
};
/** StoreExportPayload */
StoreExportPayload: {
/**
@@ -24603,6 +24950,13 @@ export interface components {
* "push": true
* },
* "enabled": true
* },
* "storage_operation": {
* "channels": {
* "email": true,
* "push": true
* },
* "enabled": true
* }
* }
* }
@@ -25006,6 +25360,13 @@ export interface components {
* "push": true
* },
* "enabled": true
* },
* "storage_operation": {
* "channels": {
* "email": true,
* "push": true
* },
* "enabled": true
* }
* }
* }
@@ -25035,7 +25396,8 @@ export interface components {
*/
content:
| components["schemas"]["MessageNotificationContent"]
| components["schemas"]["NewSharedItemNotificationContent"];
| components["schemas"]["NewSharedItemNotificationContent"]
| components["schemas"]["StorageOperationNotificationContent"];
/**
* Create time
* Format: date-time
@@ -37855,6 +38217,236 @@ export interface operations {
};
};
};
bulk_storage_operation_execute_api_histories__history_id__contents_bulk_storage_execute_post: {
parameters: {
query?: never;
header?: {
/** @description The user ID that will be used to effectively make this API call. Only admins and designated users can make API calls on behalf of other users. */
"run-as"?: string | null;
};
path: {
/** @description The encoded database identifier of the History. */
history_id: string;
};
cookie?: never;
};
requestBody: {
content: {
"application/json": components["schemas"]["StorageOperationExecuteRequest"];
};
};
responses: {
/** @description Successful Response */
200: {
headers: {
[name: string]: unknown;
};
content: {
"application/json": components["schemas"]["StorageOperationExecuteResponse"];
};
};
/** @description Request Error */
"4XX": {
headers: {
[name: string]: unknown;
};
content: {
"application/json": components["schemas"]["MessageExceptionModel"];
};
};
/** @description Server Error */
"5XX": {
headers: {
[name: string]: unknown;
};
content: {
"application/json": components["schemas"]["MessageExceptionModel"];
};
};
};
};
bulk_storage_operation_preview_api_histories__history_id__contents_bulk_storage_preview_post: {
parameters: {
query?: {
/** @description Generally a property name to filter by followed by an (often optional) hyphen and operator string. */
q?: string[] | null;
/** @description The value to filter by. */
qv?: string[] | null;
};
header?: {
/** @description The user ID that will be used to effectively make this API call. Only admins and designated users can make API calls on behalf of other users. */
"run-as"?: string | null;
};
path: {
/** @description The encoded database identifier of the History. */
history_id: string;
};
cookie?: never;
};
requestBody: {
content: {
"application/json": components["schemas"]["StorageOperationPreviewRequest"];
};
};
responses: {
/** @description Successful Response */
200: {
headers: {
[name: string]: unknown;
};
content: {
"application/json": components["schemas"]["StorageOperationPreviewResponse"];
};
};
/** @description Request Error */
"4XX": {
headers: {
[name: string]: unknown;
};
content: {
"application/json": components["schemas"]["MessageExceptionModel"];
};
};
/** @description Server Error */
"5XX": {
headers: {
[name: string]: unknown;
};
content: {
"application/json": components["schemas"]["MessageExceptionModel"];
};
};
};
};
bulk_storage_operation_run_api_histories__history_id__contents_bulk_storage_runs__run_id__get: {
parameters: {
query?: never;
header?: {
/** @description The user ID that will be used to effectively make this API call. Only admins and designated users can make API calls on behalf of other users. */
"run-as"?: string | null;
};
path: {
/** @description The encoded database identifier of the History. */
history_id: string;
run_id: string;
};
cookie?: never;
};
requestBody?: never;
responses: {
/** @description Successful Response */
200: {
headers: {
[name: string]: unknown;
};
content: {
"application/json": components["schemas"]["StorageOperationRunResponse"];
};
};
/** @description Request Error */
"4XX": {
headers: {
[name: string]: unknown;
};
content: {
"application/json": components["schemas"]["MessageExceptionModel"];
};
};
/** @description Server Error */
"5XX": {
headers: {
[name: string]: unknown;
};
content: {
"application/json": components["schemas"]["MessageExceptionModel"];
};
};
};
};
bulk_storage_operation_run_items_api_histories__history_id__contents_bulk_storage_runs__run_id__items_get: {
parameters: {
query?: {
/** @description The offset for paginated per-item run details. */
offset?: number;
/** @description The maximum number of per-item run details to return. */
limit?: number;
/**
* @description A mix of free text and GitHub-style tags used to filter the index operation.
*
* ## Query Structure
*
* GitHub-style filter tags (not be confused with Galaxy tags) are tags of the form
* `<tag_name>:<text_no_spaces>` or `<tag_name>:'<text with potential spaces>'`. The tag name
* *generally* (but not exclusively) corresponds to the name of an attribute on the model
* being indexed (i.e. a column in the database).
*
* If the tag is quoted, the attribute will be filtered exactly. If the tag is unquoted,
* generally a partial match will be used to filter the query (i.e. in terms of the implementation
* this means the database operation `ILIKE` will typically be used).
*
* Once the tagged filters are extracted from the search query, the remaining text is just
* used to search various documented attributes of the object.
*
* ## GitHub-style Tags Available
*
* `state`
* : Item state.
*
* `reason_code`
* : Item reason code. (The tag `reason` can be used a short hand alias for this tag to filter on this attribute.)
*
* `dataset_id`
* : Encoded dataset id. (The tag `dataset` can be used a short hand alias for this tag to filter on this attribute.)
*
* ## Free Text
*
* Free text search terms will be searched against the following attributes of the
* Storage operation run items: `state`, `reason_code`, `dataset_id`.
*/
search?: string | null;
};
header?: {
/** @description The user ID that will be used to effectively make this API call. Only admins and designated users can make API calls on behalf of other users. */
"run-as"?: string | null;
};
path: {
/** @description The encoded database identifier of the History. */
history_id: string;
run_id: string;
};
cookie?: never;
};
requestBody?: never;
responses: {
/** @description Successful Response */
200: {
headers: {
[name: string]: unknown;
};
content: {
"application/json": components["schemas"]["StorageOperationRunItemStatus"][];
};
};
/** @description Request Error */
"4XX": {
headers: {
[name: string]: unknown;
};
content: {
"application/json": components["schemas"]["MessageExceptionModel"];
};
};
/** @description Server Error */
"5XX": {
headers: {
[name: string]: unknown;
};
content: {
"application/json": components["schemas"]["MessageExceptionModel"];
};
};
};
};
history_contents__download_collection: {
parameters: {
query?: never;
@@ -0,0 +1,71 @@
<script setup lang="ts">
import { BPopover } from "bootstrap-vue";
import { computed } from "vue";
import { useDatasetStore } from "@/stores/datasetStore";
import { useHistoryStore } from "@/stores/historyStore";
import localize from "@/utils/localization";
import DatasetInformation from "@/components/DatasetInformation/DatasetInformation.vue";
import LoadingSpan from "@/components/LoadingSpan.vue";
const props = defineProps<{
datasetId: string;
}>();
const datasetStore = useDatasetStore();
const historyStore = useHistoryStore();
const targetId = computed(() => `storage-run-item-dataset-${props.datasetId}`);
const details = computed(() => datasetStore.storedDatasets[props.datasetId]);
const loading = computed(() => datasetStore.isLoadingDataset(props.datasetId));
const loadError = computed(() => datasetStore.getDatasetError(props.datasetId)?.message);
async function ensureDatasetDetails() {
if (details.value || loading.value) {
return;
}
await datasetStore.fetchDataset({ id: props.datasetId });
// If dataset has a history_id, ensure the history is loaded
const datasetDetails = datasetStore.storedDatasets[props.datasetId];
if (datasetDetails?.history_id && !historyStore.getHistoryById(datasetDetails.history_id)) {
await historyStore.loadHistoryById(datasetDetails.history_id);
}
}
</script>
<template>
<div>
<router-link
:id="targetId"
class="text-monospace"
:to="`/datasets/${datasetId}/details`"
@mouseenter.native="ensureDatasetDetails"
@focus.native="ensureDatasetDetails">
{{ datasetId }}
</router-link>
<BPopover :target="targetId" triggers="hover focus" boundary="window" placement="right">
<div class="dataset-details-popover">
<div v-if="loading">
<LoadingSpan :message="localize('Loading dataset details')" />
</div>
<div v-else-if="loadError" class="text-danger">
{{ loadError }}
</div>
<div v-else-if="details">
<DatasetInformation :dataset="details" />
</div>
</div>
</BPopover>
</div>
</template>
<style scoped>
:deep(.dataset-details-popover) {
max-width: 420px;
}
</style>
@@ -7,7 +7,6 @@ import VueRouter from "vue-router";
import { HttpResponse, useServerMock } from "@/api/client/__mocks__";
import { updateContentFields } from "@/components/History/model/queries";
import { setupSelectableMock } from "@/components/ObjectStore/mockServices";
import ContentItem from "./ContentItem.vue";
@@ -25,8 +24,6 @@ vi.mock("vue-router/composables", () => ({
useRouter: vi.fn(() => ({})),
}));
setupSelectableMock();
// mock queries
updateContentFields.mockImplementation(async () => {});
@@ -45,6 +42,10 @@ describe("ContentItem", () => {
suppressLucideVue2Deprecation();
server.use(
http.get("/api/object_stores", ({ response }) => {
return response(200).json([]);
}),
http.get("/api/datasets/{dataset_id}", ({ response }) => {
// We need to use untyped here because this endpoint is not
// described in the OpenAPI spec due to its complexity for now.
@@ -3,11 +3,13 @@ import { faCheckSquare, faCompress } from "@fortawesome/free-solid-svg-icons";
import { FontAwesomeIcon } from "@fortawesome/vue-fontawesome";
import type { HistorySummaryExtended } from "@/api";
import type { StorageRun } from "@/stores/storageOperationsStore";
import localize from "@/utils/localization";
import GButton from "@/components/BaseComponents/GButton.vue";
import GButtonGroup from "@/components/BaseComponents/GButtonGroup.vue";
import DefaultOperations from "@/components/History/CurrentHistory/HistoryOperations/DefaultOperations.vue";
import HistoryStorageOperationsIndicator from "@/components/History/CurrentHistory/HistoryOperations/HistoryStorageOperationsIndicator.vue";
interface Props {
history: HistorySummaryExtended;
@@ -16,6 +18,7 @@ interface Props {
expandedCount: number;
showSelection: boolean;
isMultiViewItem: boolean;
activeStorageRuns: StorageRun[];
}
const props = defineProps<Props>();
@@ -61,15 +64,22 @@ function onUpdateOperationStatus(updateTime: number) {
</GButton>
</GButtonGroup>
<GButtonGroup v-show="showSelection">
<slot name="selection-operations" />
</GButtonGroup>
<div class="d-flex align-items-center">
<GButtonGroup v-show="showSelection">
<slot name="selection-operations" />
</GButtonGroup>
<DefaultOperations
v-if="!isMultiViewItem"
v-show="!showSelection"
:history="history"
@update:operation-running="onUpdateOperationStatus" />
<HistoryStorageOperationsIndicator
:history-id="history.id"
:show-selection="showSelection"
:active-storage-run-count="activeStorageRuns.length" />
<DefaultOperations
v-if="!isMultiViewItem"
v-show="!showSelection"
:history="history"
@update:operation-running="onUpdateOperationStatus" />
</div>
</nav>
<nav v-else-if="isMultiViewItem" class="content-operations bg-secondary">
<GButton
@@ -0,0 +1,91 @@
<script setup lang="ts">
import { faSpinner } from "@fortawesome/free-solid-svg-icons";
import { FontAwesomeIcon } from "@fortawesome/vue-fontawesome";
import { BLink, BPopover } from "bootstrap-vue";
import { storeToRefs } from "pinia";
import { computed, onUnmounted, ref, watch } from "vue";
import { useStorageHistoryRunsWatcher } from "@/composables/useStorageRunWatcher";
import { useUserFlagsStore } from "@/stores/userFlagsStore";
import localize from "@/utils/localization";
import GButton from "@/components/BaseComponents/GButton.vue";
import GButtonGroup from "@/components/BaseComponents/GButtonGroup.vue";
interface Props {
historyId: string;
showSelection: boolean;
activeStorageRunCount: number;
}
const props = defineProps<Props>();
const userFlagsStore = useUserFlagsStore();
const { showStorageOperationsHelperPopover } = storeToRefs(userFlagsStore);
const isStorageHelperVisible = ref(false);
const hasActiveStorageRuns = computed(() => props.activeStorageRunCount > 0);
const storageOperationsRoute = computed(() => `/histories/${props.historyId}/storage/runs`);
const storageOperationsButtonId = computed(() => `history-storage-operations-${props.historyId}`);
const { startPolling, stopPolling } = useStorageHistoryRunsWatcher(props.historyId);
watch(
[hasActiveStorageRuns, () => props.showSelection],
([hasRuns, showSelection]) => {
isStorageHelperVisible.value = hasRuns && !showSelection && showStorageOperationsHelperPopover.value;
},
{ immediate: true },
);
watch(
hasActiveStorageRuns,
(hasRuns) => {
if (hasRuns) {
startPolling();
} else {
stopPolling();
}
},
{ immediate: true },
);
onUnmounted(() => {
stopPolling();
});
function onDoNotShowAgain() {
userFlagsStore.ignoreStorageOperationsHelperPopover();
isStorageHelperVisible.value = false;
}
</script>
<template>
<div v-if="!showSelection && hasActiveStorageRuns">
<GButtonGroup>
<GButton
:id="storageOperationsButtonId"
tooltip
:title="localize('Background operations are running')"
class="rounded-0"
size="small"
color="blue"
transparent
:to="storageOperationsRoute">
<FontAwesomeIcon :icon="faSpinner" spin fixed-width />
</GButton>
</GButtonGroup>
<BPopover
:show.sync="isStorageHelperVisible"
:target="storageOperationsButtonId"
triggers="manual hover"
placement="bottomleft"
boundary="window">
<div class="d-flex flex-column flex-gapx-1">
<span>{{ localize("Background operations are running. Click this spinner to open status.") }}</span>
<BLink @click="onDoNotShowAgain">{{ localize("Do not show this again") }}</BLink>
</div>
</BPopover>
</div>
</template>
@@ -15,6 +15,12 @@ vi.mock("@/composables/confirmDialog", () => ({
}),
}));
vi.mock("@/stores/objectStoreStore", () => ({
useObjectStoreStore: () => ({
selectableObjectStores: [{ object_store_id: "other", name: "Other Store" }],
}),
}));
const localVue = getLocalVue();
const { server, http } = useServerMock();
@@ -22,7 +28,6 @@ const { server, http } = useServerMock();
const FAKE_HISTORY_ID = "fake_history_id";
const FAKE_HISTORY = { id: FAKE_HISTORY_ID, update_time: new Date() };
const BULK_SUCCESS_RESPONSE = { success_count: 1, errors: [] };
const NO_TASKS_CONFIG = {
enable_celery_tasks: false,
};
@@ -326,6 +331,18 @@ describe("History Selection Operations", () => {
expect(errorEvent).toHaveProperty("result");
expect(errorEvent.result).toEqual(BULK_ERROR_RESPONSE);
});
it("should hide the selection when a storage operation completes", async () => {
const wizardModal = wrapper.findComponent({ name: "StorageOperationWizardModal" });
expect(wrapper.emitted("update:show-selection")).toBeFalsy();
wizardModal.vm.$emit("completed");
await flushPromises();
expect(wrapper.emitted("update:show-selection")).toBeTruthy();
expect(wrapper.emitted("update:show-selection").at(-1)).toEqual([false]);
});
});
});
@@ -340,6 +357,11 @@ describe("History Selection Operations", () => {
const option = '[data-description="change data type"]';
expect(wrapper.find(option).exists()).toBe(false);
});
it("should hide `Manage Storage Location` option", async () => {
const option = '[data-description="storage operation"]';
expect(wrapper.find(option).exists()).toBe(false);
});
});
});
});
@@ -42,7 +42,7 @@
<span v-localize>Change Database/Build</span>
</b-dropdown-item>
<b-dropdown-item
v-if="isConfigLoaded && config.enable_celery_tasks"
v-if="isCeleryEnabled"
data-description="change data type"
@click="showChangeDatatypeModal = true">
<span v-localize>Change data type</span>
@@ -53,6 +53,14 @@
<b-dropdown-item data-description="remove tags" @click="showRemoveTagsModal = true">
<span v-localize>Remove tags</span>
</b-dropdown-item>
<b-dropdown-item
v-if="isCeleryEnabled && hasSelectableObjectStores"
data-description="storage operation"
:disabled="isAnonymous"
:title="storageOperationTitle"
@click="openStorageOperationModal">
<span v-localize>Manage Storage Location</span>
</b-dropdown-item>
<b-dropdown-divider v-if="showBuildOptions" />
<b-dropdown-item v-if="showBuildOptions" data-description="auto build list" @click="listWizard(false)">
<span v-localize>Auto Build List</span>
@@ -123,6 +131,14 @@
<StatelessTags :key="showRemoveTagsModal" v-model="selectedTags" class="tags" />
<GTip :tips="['Press Enter after typing each tag.']" />
</GModal>
<StorageOperationWizardModal
:show.sync="showStorageOperationModal"
:history="history"
:filter-text="filterText"
:content-selection="contentSelection"
:is-query-selection="isQuerySelection"
:num-selected="numSelected"
@completed="onStorageOperationCompleted" />
<CollectionCreatorIndex
v-if="collectionModalType"
:history-id="history.id"
@@ -157,7 +173,10 @@ import { StatelessTags } from "@/components/Tags";
import { useConfig } from "@/composables/config";
import { useConfirmDialog } from "@/composables/confirmDialog";
import { useCollectionBuilderItemSelection } from "@/stores/collectionBuilderItemsStore";
import { useObjectStoreStore } from "@/stores/objectStoreStore";
import { useUserStore } from "@/stores/userStore";
import StorageOperationWizardModal from "./StorageOperationWizardModal.vue";
import GModal from "@/components/BaseComponents/GModal.vue";
import GTip from "@/components/BaseComponents/GTip.vue";
import CollectionCreatorIndex from "@/components/Collections/CollectionCreatorIndex.vue";
@@ -171,6 +190,7 @@ export default {
GModal,
GTip,
SingleItemSelector,
StorageOperationWizardModal,
StatelessTags,
},
props: {
@@ -185,12 +205,15 @@ export default {
setup() {
const { config, isConfigLoaded } = useConfig(true);
const { confirm } = useConfirmDialog();
const objectStoreStore = useObjectStoreStore();
const { isAnonymous } = useUserStore();
// Modals for selection operations
const showChangeDbKeyModal = ref(false);
const showChangeDatatypeModal = ref(false);
const showAddTagsModal = ref(false);
const showRemoveTagsModal = ref(false);
const showStorageOperationModal = ref(false);
const selectedDbKey = ref({ id: "?", text: "unspecified (?)" });
const selectedDatatype = ref({ id: "auto", text: "Auto-detect" });
@@ -205,11 +228,14 @@ export default {
return {
config,
confirm,
isAnonymous,
isConfigLoaded,
objectStoreStore,
showChangeDbKeyModal,
showChangeDatatypeModal,
showAddTagsModal,
showRemoveTagsModal,
showStorageOperationModal,
selectedDbKey,
selectedDatatype,
resetDbKey,
@@ -272,6 +298,13 @@ export default {
noTagsSelected() {
return this.selectedTags.length === 0;
},
isCeleryEnabled() {
return this.isConfigLoaded && this.config.enable_celery_tasks;
},
hasSelectableObjectStores() {
const stores = this.objectStoreStore.selectableObjectStores;
return stores != null && stores.some((s) => s.object_store_id && !s.hidden);
},
areAllSelectedPurged() {
for (const item of this.contentSelection.values()) {
if (Object.prototype.hasOwnProperty.call(item, "purged") && !item["purged"]) {
@@ -318,6 +351,12 @@ export default {
isAnyDeletedStateAllowed() {
return HistoryFilters.checkFilter(this.filterText, "deleted", "any");
},
storageOperationTitle() {
if (this.isAnonymous) {
return this.localize("Log in to") + " " + this.localize("Manage Storage Location");
}
return this.localize("Manage Storage Location");
},
},
watch: {
hasSelection(newVal) {
@@ -390,6 +429,12 @@ export default {
this.runOnSelection(addTagsToSelectedContent, { tags: this.selectedTags });
this.selectedTags = [];
},
openStorageOperationModal() {
this.showStorageOperationModal = true;
},
onStorageOperationCompleted() {
this.$emit("update:show-selection", false);
},
removeTagsFromSelected() {
this.runOnSelection(removeTagsFromSelectedContent, { tags: this.selectedTags });
this.selectedTags = [];
@@ -0,0 +1,301 @@
<script setup lang="ts">
import { faInfoCircle } from "@fortawesome/free-solid-svg-icons";
import { FontAwesomeIcon } from "@fortawesome/vue-fontawesome";
import { BAlert, BBadge, BListGroup, BListGroupItem } from "bootstrap-vue";
import { computed, onMounted, ref, watch } from "vue";
import { GalaxyApi } from "@/api";
import type { StorageOperationPreviewResponse } from "@/api/histories";
import { useObjectStoreStore } from "@/stores/objectStoreStore";
import { useQuotaUsageStore } from "@/stores/quotaUsageStore";
import { getIneligibleReasonDescription } from "@/utils/storageOperations";
import { bytesToString } from "@/utils/utils";
import StorageQuotaImpactBar from "@/components/History/StorageOperations/StorageQuotaImpactBar.vue";
import Popper from "@/components/Popper/Popper.vue";
const props = defineProps<{
preview: StorageOperationPreviewResponse;
targetStoreId: string;
}>();
interface QuotaUsageDeltaEntry {
storeId: string;
usageDeltaBytes: number;
}
interface QuotaImpactEntry extends QuotaUsageDeltaEntry {
isTargetStore: boolean;
quotaLimitBytes: number | null;
currentUsageBytes: number;
}
const objectStoreStore = useObjectStoreStore();
const quotaUsageStore = useQuotaUsageStore();
const objectStoreUsageById = ref<Record<string, number>>({});
const objectStoreUsageLoaded = ref(false);
const objectStoreUsageLoading = ref(false);
const transferEstimate = computed(() => {
const bytes = props.preview.estimates?.bytes_to_transfer;
if (bytes == null) {
return "Not available";
}
return bytesToString(bytes);
});
const quotaUsageDeltaEntries = computed<QuotaUsageDeltaEntry[]>(() => {
const transfers = props.preview.estimates?.quota_delta_transfers ?? [];
const byStore = new Map<string, number>();
for (const transfer of transfers) {
const bytes = transfer.bytes ?? 0;
// Usage decreases on the source and increases on the target.
byStore.set(transfer.source_object_store_id, (byStore.get(transfer.source_object_store_id) ?? 0) - bytes);
byStore.set(transfer.target_object_store_id, (byStore.get(transfer.target_object_store_id) ?? 0) + bytes);
}
return Array.from(byStore.entries())
.map(([storeId, usageDeltaBytes]) => ({
storeId,
usageDeltaBytes,
}))
.filter((entry) => entry.usageDeltaBytes !== 0)
.sort((a, b) => a.storeId.localeCompare(b.storeId));
});
const affectedStoreIdsKey = computed(() => quotaUsageDeltaEntries.value.map((entry) => entry.storeId).join("|"));
function clearObjectStoreUsage(): void {
objectStoreUsageById.value = {};
objectStoreUsageLoaded.value = false;
}
async function ensureQuotaUsageLoadedOnce(): Promise<void> {
if (!quotaUsageStore.isLoaded) {
await quotaUsageStore.loadQuotaUsages();
}
}
async function fetchObjectStoreUsageMap(): Promise<Record<string, number> | null> {
const { data, error } = await GalaxyApi().GET("/api/users/{user_id}/objectstore_usage", {
params: { path: { user_id: "current" } },
});
if (error || !data) {
return null;
}
return Object.fromEntries(data.map((entry) => [entry.object_store_id, entry.total_disk_usage]));
}
async function loadObjectStoreUsageIfNeeded(): Promise<void> {
if (!affectedStoreIdsKey.value) {
clearObjectStoreUsage();
return;
}
if (objectStoreUsageLoaded.value || objectStoreUsageLoading.value) {
return;
}
objectStoreUsageLoading.value = true;
try {
const usageMap = await fetchObjectStoreUsageMap();
if (!usageMap) {
clearObjectStoreUsage();
return;
}
objectStoreUsageById.value = usageMap;
objectStoreUsageLoaded.value = true;
} finally {
objectStoreUsageLoading.value = false;
}
}
function handleAffectedStoresChanged(storeIdsKey: string): void {
if (!storeIdsKey) {
clearObjectStoreUsage();
return;
}
void loadObjectStoreUsageIfNeeded();
}
onMounted(() => {
void ensureQuotaUsageLoadedOnce();
});
watch(affectedStoreIdsKey, handleAffectedStoresChanged, { immediate: true });
const quotaSourceByStoreId = computed<Record<string, string | null>>(() => {
const stores = objectStoreStore.selectableObjectStores ?? [];
return Object.fromEntries(
stores
.filter((store) => Boolean(store.object_store_id))
.map((store) => [store.object_store_id!, store.quota.source ?? null]),
);
});
const maxUsageScaleBytes = computed(() => {
const scaleCandidates = quotaImpactEntries.value.flatMap((entry) => {
const currentUsage = entry.currentUsageBytes;
const projectedUsage = Math.max(0, currentUsage + entry.usageDeltaBytes);
return [currentUsage, projectedUsage];
});
if (!scaleCandidates.length) {
return 1;
}
return Math.max(1, ...scaleCandidates);
});
const quotaProjection = computed<QuotaProjection | null>(() => props.preview.estimates.quota_projection ?? null);
const quotaImpactEntries = computed<QuotaImpactEntry[]>(() => {
return quotaUsageDeltaEntries.value.map((entry) => {
const isTargetStore = entry.storeId === props.targetStoreId;
const quotaSourceLabel = quotaSourceByStoreId.value[entry.storeId] ?? null;
const quotaUsage = quotaUsageStore.getQuotaUsageBySourceLabel(quotaSourceLabel) ?? null;
return {
...entry,
isTargetStore,
quotaLimitBytes: quotaUsage?.quotaInBytes ?? null,
currentUsageBytes:
objectStoreUsageById.value[entry.storeId] ??
(isTargetStore && quotaProjection.value
? quotaProjection.value.projected_usage + entry.usageDeltaBytes
: 0),
};
});
});
const ineligibleReasonBreakdown = computed(() => {
return (props.preview.eligibility?.reasons ?? [])
.map((reason) => ({
code: reason.reason_code,
count: reason.count,
}))
.sort((a, b) => b.count - a.count);
});
const hasIneligible = computed(() => props.preview.eligibility.ineligible_count > 0);
const noneEligible = computed(() => props.preview.eligibility.eligible_count === 0);
const hasWarnings = computed(() => Boolean(props.preview.warnings?.length) || noneEligible.value);
type QuotaProjection = NonNullable<StorageOperationPreviewResponse["estimates"]["quota_projection"]>;
const targetStoreName = computed(() => {
return objectStoreStore.getObjectStoreNameById(props.targetStoreId) ?? "Unknown storage location";
});
function getObjectStoreLabel(storeId: string) {
return objectStoreStore.getObjectStoreNameById(storeId) ?? "Unknown storage location";
}
</script>
<template>
<div class="storage-operation-preview-report">
<!-- Destination -->
<p class="mb-3">
<span class="text-muted">Moving to: </span>
<strong>{{ targetStoreName }}</strong>
</p>
<!-- Selection counts -->
<div class="d-flex flex-wrap mb-3">
<div class="mr-4 mb-2">
<div class="font-weight-bold h5 mb-0">{{ preview.selection_counts.selected_items_count }}</div>
<small class="text-muted">Selected items</small>
</div>
<div class="mr-4 mb-2">
<div class="font-weight-bold h5 mb-0">{{ preview.selection_counts.expanded_leaf_count }}</div>
<small class="text-muted">Expanded datasets</small>
</div>
<div class="mr-4 mb-2">
<div class="font-weight-bold h5 mb-0">{{ preview.selection_counts.unique_dataset_count }}</div>
<small class="text-muted">Unique datasets</small>
</div>
</div>
<!-- Eligibility summary -->
<div class="d-flex align-items-center flex-wrap mb-3">
<BBadge variant="success" class="mr-2 mb-1 px-2 py-1">
{{ preview.eligibility.eligible_count }} eligible
</BBadge>
<BBadge :variant="hasIneligible ? 'danger' : 'secondary'" class="mr-3 mb-1 px-2 py-1">
{{ preview.eligibility.ineligible_count }} ineligible
</BBadge>
<span class="text-muted small mb-1">Estimated transfer: {{ transferEstimate }}</span>
</div>
<!-- Storage quota changes -->
<div v-if="quotaUsageDeltaEntries.length" class="mb-3">
<div class="d-flex align-items-center mb-2">
<p class="mb-0 font-weight-bold">Quota availability impact by location:</p>
<Popper placement="top" mode="light">
<template v-slot:reference>
<span class="ml-2 text-muted" aria-label="Quota impact help" tabindex="0">
<FontAwesomeIcon :icon="faInfoCircle" />
</span>
</template>
<div class="p-2 small" style="max-width: 28rem">
Based on current usage and quota limits, the bars below illustrate how available quota will
change on each affected storage location if the operation is executed. The "Current" and "After
Change" values represent total usage on that storage location before and after the operation,
while "Gain" or "Loss" reflects the usage change for that location. These are estimates based on
current information and actual results may vary at execution time.
</div>
</Popper>
</div>
<p class="mb-2 text-muted small">You can hover over each bar for more details.</p>
<StorageQuotaImpactBar
v-for="entry in quotaImpactEntries"
:key="entry.storeId"
:store-label="getObjectStoreLabel(entry.storeId)"
:current-usage-bytes="entry.currentUsageBytes"
:delta-bytes="entry.usageDeltaBytes"
:quota-limit-bytes="entry.quotaLimitBytes"
:scale-max-bytes="maxUsageScaleBytes"
:is-target-store="entry.isTargetStore" />
<div v-if="!quotaImpactEntries.length" class="text-muted small">
No quota availability changes were detected for this operation.
</div>
</div>
<!-- Ineligibility breakdown -->
<div v-if="ineligibleReasonBreakdown.length" class="mb-3">
<p class="mb-1 font-weight-bold">Why some datasets cannot be moved:</p>
<BListGroup flush>
<BListGroupItem
v-for="reason in ineligibleReasonBreakdown"
:key="reason.code"
class="px-0 py-2 d-flex align-items-start border-0">
<BBadge variant="danger" class="mr-2 mt-1 flex-shrink-0">{{ reason.count }}</BBadge>
<div>
<div class="font-weight-bold small">
{{ getIneligibleReasonDescription(reason.code).label }}
</div>
<div v-if="getIneligibleReasonDescription(reason.code).description" class="text-muted small">
{{ getIneligibleReasonDescription(reason.code).description }}
</div>
</div>
</BListGroupItem>
</BListGroup>
</div>
<!-- Warnings -->
<BAlert v-if="hasWarnings" show variant="warning" class="mb-0 py-2">
<div class="font-weight-bold small mb-1">Warnings</div>
<div v-for="(warning, index) in preview.warnings" :key="index" class="small py-1">
{{ warning }}
</div>
<div v-if="noneEligible" class="small py-1">
All selected datasets are ineligible for transfer. Please review the reasons above and adjust your
selection or target storage location.
</div>
</BAlert>
</div>
</template>
@@ -0,0 +1,330 @@
<script setup lang="ts">
import { BAlert, BButton } from "bootstrap-vue";
import { storeToRefs } from "pinia";
import { computed, ref, watch } from "vue";
import Multiselect from "vue-multiselect";
import type { HistoryContentItemBase, UserConcreteObjectStoreModel } from "@/api";
import type { HistoryReference, StorageOperationPreviewResponse } from "@/api/histories";
import { useWizard } from "@/components/Common/Wizard/useWizard";
import { HistoryFilters } from "@/components/History/HistoryFilters";
import { bulkStorageExecute, bulkStoragePreview } from "@/components/History/model/queries";
import { useConfig } from "@/composables/config";
import { Toast } from "@/composables/toast";
import { useObjectStoreStore } from "@/stores/objectStoreStore";
import { useQuotaUsageStore } from "@/stores/quotaUsageStore";
import { useStorageOperationsStore } from "@/stores/storageOperationsStore";
import { errorMessageAsString } from "@/utils/simple-error";
import { toTrackedStorageRun } from "@/utils/storageOperations";
import GModal from "@/components/BaseComponents/GModal.vue";
import GenericWizard from "@/components/Common/Wizard/GenericWizard.vue";
import StorageOperationPreviewReport from "@/components/History/CurrentHistory/HistoryOperations/StorageOperationPreviewReport.vue";
import LoadingSpan from "@/components/LoadingSpan.vue";
import ObjectStoreBadges from "@/components/ObjectStore/ObjectStoreBadges.vue";
import ProvidedQuotaSourceUsageBar from "@/components/User/DiskUsage/Quota/ProvidedQuotaSourceUsageBar.vue";
const props = defineProps<{
show: boolean;
history: HistoryReference;
filterText: string;
contentSelection: Map<unknown, HistoryContentItemBase>;
isQuerySelection: boolean;
numSelected: number;
}>();
const emit = defineEmits<{
(e: "update:show", value: boolean): void;
(e: "completed"): void;
}>();
const objectStoreStore = useObjectStoreStore();
const quotaUsageStore = useQuotaUsageStore();
const storageOperationsStore = useStorageOperationsStore();
const { loading: objectStoresLoading, loadErrorMessage } = storeToRefs(objectStoreStore);
const { config, isConfigLoaded } = useConfig(true);
const selectedTargetObjectStoreId = ref<string | null>(null);
const storagePreview = ref<StorageOperationPreviewResponse | null>(null);
const storagePreviewLoading = ref(false);
const storageExecuting = ref(false);
const previewError = ref<string | null>(null);
const executionError = ref<string | null>(null);
const notifyOnCompletion = ref(true);
const dropdownOpen = ref(false);
const isAnyItemEligible = computed(() => {
return (storagePreview.value?.eligibility.eligible_count ?? 0) > 0;
});
const notificationSystemEnabled = computed(
() => isConfigLoaded.value && config.value?.enable_notification_system === true,
);
const shouldNotifyOnCompletion = computed(() => notifyOnCompletion.value && notificationSystemEnabled.value);
const wizard = useWizard({
configure: {
label: "Destination",
instructions: computed(() => `Choose the storage location to move ${props.numSelected} selected item(s) to.`),
isValid: () => Boolean(selectedTargetObjectStoreId.value),
isSkippable: () => false,
},
preview: {
label: "Preview",
instructions: "Review what will be moved and the estimated impact before starting.",
isValid: () => Boolean(storagePreview.value?.snapshot_id) && isAnyItemEligible.value && !storageExecuting.value,
isSkippable: () => false,
},
});
const isBusy = computed(() => storagePreviewLoading.value || storageExecuting.value);
const showProxy = computed({
get: () => props.show,
set: (value: boolean) => emit("update:show", value),
});
const storageTargetOptions = computed<UserConcreteObjectStoreModel[]>(() => {
const stores = objectStoreStore.selectableObjectStores ?? [];
return stores.filter(
(store): store is UserConcreteObjectStoreModel & { object_store_id: string } =>
Boolean(store.object_store_id) && !store.hidden,
);
});
const selectedTargetObjectStore = computed<UserConcreteObjectStoreModel | null>(() => {
if (!selectedTargetObjectStoreId.value) {
return null;
}
return storageTargetOptions.value.find((s) => s.object_store_id === selectedTargetObjectStoreId.value) ?? null;
});
// Trigger preview fetch when wizard navigates to the preview step.
watch(wizard.index, async (newIndex, oldIndex) => {
if (newIndex === 1) {
storagePreview.value = null;
await previewStorageOperation();
} else if (newIndex === 0 && oldIndex === 1) {
// Went back to configure — clear preview/error state.
storagePreview.value = null;
previewError.value = null;
executionError.value = null;
}
});
watch(
() => props.show,
(newVal) => {
if (newVal) {
resetState();
}
},
);
function onCancel() {
showProxy.value = false;
resetState();
}
function onTargetStoreChanged() {
previewError.value = null;
executionError.value = null;
}
watch(dropdownOpen, (isOpen) => {
if (isOpen) {
void quotaUsageStore.loadQuotaUsages(true);
}
});
function onTargetStoreSelected(target: UserConcreteObjectStoreModel | null) {
selectedTargetObjectStoreId.value = target?.object_store_id ?? null;
onTargetStoreChanged();
}
function resetState() {
wizard.goTo("configure");
selectedTargetObjectStoreId.value = null;
storagePreview.value = null;
storagePreviewLoading.value = false;
storageExecuting.value = false;
previewError.value = null;
executionError.value = null;
notifyOnCompletion.value = true;
dropdownOpen.value = false;
}
async function previewStorageOperation() {
if (!selectedTargetObjectStoreId.value) {
return;
}
previewError.value = null;
executionError.value = null;
storagePreviewLoading.value = true;
try {
const filters = HistoryFilters.getQueryDict(props.filterText);
const items = getExplicitlySelectedItems();
const previewResponse = await bulkStoragePreview(
props.history,
selectedTargetObjectStoreId.value,
filters,
items,
);
if (!previewResponse) {
throw new Error("Failed to preview storage operation.");
}
storagePreview.value = previewResponse;
} catch (error) {
storagePreview.value = null;
previewError.value =
errorMessageAsString(error) || "An unknown error occurred while previewing the storage operation.";
Toast.error(previewError.value, "Storage Preview Failed");
wizard.goTo("configure");
} finally {
storagePreviewLoading.value = false;
}
}
async function executeStorageOperation() {
const snapshotId = storagePreview.value?.snapshot_id;
if (!snapshotId) {
return;
}
storageExecuting.value = true;
executionError.value = null;
try {
const executeResponse = await bulkStorageExecute(
props.history,
snapshotId,
undefined,
shouldNotifyOnCompletion.value,
);
storageOperationsStore.startRun(toTrackedStorageRun(props.history.id, executeResponse.run));
Toast.success("Storage operation submitted successfully.", "Storage Operation Submitted");
emit("completed");
showProxy.value = false;
resetState();
} catch (error) {
executionError.value =
errorMessageAsString(error) || "An unknown error occurred while executing the storage operation.";
Toast.error(executionError.value, "Storage Operation Failed");
} finally {
storageExecuting.value = false;
}
}
function getExplicitlySelectedItems(): HistoryContentItemBase[] {
if (props.isQuerySelection) {
return [];
}
return Array.from(props.contentSelection.values()).map((item) => {
return {
id: item.id,
history_content_type: item.history_content_type,
};
});
}
</script>
<template>
<GModal :show.sync="showProxy" title="Move Datasets to New Storage Location" size="medium" overflow-visible>
<BAlert v-if="previewError" show variant="danger" dismissible class="mb-2" @dismissed="previewError = null">
{{ previewError }}
</BAlert>
<BAlert v-if="executionError" show variant="danger" dismissible class="mb-2" @dismissed="executionError = null">
{{ executionError }}
</BAlert>
<GenericWizard
:use="wizard"
:is-busy="isBusy"
submit-button-label="Run operation"
container-component="div"
@submit="executeStorageOperation">
<template v-slot:cancel-button>
<BButton variant="secondary" @click="onCancel">Cancel</BButton>
</template>
<div v-if="wizard.isCurrent('configure')">
<div class="mb-3">
<label class="d-block mb-1 font-weight-bold" for="storage-target-select">
Target storage location
</label>
<LoadingSpan v-if="objectStoresLoading" message="Loading Galaxy storage information" />
<BAlert v-else-if="loadErrorMessage" show variant="danger" class="mb-2">
{{ loadErrorMessage }}
</BAlert>
<Multiselect
v-else
id="storage-target-select"
:value="selectedTargetObjectStore"
:options="storageTargetOptions"
:allow-empty="false"
:searchable="false"
:show-labels="false"
track-by="object_store_id"
label="name"
placeholder="Select target storage location"
class="w-100 multiselect--soft-option-highlight"
@input="onTargetStoreSelected"
@open="dropdownOpen = true"
@close="dropdownOpen = false">
<template v-slot:singleLabel="{ option }">
<span>{{ option.name ?? "Unknown storage location" }}</span>
</template>
<template v-slot:option="{ option }">
<div class="w-100 text-wrap py-1">
<div class="d-flex align-items-start justify-content-between">
<span class="font-weight-bold">{{
option.name ?? "Unknown storage location"
}}</span>
<ObjectStoreBadges :badges="option.badges" size="lg" class="ml-2 flex-shrink-0" />
</div>
<div v-if="option.description" class="small text-muted mt-1 text-break">
{{ option.description }}
</div>
<ProvidedQuotaSourceUsageBar
v-if="dropdownOpen && option.quota && option.quota.enabled"
:object-store="option"
class="mt-1" />
</div>
</template>
</Multiselect>
</div>
<div v-if="notificationSystemEnabled" class="mb-2">
<label class="d-flex align-items-center mb-0">
<input v-model="notifyOnCompletion" type="checkbox" class="mr-2" />
Notify me when the storage operation completes
</label>
<small class="text-muted d-block mt-1">
When disabled, no completion notification will be sent for this run.
</small>
</div>
</div>
<div v-else-if="wizard.isCurrent('preview')">
<LoadingSpan
v-if="storagePreviewLoading"
class="text-center text-muted py-3"
message="Loading preview information for the selected items. This may take a few moments. Please do not close this dialog while the preview is loading." />
<BAlert v-else-if="!storagePreview" show variant="info">
No preview information is available. Please go back and ensure you have selected a target storage
location.
</BAlert>
<StorageOperationPreviewReport
v-else-if="storagePreview"
:preview="storagePreview"
:target-store-id="selectedTargetObjectStoreId || ''" />
</div>
</GenericWizard>
</GModal>
</template>
@@ -11,6 +11,7 @@ import { deleteContent, updateContentFields } from "@/components/History/model/q
import { useSelectedItems } from "@/composables/selectedItems/selectedItems";
import { useHistoryItemsStore } from "@/stores/historyItemsStore";
import { useHistoryStore } from "@/stores/historyStore";
import { useStorageOperationsStore } from "@/stores/storageOperationsStore";
import { useUserStore } from "@/stores/userStore";
import { type Alias, getOperatorForAlias } from "@/utils/filtering";
import { setItemDragstart } from "@/utils/setDrag";
@@ -76,6 +77,7 @@ const { lastCheckedTime, totalMatchesCount, isWatching } = storeToRefs(useHistor
const historyStore = useHistoryStore();
const historyItemsStore = useHistoryItemsStore();
const storageOperationsStore = useStorageOperationsStore();
const { currentUser } = storeToRefs(useUserStore());
const historyIdComputed = computed(() => props.history.id);
@@ -111,6 +113,10 @@ const historyItems = computed(() => {
return historyItemsStore.getHistoryItems(props.history.id, filterText.value);
});
const activeStorageRuns = computed(() => {
return storageOperationsStore.getActiveRuns(props.history.id);
});
const visibleHistoryItems = computed(() => {
return historyItems.value.filter((item) => !invisibleHistoryItems.value[item.hid]);
});
@@ -453,6 +459,7 @@ const {
:history="history"
:editable="canEditHistory"
:is-multi-view-item="isMultiViewItem"
:active-storage-runs="activeStorageRuns"
:show-selection="showSelection"
:expanded-count="expandedCount"
:has-matches="hasMatches(historyItems)"
@@ -33,6 +33,10 @@ describe("MultipleView", () => {
const fakeSummaries = getFakeHistorySummaries(count);
server.use(
http.get("/api/object_stores", ({ response }) => {
return response(200).json([]);
}),
http.get("/api/histories/{history_id}", ({ response, params }) => {
const { history_id } = params;
const summary = fakeSummaries.find((s) => s.id === history_id);
@@ -0,0 +1,223 @@
<script setup lang="ts">
import { BAlert, BFormInput } from "bootstrap-vue";
import { computed, onMounted, onUnmounted, ref, watch } from "vue";
import { useStorageHistoryRunsWatcher } from "@/composables/useStorageRunWatcher";
import { useHistoryStore } from "@/stores/historyStore";
import { type StorageRun, useStorageOperationsStore } from "@/stores/storageOperationsStore";
import localize from "@/utils/localization";
import { isTerminalRunState } from "@/utils/storageOperations";
import StorageOperationRunsTable from "./StorageOperationRunsTable.vue";
import Heading from "@/components/Common/Heading.vue";
type EnrichedStorageRun = ReturnType<typeof enrichRunWithSummary>;
const props = defineProps<{
historyId: string;
}>();
const historyStore = useHistoryStore();
const storageOperationsStore = useStorageOperationsStore();
const sortBy = ref("create_time");
const sortDesc = ref(true);
const filterText = ref("");
const inProgressPage = ref(1);
const finishedPage = ref(1);
const perPage = 20;
const { startPolling, stopPolling, runSummariesByRunId } = useStorageHistoryRunsWatcher(props.historyId);
const trackedRuns = computed(() => {
return storageOperationsStore.runs.filter((run) => run.historyId === props.historyId);
});
/**
* Enrich tracked runs with computed fields from API summaries
*/
function enrichRunWithSummary(run: StorageRun) {
const summary = runSummariesByRunId.value[run.run_id];
const succeededCount = summary?.succeeded_count ?? run.succeeded_count;
const failedCount = summary?.failed_count ?? run.failed_count;
const skippedCount = summary?.skipped_count ?? run.skipped_count;
const totalCount = summary?.total_count ?? run.total_count;
const totalBytesProcessed = summary?.total_bytes_processed ?? run.total_bytes_processed;
const updateTime = summary?.update_time ?? run.update_time;
const processed = succeededCount + failedCount + skippedCount;
const progressPercent = totalCount > 0 ? Math.min(100, Math.round((processed / totalCount) * 100)) : 0;
return {
...run,
total_count: totalCount,
succeeded_count: succeededCount,
failed_count: failedCount,
skipped_count: skippedCount,
total_bytes_processed: totalBytesProcessed,
update_time: updateTime,
progressPercent,
};
}
const tableRows = computed(() => {
return trackedRuns.value.map(enrichRunWithSummary);
});
const filteredRows = computed(() => {
const filter = filterText.value.trim().toLowerCase();
if (!filter) {
return tableRows.value;
}
return tableRows.value.filter((row) => {
const haystack = [row.state, row.mode, row.target_object_store_id, row.total_count, row.create_time, row.run_id]
.join(" ")
.toLowerCase();
return haystack.includes(filter);
});
});
const displayRows = computed(() => {
return [...filteredRows.value].sort((a, b) => {
const aValue = getSortableValue(a);
const bValue = getSortableValue(b);
let comparison = 0;
if (typeof aValue === "number" && typeof bValue === "number") {
comparison = aValue - bValue;
} else {
comparison = String(aValue).localeCompare(String(bValue));
}
return sortDesc.value ? -comparison : comparison;
});
});
const hasRuns = computed(() => tableRows.value.length > 0);
const historyName = computed(() => historyStore.getHistoryNameById(props.historyId));
const inProgressRows = computed(() => {
return displayRows.value.filter((row) => !isTerminalRunState(row.state));
});
const finishedRows = computed(() => {
return displayRows.value.filter((row) => isTerminalRunState(row.state));
});
const inProgressMaxPage = computed(() => {
return Math.max(1, Math.ceil(inProgressRows.value.length / perPage));
});
const finishedMaxPage = computed(() => {
return Math.max(1, Math.ceil(finishedRows.value.length / perPage));
});
function dismissStorageRun(runId: string) {
storageOperationsStore.clearRun(runId);
}
function onSortChanged(newSortBy: string, newSortDesc: boolean) {
sortBy.value = newSortBy;
sortDesc.value = newSortDesc;
}
function getSortableValue(row: EnrichedStorageRun) {
const key = sortBy.value;
if (key === "create_time" || key === "update_time") {
const val = key === "create_time" ? row.create_time : row.update_time;
const timestamp = Date.parse(val);
return Number.isNaN(timestamp) ? 0 : timestamp;
}
return row[key as keyof EnrichedStorageRun] ?? "";
}
onMounted(() => {
startPolling();
});
onUnmounted(() => {
stopPolling();
});
watch(displayRows, () => {
if (inProgressPage.value > inProgressMaxPage.value) {
inProgressPage.value = inProgressMaxPage.value;
}
if (finishedPage.value > finishedMaxPage.value) {
finishedPage.value = finishedMaxPage.value;
}
});
watch(filterText, () => {
inProgressPage.value = 1;
finishedPage.value = 1;
});
function onInProgressPageChange(page: number) {
inProgressPage.value = page;
}
function onFinishedPageChange(page: number) {
finishedPage.value = page;
}
</script>
<template>
<div class="storage-operation-history-view">
<Heading h2 separator size="md">{{ localize("History Storage Operations") }}</Heading>
<p class="mb-3">
{{ localize("Here you can track background storage operations related to the history") }}
<strong>{{ historyName }}</strong>
{{ localize(", such as moving or copying data across storage locations.") }}
</p>
<BAlert v-if="!hasRuns" show variant="secondary">
{{ localize("No tracked storage operations for this history.") }}
</BAlert>
<template v-else>
<div class="d-flex flex-wrap align-items-center mb-2 gap-2">
<BFormInput
v-model="filterText"
class="storage-run-filter"
:placeholder="localize('Filter runs by target store location...')" />
</div>
<StorageOperationRunsTable
:title="localize('In Progress')"
:rows="inProgressRows"
:current-page="inProgressPage"
:per-page="perPage"
:sort-by="sortBy"
:sort-desc="sortDesc"
:empty-message="localize('No in-progress runs found.')"
pagination-class="mt-2 mb-3"
@sort-changed="onSortChanged"
@page-change="onInProgressPageChange"
@dismiss="dismissStorageRun" />
<StorageOperationRunsTable
:title="localize('Finished')"
:rows="finishedRows"
:current-page="finishedPage"
:per-page="perPage"
:sort-by="sortBy"
:sort-desc="sortDesc"
:empty-message="localize('No finished runs found.')"
heading-class="mt-3"
pagination-class="mt-2"
@sort-changed="onSortChanged"
@page-change="onFinishedPageChange"
@dismiss="dismissStorageRun" />
</template>
</div>
</template>
<style scoped>
.storage-run-filter {
min-width: 280px;
max-width: 520px;
}
</style>
@@ -0,0 +1,107 @@
<script setup lang="ts">
import { BPopover, BProgress, BProgressBar } from "bootstrap-vue";
import { computed } from "vue";
import localize from "@/utils/localization";
type ProgressHeightClass = "storage-operation-progress-sm" | "storage-operation-progress-md";
interface Props {
totalCount: number;
succeededCount: number;
failedCount: number;
skippedCount: number;
heightClass?: ProgressHeightClass;
showDetails?: boolean;
}
const props = withDefaults(defineProps<Props>(), {
heightClass: "storage-operation-progress-sm",
showDetails: true,
});
const processedCount = computed(() => props.succeededCount + props.failedCount + props.skippedCount);
const progressPercent = computed(() => {
if (props.totalCount <= 0) {
return 0;
}
return Math.min(100, Math.round((processedCount.value / props.totalCount) * 100));
});
const succeededPercent = computed(() => {
if (props.totalCount <= 0) {
return 0;
}
return Math.round((props.succeededCount / props.totalCount) * 100);
});
const failedPercent = computed(() => {
if (props.totalCount <= 0) {
return 0;
}
return Math.round((props.failedCount / props.totalCount) * 100);
});
const skippedPercent = computed(() => {
if (props.totalCount <= 0) {
return 0;
}
return Math.round((props.skippedCount / props.totalCount) * 100);
});
const progressLabel = computed(() => {
return localize(`Progress: ${processedCount.value} / ${props.totalCount} (${progressPercent.value}%)`);
});
const popoverTargetId = `storage-operation-progress-${Math.random().toString(36).slice(2, 10)}`;
</script>
<template>
<div>
<div :id="popoverTargetId" class="storage-operation-progress-target">
<BProgress :max="props.totalCount || 1" :class="props.heightClass" :aria-label="progressLabel">
<BProgressBar variant="success" :value="props.succeededCount" />
<BProgressBar variant="danger" :value="props.failedCount" />
<BProgressBar variant="warning" :value="props.skippedCount" />
</BProgress>
</div>
<BPopover
v-if="props.showDetails"
:target="popoverTargetId"
triggers="hover focus"
boundary="window"
placement="top">
<div class="storage-operation-progress-popover">
<div>
<strong>{{ localize("Success") }}:</strong> {{ props.succeededCount }} ({{ succeededPercent }}%)
</div>
<div>
<strong>{{ localize("Failed") }}:</strong> {{ props.failedCount }} ({{ failedPercent }}%)
</div>
<div>
<strong>{{ localize("Skipped") }}:</strong> {{ props.skippedCount }} ({{ skippedPercent }}%)
</div>
<div class="text-muted mt-1">
<strong>{{ localize("Total Processed") }}:</strong> {{ processedCount }} /
{{ props.totalCount }} ({{ progressPercent }}%)
</div>
</div>
</BPopover>
</div>
</template>
<style scoped>
.storage-operation-progress-popover {
min-width: 220px;
}
.storage-operation-progress-sm {
height: 0.6rem;
}
.storage-operation-progress-md {
height: 1rem;
}
</style>
@@ -0,0 +1,86 @@
<script setup lang="ts">
import { faCheckCircle, faClock, faExclamationCircle, faExclamationTriangle } from "@fortawesome/free-solid-svg-icons";
import { FontAwesomeIcon } from "@fortawesome/vue-fontawesome";
import { computed } from "vue";
import localize from "@/utils/localization";
interface Props {
state: string;
failedCount?: number;
skippedCount?: number;
}
const props = withDefaults(defineProps<Props>(), {
failedCount: 0,
skippedCount: 0,
});
const statusLabel = computed(() => {
if (props.state === "completed") {
if ((props.failedCount ?? 0) > 0) {
return localize("Completed with failures");
}
if ((props.skippedCount ?? 0) > 0) {
return localize("Completed with skips");
}
return localize("Completed successfully");
}
if (props.state === "running") {
return localize("Running");
}
if (props.state === "failed") {
return localize("Failed");
}
return props.state;
});
const statusIcon = computed(() => {
if (props.state === "completed") {
if ((props.failedCount ?? 0) > 0) {
return faExclamationCircle;
}
if ((props.skippedCount ?? 0) > 0) {
return faExclamationTriangle;
}
return faCheckCircle;
}
if (props.state === "failed") {
return faExclamationCircle;
}
return faClock;
});
const iconClass = computed(() => {
if (props.state === "failed") {
return "text-danger";
}
if (props.state === "completed") {
if ((props.failedCount ?? 0) > 0) {
return "text-danger";
}
if ((props.skippedCount ?? 0) > 0) {
return "text-warning";
}
return "text-success";
}
if (props.state === "running") {
return "text-info";
}
return "text-secondary";
});
</script>
<template>
<div v-g-tooltip.hover class="storage-run-state-badge" :title="statusLabel">
<FontAwesomeIcon :icon="statusIcon" fixed-width :class="iconClass" />
</div>
</template>
<style scoped>
.storage-run-state-badge {
display: inline-flex;
align-items: center;
gap: 0.4rem;
}
</style>
@@ -0,0 +1,322 @@
<script setup lang="ts">
import { BAlert, BBadge, BPagination } from "bootstrap-vue";
import { computed, onBeforeUnmount, onMounted, ref, watch } from "vue";
import {
getStorageOperationRunItemsWithTotal,
type HistoryReference,
type StorageOperationRunItemStatus,
} from "@/api/histories";
import type { TableField } from "@/components/Common/GTable.types";
import { useStorageRunWatcher } from "@/composables/useStorageRunWatcher";
import { useObjectStoreStore } from "@/stores/objectStoreStore";
import { useStorageOperationsStore } from "@/stores/storageOperationsStore";
import Filtering, { contains } from "@/utils/filtering";
import localize from "@/utils/localization";
import { getIneligibleReasonDescription, toTrackedStorageRun } from "@/utils/storageOperations";
import { bytesToString } from "@/utils/utils";
import BreadcrumbHeading from "@/components/Common/BreadcrumbHeading.vue";
import DatasetPopoverLink from "@/components/Common/DatasetPopoverLink.vue";
import FilterMenu from "@/components/Common/FilterMenu.vue";
import GTable from "@/components/Common/GTable.vue";
import Heading from "@/components/Common/Heading.vue";
import StorageOperationOutcomeProgress from "@/components/History/StorageOperations/StorageOperationOutcomeProgress.vue";
import StorageOperationRunStateBadge from "@/components/History/StorageOperations/StorageOperationRunStateBadge.vue";
import LoadingSpan from "@/components/LoadingSpan.vue";
const props = defineProps<{
historyId: string;
runId: string;
}>();
const history: HistoryReference = {
id: props.historyId,
model_class: "History",
};
const { runStatus, isTerminal, startPolling, stopPolling } = useStorageRunWatcher(history, props.runId);
const storageOperationsStore = useStorageOperationsStore();
const objectStoreStore = useObjectStoreStore();
const run = computed(() => runStatus.value?.run);
const pageItems = ref<StorageOperationRunItemStatus[]>([]);
const isLoadingItems = ref(false);
const itemsTotalMatches = ref(0);
const filterText = ref("");
const showAdvanced = ref(false);
const currentPage = ref(1);
const perPage = 50;
const runItemFilterClass = new Filtering(
{
dataset_id: {
placeholder: "dataset id",
type: String,
handler: contains("dataset_id"),
menuItem: true,
},
reason_code: {
placeholder: "reason code",
type: String,
handler: contains("reason_code"),
menuItem: true,
},
},
undefined,
true,
false,
);
const breadcrumbItems = computed(() => [
{ title: "History Storage Operations", to: `/histories/${props.historyId}/storage/runs` },
{ title: "Storage Operation Run" },
]);
const tableFields: TableField[] = [
{ key: "dataset_id", label: localize("Dataset"), width: "260px" },
{ key: "state", label: localize("State"), width: "120px" },
{ key: "reason_code", label: localize("Reason code"), width: "180px" },
{ key: "reason_text", label: localize("Reason"), sortable: false, class: "run-item-reason" },
];
const failedOrSkippedCount = computed(() => (run.value?.failed_count ?? 0) + (run.value?.skipped_count ?? 0));
const displayItems = computed(() => pageItems.value);
const totalDataCopiedDisplay = computed(() => {
const totalBytesProcessed = run.value?.total_bytes_processed ?? 0;
const succeededCount = run.value?.succeeded_count ?? 0;
if (totalBytesProcessed === 0 && succeededCount > 0) {
return localize("No data transfer required");
}
return bytesToString(totalBytesProcessed);
});
const failedItemsEmptyStateMessage = computed(() => {
if (!isTerminal.value) {
return localize("No failed or skipped items yet. This run is still in progress.");
}
return localize("No failed or skipped items for this run.");
});
const maxPage = computed(() => {
return Math.max(1, Math.ceil(itemsTotalMatches.value / perPage));
});
const targetStoreDisplayName = computed(() => {
return (
objectStoreStore.getObjectStoreNameById(run.value?.target_object_store_id) ??
run.value?.target_object_store_id ??
""
);
});
function onPageChange(page: number) {
currentPage.value = page;
}
function buildItemsSearchQuery(filterText: string) {
const filter = filterText.trim();
return filter ? `state:failed state:skipped ${filter}` : "state:failed state:skipped";
}
async function loadFailedOrSkippedPage() {
if (!run.value || failedOrSkippedCount.value === 0) {
pageItems.value = [];
itemsTotalMatches.value = 0;
return;
}
isLoadingItems.value = true;
try {
const offset = (currentPage.value - 1) * perPage;
const search = buildItemsSearchQuery(filterText.value);
const result = await getStorageOperationRunItemsWithTotal(history, props.runId, {
offset,
limit: perPage,
search,
});
pageItems.value = result.data;
itemsTotalMatches.value = result.totalMatches ?? failedOrSkippedCount.value;
} finally {
isLoadingItems.value = false;
}
}
function getItemStateVariant(state: string) {
if (state === "failed") {
return "danger";
}
if (state === "skipped") {
return "warning";
}
return "secondary";
}
function getReasonText(reasonCode: string | null | undefined) {
if (!reasonCode) {
return "-";
}
return localize(getIneligibleReasonDescription(reasonCode).description);
}
watch(filterText, () => {
currentPage.value = 1;
void loadFailedOrSkippedPage();
});
watch(maxPage, () => {
if (currentPage.value > maxPage.value) {
currentPage.value = maxPage.value;
}
});
watch(
[currentPage, failedOrSkippedCount],
() => {
void loadFailedOrSkippedPage();
},
{ immediate: true },
);
watch(
run,
(currentRun) => {
if (!currentRun) {
return;
}
storageOperationsStore.startRun(toTrackedStorageRun(props.historyId, currentRun));
},
{ immediate: true },
);
onMounted(() => {
startPolling();
});
onBeforeUnmount(() => {
stopPolling();
});
</script>
<template>
<div class="storage-operation-run-view p-3">
<BreadcrumbHeading :items="breadcrumbItems" />
<LoadingSpan v-if="!run" :message="localize('Loading storage operation run status')" />
<template v-else>
<BAlert show variant="secondary">
<div class="d-flex align-items-center flex-wrap">
<strong class="mr-2">{{ localize("State:") }}</strong>
<StorageOperationRunStateBadge
class="mr-3"
:state="run.state"
:failed-count="run.failed_count"
:skipped-count="run.skipped_count" />
<span class="mr-3">
<strong>{{ localize("Mode:") }}</strong> {{ run.mode }}
</span>
<span>
<strong>{{ localize("Target store:") }}</strong> {{ targetStoreDisplayName }}
</span>
</div>
</BAlert>
<div class="mb-3">
<strong>{{ localize("Total:") }}</strong> {{ run.total_count }}
<span class="mx-2">|</span>
<strong>{{ localize("Succeeded:") }}</strong> {{ run.succeeded_count }}
<span class="mx-2">|</span>
<strong>{{ localize("Failed:") }}</strong> {{ run.failed_count }}
<span class="mx-2">|</span>
<strong>{{ localize("Skipped:") }}</strong> {{ run.skipped_count }}
<span class="mx-2">|</span>
<strong>{{ localize("Data transferred:") }}</strong> {{ totalDataCopiedDisplay }}
</div>
<div class="mb-3">
<StorageOperationOutcomeProgress
:total-count="run.total_count"
:succeeded-count="run.succeeded_count"
:failed-count="run.failed_count"
:skipped-count="run.skipped_count"
height-class="storage-operation-progress-md" />
</div>
<BAlert v-if="!isTerminal" show variant="info">
{{ localize("This run is still in progress. Status is updated automatically.") }}
</BAlert>
<div class="mt-3">
<Heading h3 size="sm">{{ localize("Failed or Skipped Items") }}</Heading>
<FilterMenu
class="run-item-filter mb-2"
name="Storage Run Items"
view="compact"
placeholder="search failed/skipped items"
:filter-class="runItemFilterClass"
:filter-text.sync="filterText"
:loading="isLoadingItems"
:show-advanced.sync="showAdvanced" />
<LoadingSpan v-if="isLoadingItems" class="mb-2" :message="localize('Loading failed/skipped items')" />
<GTable
id="storage-operation-run-items-table"
striped
hover
stacked="md"
show-empty
:fields="tableFields"
:items="displayItems"
:current-page="1"
:per-page="0"
:local-sorting="false"
:local-filtering="false"
:empty-state="{ message: failedItemsEmptyStateMessage }">
<template v-slot:cell(dataset_id)="slot">
<DatasetPopoverLink :dataset-id="slot.item.dataset_id" />
</template>
<template v-slot:cell(state)="slot">
<BBadge :variant="getItemStateVariant(slot.item.state)">{{ slot.item.state }}</BBadge>
</template>
<template v-slot:cell(reason_code)="slot">
<span>{{ slot.item.reason_code || "-" }}</span>
</template>
<template v-slot:cell(reason_text)="slot">
<span>{{ getReasonText(slot.item.reason_code) }}</span>
</template>
</GTable>
<div v-if="itemsTotalMatches > perPage" class="d-flex justify-content-end mt-2">
<BPagination
:value="currentPage"
:total-rows="itemsTotalMatches"
:per-page="perPage"
align="right"
size="sm"
first-number
last-number
@change="onPageChange" />
</div>
</div>
</template>
</div>
</template>
<style scoped>
.run-item-filter {
min-width: 280px;
max-width: 520px;
}
:deep(.run-item-reason) {
white-space: normal;
word-break: break-word;
}
</style>
@@ -0,0 +1,186 @@
<script setup lang="ts">
import { faEye, faTimes } from "@fortawesome/free-solid-svg-icons";
import { FontAwesomeIcon } from "@fortawesome/vue-fontawesome";
import { BPagination } from "bootstrap-vue";
import { computed } from "vue";
import type { TableField } from "@/components/Common/GTable.types";
import { useObjectStoreStore } from "@/stores/objectStoreStore";
import type { StorageRun } from "@/stores/storageOperationsStore";
import localize from "@/utils/localization";
import { isTerminalRunState } from "@/utils/storageOperations";
import { bytesToString } from "@/utils/utils";
import GButton from "@/components/BaseComponents/GButton.vue";
import GButtonGroup from "@/components/BaseComponents/GButtonGroup.vue";
import GTable from "@/components/Common/GTable.vue";
import Heading from "@/components/Common/Heading.vue";
import StorageOperationOutcomeProgress from "@/components/History/StorageOperations/StorageOperationOutcomeProgress.vue";
import StorageOperationRunStateBadge from "@/components/History/StorageOperations/StorageOperationRunStateBadge.vue";
import UtcDate from "@/components/UtcDate.vue";
type StorageOperationTableRow = StorageRun & {
progressPercent: number;
};
interface Props {
title: string;
rows: StorageOperationTableRow[];
currentPage: number;
perPage: number;
sortBy: string;
sortDesc: boolean;
emptyMessage: string;
headingClass?: string;
paginationClass?: string;
}
const props = withDefaults(defineProps<Props>(), {
headingClass: undefined,
paginationClass: undefined,
});
const emit = defineEmits<{
(e: "sort-changed", sortBy: string, sortDesc: boolean): void;
(e: "page-change", page: number): void;
(e: "dismiss", runId: string): void;
}>();
const objectStoreStore = useObjectStoreStore();
const fields: TableField[] = [
{ key: "state", label: localize("Status"), sortable: true, width: "20px", align: "center" },
{ key: "mode", label: localize("Mode"), sortable: false, width: "80px" },
{ key: "target_object_store_id", label: localize("Target store"), sortable: true },
{ key: "total_count", label: localize("Datasets"), sortable: true, align: "center", width: "120px" },
{ key: "total_bytes_processed", label: localize("Data transferred"), sortable: true, width: "160px" },
{ key: "progressPercent", label: localize("Progress"), sortable: true, width: "220px" },
{ key: "create_time", label: localize("Started"), sortable: true, width: "180px" },
{ key: "update_time", label: localize("Completed"), sortable: true, width: "180px" },
{ key: "actions", label: localize("Actions"), sortable: false, width: "90px" },
];
const showPagination = computed(() => props.rows.length > props.perPage);
function onSortChanged(newSortBy: string, newSortDesc: boolean) {
emit("sort-changed", newSortBy, newSortDesc);
}
function onPageChange(page: number) {
emit("page-change", page);
}
function dismissStorageRun(runId: string) {
emit("dismiss", runId);
}
function getTargetStoreDisplayName(targetObjectStoreId: string) {
return objectStoreStore.getObjectStoreNameById(targetObjectStoreId) ?? targetObjectStoreId;
}
function formatDataCopied(totalBytesProcessed: number, succeededCount: number) {
if (totalBytesProcessed === 0 && succeededCount > 0) {
return localize("No data transfer required");
}
return bytesToString(totalBytesProcessed);
}
</script>
<template>
<div>
<Heading h3 size="sm" :class="props.headingClass">{{ props.title }}</Heading>
<GTable
striped
hover
stacked="md"
show-empty
:fields="fields"
:items="props.rows"
:current-page="props.currentPage"
:per-page="props.perPage"
:sort-by="props.sortBy"
:sort-desc="props.sortDesc"
:local-sorting="false"
:local-filtering="false"
:empty-state="{ message: props.emptyMessage }"
@sort-changed="onSortChanged">
<template v-slot:cell(state)="slot">
<StorageOperationRunStateBadge
:state="slot.item.state"
:failed-count="slot.item.failed_count"
:skipped-count="slot.item.skipped_count" />
</template>
<template v-slot:cell(mode)="slot">
<span class="text-monospace">{{ slot.item.mode }}</span>
</template>
<template v-slot:cell(target_object_store_id)="slot">
{{ getTargetStoreDisplayName(slot.item.target_object_store_id) }}
</template>
<template v-slot:cell(total_bytes_processed)="slot">
<span>{{ formatDataCopied(slot.item.total_bytes_processed, slot.item.succeeded_count) }}</span>
</template>
<template v-slot:cell(progressPercent)="slot">
<StorageOperationOutcomeProgress
:total-count="slot.item.total_count"
:succeeded-count="slot.item.succeeded_count"
:failed-count="slot.item.failed_count"
:skipped-count="slot.item.skipped_count" />
</template>
<template v-slot:cell(create_time)="slot">
<UtcDate :date="slot.item.create_time" mode="elapsed" />
</template>
<template v-slot:cell(update_time)="slot">
<span v-if="isTerminalRunState(slot.item.state)">
<UtcDate :date="slot.item.update_time" mode="elapsed" />
</span>
<span v-else class="text-muted">—</span>
</template>
<template v-slot:cell(actions)="slot">
<GButtonGroup aria-label="Actions">
<GButton
tooltip
tooltip-placement="bottom"
size="small"
color="blue"
outline
icon-only
:title="localize('View details')"
:to="slot.item.runUrl">
<FontAwesomeIcon :icon="faEye" fixed-width />
</GButton>
<GButton
tooltip
tooltip-placement="bottom"
size="small"
color="red"
outline
icon-only
:title="localize('Dismiss')"
@click.stop="dismissStorageRun(slot.item.run_id)">
<FontAwesomeIcon :icon="faTimes" fixed-width />
</GButton>
</GButtonGroup>
</template>
</GTable>
<div v-if="showPagination" class="d-flex justify-content-end" :class="props.paginationClass">
<BPagination
:value="props.currentPage"
:total-rows="props.rows.length"
:per-page="props.perPage"
align="right"
size="sm"
first-number
last-number
@change="onPageChange" />
</div>
</div>
</template>
@@ -0,0 +1,168 @@
<script setup lang="ts">
import { BBadge } from "bootstrap-vue";
import { computed, ref } from "vue";
import { bytesToString } from "@/utils/utils";
import Popper from "@/components/Popper/Popper.vue";
interface Props {
storeLabel: string;
currentUsageBytes: number;
deltaBytes: number;
quotaLimitBytes?: number | null;
scaleMaxBytes: number;
isTargetStore?: boolean;
}
const props = withDefaults(defineProps<Props>(), {
isTargetStore: false,
quotaLimitBytes: null,
});
const barReference = ref<HTMLElement | null>(null);
// deltaBytes is a usage delta: negative means less used space (quota gain), positive means more used space (quota loss).
const isGain = computed(() => props.deltaBytes < 0);
const projectedUsageBytes = computed(() => Math.max(0, props.currentUsageBytes + props.deltaBytes));
const percentBaselineBytes = computed(() => {
if (props.quotaLimitBytes && props.quotaLimitBytes > 0) {
return props.quotaLimitBytes;
}
return props.scaleMaxBytes;
});
const formattedDelta = computed(() => {
const sign = isGain.value ? "+" : "-";
return `${sign}${bytesToString(Math.abs(props.deltaBytes))}`;
});
const currentPercent = computed(() => {
if (percentBaselineBytes.value <= 0) {
return 0;
}
return Math.min(100, (props.currentUsageBytes / percentBaselineBytes.value) * 100);
});
const projectedPercent = computed(() => {
if (percentBaselineBytes.value <= 0) {
return 0;
}
return Math.min(100, (projectedUsageBytes.value / percentBaselineBytes.value) * 100);
});
const deltaStartPercent = computed(() => {
return isGain.value ? projectedPercent.value : currentPercent.value;
});
const deltaWidthPercent = computed(() => {
return Math.max(0, Math.abs(projectedPercent.value - currentPercent.value));
});
const deltaChangePercentOfCurrent = computed(() => {
if (props.quotaLimitBytes && props.quotaLimitBytes > 0) {
return (Math.abs(props.deltaBytes) / props.quotaLimitBytes) * 100;
}
if (props.currentUsageBytes <= 0) {
return 0;
}
return (Math.abs(props.deltaBytes) / props.currentUsageBytes) * 100;
});
</script>
<template>
<div class="storage-quota-impact-bar py-2">
<div class="d-flex align-items-center mb-1">
<span class="small text-muted font-weight-bold">{{ props.storeLabel }}</span>
<BBadge v-if="props.isTargetStore" variant="info" class="ml-2">Target</BBadge>
</div>
<div
ref="barReference"
class="quota-usage-bar"
role="img"
tabindex="0"
:aria-label="`${props.storeLabel} usage change`">
<div class="quota-usage-current" :style="{ width: `${currentPercent}%` }"></div>
<div
class="quota-usage-delta"
:class="{ gain: isGain, loss: !isGain }"
:style="{ left: `${deltaStartPercent}%`, width: `${deltaWidthPercent}%` }"></div>
</div>
<Popper v-if="barReference" :reference-el="barReference || undefined" placement="right-end" mode="light">
<div class="p-2">
<div class="font-weight-bold mb-1">{{ props.storeLabel }}</div>
<div class="d-flex justify-content-between">
<span class="text-muted mr-2">Current</span>
<span>{{ bytesToString(props.currentUsageBytes) }}</span>
</div>
<div v-if="props.quotaLimitBytes" class="d-flex justify-content-between">
<span class="text-muted mr-2">Quota Limit</span>
<span>{{ bytesToString(props.quotaLimitBytes) }}</span>
</div>
<hr class="my-2" />
<div class="d-flex justify-content-between">
<span class="text-muted mr-2">After Change</span>
<span>{{ bytesToString(projectedUsageBytes) }}</span>
</div>
<div class="d-flex justify-content-between mt-1">
<span class="text-muted mr-2">{{ isGain ? "Gain" : "Loss" }}</span>
<span>
{{ formattedDelta }}
({{ deltaChangePercentOfCurrent.toFixed(2) }}%)
</span>
</div>
</div>
</Popper>
</div>
</template>
<style lang="scss" scoped>
@import "bootstrap/scss/_functions.scss";
@import "@/style/scss/theme/blue.scss";
.storage-quota-impact-bar + .storage-quota-impact-bar {
border-top: 1px solid $border-color;
}
.quota-usage-bar {
position: relative;
height: 0.85rem;
width: 100%;
background: $gray-300;
border-radius: 999px;
overflow: hidden;
}
.quota-usage-current {
height: 100%;
background: $gray-600;
}
.quota-usage-delta {
position: absolute;
top: 0;
height: 100%;
min-width: 2px;
}
.quota-usage-delta.gain {
background: theme-color("success");
opacity: 0.85;
}
.quota-usage-delta.loss {
background: theme-color("warning");
opacity: 0.9;
}
</style>
@@ -0,0 +1,225 @@
import flushPromises from "flush-promises";
import { describe, expect, it } from "vitest";
import { useServerMock } from "@/api/client/__mocks__";
import type {
HistoryReference,
StorageOperationExecuteResponse,
StorageOperationPreviewResponse,
StorageOperationRunResponse,
} from "@/api/histories";
import { useStorageRunWatcher } from "@/composables/useStorageRunWatcher";
import { bulkStorageExecute, bulkStoragePreview, bulkStorageRunStatus } from "./queries";
const HISTORY: HistoryReference = { id: "history1", model_class: "History" };
const SNAPSHOT_ID = "aaaabbbb-0000-1111-2222-333344445555";
const RUN_ID = "ccccdddd-0000-1111-2222-333344445555";
const PREVIEW_RESPONSE: StorageOperationPreviewResponse = {
snapshot_id: SNAPSHOT_ID,
expires_at: "2099-01-01T00:00:00",
selection_counts: { selected_items_count: 1, expanded_leaf_count: 1, unique_dataset_count: 1 },
eligibility: {
eligible_count: 1,
ineligible_count: 0,
reasons: [],
},
estimates: { bytes_to_transfer: 0, quota_delta_transfers: [] },
warnings: [],
};
const EXECUTE_RESPONSE: StorageOperationExecuteResponse = {
run: {
run_id: RUN_ID,
state: "pending",
mode: "move",
target_object_store_id: "other",
create_time: "2099-01-01T00:00:00",
update_time: "2099-01-01T00:00:00",
total_count: 1,
succeeded_count: 0,
failed_count: 0,
skipped_count: 0,
total_bytes_processed: 0,
},
};
const RUN_PENDING_RESPONSE: StorageOperationRunResponse = {
run: { ...EXECUTE_RESPONSE.run },
items: [],
};
const RUN_COMPLETED_RESPONSE: StorageOperationRunResponse = {
run: {
...EXECUTE_RESPONSE.run,
state: "completed",
succeeded_count: 1,
total_bytes_processed: 42,
},
items: [
{
dataset_id: "ds1",
state: "succeeded",
reason_code: null,
bytes_processed: 0,
create_time: "2099-01-01T00:00:00",
update_time: "2099-01-01T00:00:00",
},
],
};
const RUN_FAILED_RESPONSE: StorageOperationRunResponse = {
run: {
...EXECUTE_RESPONSE.run,
state: "completed",
failed_count: 1,
total_bytes_processed: 0,
},
items: [
{
dataset_id: "ds1",
state: "failed",
reason_code: "already_in_target",
bytes_processed: 0,
create_time: "2099-01-01T00:00:00",
update_time: "2099-01-01T00:00:00",
},
],
};
const { server, http } = useServerMock();
describe("bulkStoragePreview", () => {
it("posts to the preview endpoint and returns snapshot data", async () => {
server.use(
http.post("/api/histories/{history_id}/contents/bulk/storage/preview", ({ response }) =>
response(200).json(PREVIEW_RESPONSE),
),
);
const result = await bulkStoragePreview(HISTORY, "other", {}, []);
expect(result).toBeDefined();
expect(result!.snapshot_id).toBe(SNAPSHOT_ID);
expect(result!.eligibility.eligible_count).toBe(1);
expect(result!.eligibility.ineligible_count).toBe(0);
});
it("propagates server errors as exceptions", async () => {
server.use(
http.post("/api/histories/{history_id}/contents/bulk/storage/preview", ({ response }) =>
response("4XX").json({ err_msg: "Not found", err_code: 404 }, { status: 404 }),
),
);
await expect(bulkStoragePreview(HISTORY, "other", {}, [])).rejects.toThrow();
});
});
describe("bulkStorageExecute", () => {
it("posts snapshot_id and execution_policy and returns run summary", async () => {
server.use(
http.post("/api/histories/{history_id}/contents/bulk/storage/execute", ({ response }) =>
response(200).json(EXECUTE_RESPONSE),
),
);
const result = await bulkStorageExecute(HISTORY, SNAPSHOT_ID);
expect(result!.run.run_id).toBe(RUN_ID);
expect(result!.run.state).toBe("pending");
});
it("propagates server errors as exceptions", async () => {
server.use(
http.post("/api/histories/{history_id}/contents/bulk/storage/execute", ({ response }) =>
response("4XX").json({ err_msg: "Snapshot expired", err_code: 400 }, { status: 400 }),
),
);
await expect(bulkStorageExecute(HISTORY, SNAPSHOT_ID)).rejects.toThrow();
});
});
describe("bulkStorageRunStatus", () => {
it("fetches run status summary", async () => {
server.use(
http.get("/api/histories/{history_id}/contents/bulk/storage/runs/{run_id}", ({ response }) =>
response(200).json(RUN_FAILED_RESPONSE),
),
);
const result = await bulkStorageRunStatus(HISTORY, RUN_ID);
expect(result!.run.failed_count).toBe(1);
});
});
describe("useStorageRunWatcher", () => {
it("starts as non-terminal and reflects the latest polled state", async () => {
server.use(
http.get("/api/histories/{history_id}/contents/bulk/storage/runs/{run_id}", ({ response }) =>
response(200).json(RUN_PENDING_RESPONSE),
),
);
const { runStatus, isTerminal, startPolling } = useStorageRunWatcher(HISTORY, RUN_ID);
expect(isTerminal.value).toBe(false);
expect(runStatus.value).toBeNull();
startPolling();
await flushPromises();
expect(runStatus.value!.run.state).toBe("pending");
expect(isTerminal.value).toBe(false);
});
it("marks isTerminal=true when run reaches completed state and stops polling", async () => {
server.use(
http.get("/api/histories/{history_id}/contents/bulk/storage/runs/{run_id}", ({ response }) =>
response(200).json(RUN_COMPLETED_RESPONSE),
),
);
const { runStatus, isTerminal, startPolling } = useStorageRunWatcher(HISTORY, RUN_ID);
startPolling();
await flushPromises();
expect(runStatus.value!.run.state).toBe("completed");
expect(isTerminal.value).toBe(true);
});
it("marks isTerminal=true when run reaches failed state", async () => {
server.use(
http.get("/api/histories/{history_id}/contents/bulk/storage/runs/{run_id}", ({ response }) =>
response(200).json(RUN_FAILED_RESPONSE),
),
);
const { isTerminal, startPolling } = useStorageRunWatcher(HISTORY, RUN_ID);
startPolling();
await flushPromises();
expect(isTerminal.value).toBe(true);
});
it("tracks run state for failure runs", async () => {
server.use(
http.get("/api/histories/{history_id}/contents/bulk/storage/runs/{run_id}", ({ response }) =>
response(200).json(RUN_FAILED_RESPONSE),
),
);
const { runStatus, startPolling } = useStorageRunWatcher(HISTORY, RUN_ID);
startPolling();
await flushPromises();
expect(runStatus.value!.run.state).toBe("completed");
expect(runStatus.value!.run.failed_count).toBe(1);
});
});
+68 -4
View File
@@ -1,8 +1,18 @@
import { type components, GalaxyApi, type HistoryItemSummary, type HistorySummary } from "@/api";
import type { components, HistoryItemSummary } from "@/api";
import { GalaxyApi } from "@/api";
import { createHistoryDatasetCollectionInstanceSimple } from "@/api/datasetCollections";
import type {
HistoryReference,
StorageOperationExecutePolicy,
StorageOperationExecuteResponse,
StorageOperationPreviewResponse,
StorageOperationRunResponse,
} from "@/api/histories";
import { getStorageOperationRunStatus } from "@/api/histories";
import { rethrowSimple } from "@/utils/simple-error";
type BulkOperation = components["schemas"]["HistoryContentItemOperation"];
type HistoryContentItem = components["schemas"]["HistoryContentItem"];
type QueryFilters = Record<string, unknown>;
export function filtersToQueryValues(filters: QueryFilters) {
@@ -57,10 +67,10 @@ export async function updateContentFields(content: HistoryItemSummary, newFields
* the filters will determine which items are processed.
*/
export async function bulkUpdate(
history: HistorySummary,
history: HistoryReference,
operation: BulkOperation,
filters: QueryFilters,
items = [],
items: HistoryContentItem[] = [],
params = null,
) {
const { data, error } = await GalaxyApi().PUT("/api/histories/{history_id}/contents/bulk", {
@@ -81,7 +91,7 @@ export async function bulkUpdate(
return data;
}
export async function createDatasetCollection(history: HistorySummary, inputs = {}) {
export async function createDatasetCollection(history: HistoryReference, inputs = {}) {
const defaults = {
collection_type: "list",
copy_elements: true,
@@ -94,3 +104,57 @@ export async function createDatasetCollection(history: HistorySummary, inputs =
const payload = Object.assign({}, defaults, inputs);
return createHistoryDatasetCollectionInstanceSimple(payload);
}
export async function bulkStoragePreview(
history: HistoryReference,
targetObjectStoreId: string,
filters: QueryFilters,
items: HistoryContentItem[] = [],
): Promise<StorageOperationPreviewResponse> {
const { data, error } = await GalaxyApi().POST("/api/histories/{history_id}/contents/bulk/storage/preview", {
params: {
path: { history_id: history.id },
query: filtersToQueryValues(filters),
},
body: {
mode: "move",
target_object_store_id: targetObjectStoreId,
items,
},
});
if (error) {
rethrowSimple(error);
}
return data;
}
export async function bulkStorageExecute(
history: HistoryReference,
snapshotId: string,
executionPolicy: StorageOperationExecutePolicy = { skip_ineligible: true },
notifyOnCompletion = true,
): Promise<StorageOperationExecuteResponse> {
const { data, error } = await GalaxyApi().POST("/api/histories/{history_id}/contents/bulk/storage/execute", {
params: {
path: { history_id: history.id },
},
body: {
snapshot_id: snapshotId,
execution_policy: executionPolicy,
notify_on_completion: notifyOnCompletion,
},
});
if (error) {
rethrowSimple(error);
}
return data;
}
export async function bulkStorageRunStatus(
history: HistoryReference,
runId: string,
): Promise<StorageOperationRunResponse> {
return getStorageOperationRunStatus(history, runId);
}
@@ -61,6 +61,8 @@ function handleMessageClick(event: MouseEvent) {
const title = computed(() => {
if (props.notification.category === "new_shared_item") {
return `${sharedItemType.value} shared with you by ${props.notification.content.owner_name}`;
} else if (props.notification.category === "storage_operation") {
return props.notification.content.subject;
} else {
return props.notification.content.subject;
}
@@ -68,7 +70,12 @@ const title = computed(() => {
const titleIcon = computed(() => {
return {
icon: props.notification.category === "new_shared_item" ? faRetweet : faInbox,
icon:
props.notification.category === "new_shared_item"
? faRetweet
: props.notification.category === "storage_operation"
? faHourglassHalf
: faInbox,
class: `text-${notificationVariant.value}`,
};
});
@@ -176,6 +183,31 @@ function markNotificationAsSeen() {
<em>{{ props.notification.content.item_type }}</em
>{{ " " }}<span> with you.</span>
</template>
<template v-else-if="props.notification.category === 'storage_operation'">
<p>
{{ props.notification.content.message }}
</p>
<p class="mb-1">
<strong>Mode:</strong> {{ props.notification.content.mode }}
<span class="mx-2">|</span>
<strong>State:</strong> {{ props.notification.content.state }}
</p>
<p class="mb-1">
<strong>Total:</strong> {{ props.notification.content.total_count }}
<span class="mx-2">|</span>
<strong>Succeeded:</strong> {{ props.notification.content.succeeded_count }}
<span class="mx-2">|</span>
<strong>Failed:</strong> {{ props.notification.content.failed_count }}
<span class="mx-2">|</span>
<strong>Skipped:</strong> {{ props.notification.content.skipped_count }}
</p>
<BLink
class="text-primary"
:href="props.notification.content.run_url"
@click="markNotificationAsSeen()">
Open storage operation run status
</BLink>
</template>
<template v-else>
<span
class="notification-message"
@@ -37,7 +37,9 @@ const quotaSourceLabel = computed(() => props.objectStore.quota.source ?? null);
const quotaSourceKey = computed(() => quotaSourceLabel.value ?? "__null__");
const quotaUsage = computed(() => quotaUsageStore.getQuotaUsageBySourceLabel(quotaSourceLabel.value) ?? null);
const isLoadingUsage = computed(() => Boolean(quotaUsageStore.loadingBySource[quotaSourceKey.value]));
const isLoadingUsage = computed(
() => quotaUsageStore.loadingAll || Boolean(quotaUsageStore.loadingBySource[quotaSourceKey.value]),
);
watch(
() => [props.objectStore.quota.enabled, quotaSourceLabel.value] as const,
@@ -46,7 +48,15 @@ watch(
return;
}
void quotaUsageStore.loadQuotaUsageForSource(sourceLabel, true);
// Load all usages in a single bulk request first, then read per-source from cache.
// This avoids N individual /api/users/current/usage/{label} requests when
// multiple ProvidedQuotaSourceUsageBar instances mount simultaneously (e.g. in
// a dropdown with many object store options).
if (!quotaUsageStore.isLoaded) {
void quotaUsageStore.loadQuotaUsages();
} else {
void quotaUsageStore.loadQuotaUsageForSource(sourceLabel, false);
}
},
{ immediate: true },
);
@@ -13,6 +13,8 @@ const categoryDescriptionMap: Record<NotificationCategory, string> = {
`,
new_shared_item:
"You will receive these notifications when someone shares an item with you i.e. a history, workflow, visualization, etc.",
storage_operation:
"You will receive these notifications when a bulk storage operation completes (including failed outcomes).",
};
interface NotificationsCategorySettingsProps {
@@ -0,0 +1,212 @@
import { computed, readonly, type Ref, ref } from "vue";
import { getStorageOperationRunStatus, type HistoryReference, type StorageOperationRunResponse } from "@/api/histories";
import { useResourceWatcher } from "@/composables/resourceWatcher";
import { useQuotaUsageStore } from "@/stores/quotaUsageStore";
import { useStorageOperationsStore } from "@/stores/storageOperationsStore";
import { useUserStore } from "@/stores/userStore";
import { isTerminalRunState } from "@/utils/storageOperations";
const STORAGE_RUN_POLLING_INTERVAL = 10000; // 10 seconds
export interface StorageRunWatcherOptions {
pollInterval?: number;
enableBackgroundPolling?: boolean;
}
const DEFAULT_RUN_WATCHER_OPTIONS: Required<StorageRunWatcherOptions> = {
pollInterval: STORAGE_RUN_POLLING_INTERVAL,
enableBackgroundPolling: false,
};
type RunSummaries = Record<string, StorageOperationRunResponse["run"]>;
interface SharedHistoryWatcher {
runSummariesByRunId: Ref<RunSummaries>;
isPolling: Readonly<Ref<boolean>>;
startPolling: () => void;
stopPolling: () => void;
activePollers: number;
}
const sharedHistoryWatchers = new Map<string, SharedHistoryWatcher>();
function resolveWatcherOptions(options?: StorageRunWatcherOptions): Required<StorageRunWatcherOptions> {
return {
...DEFAULT_RUN_WATCHER_OPTIONS,
...options,
};
}
/**
* Poll storage-operation run status until it reaches a terminal state.
*/
export function useStorageRunWatcher(history: HistoryReference, runId: string, options?: StorageRunWatcherOptions) {
const { pollInterval, enableBackgroundPolling } = resolveWatcherOptions(options);
const runStatus = ref<StorageOperationRunResponse | null>(null);
const isTerminal = computed(() => {
const state = runStatus.value?.run.state;
return state !== undefined && isTerminalRunState(state);
});
async function fetchStatus() {
const data = await getStorageOperationRunStatus(history, runId);
runStatus.value = data ?? null;
if (isTerminal.value) {
stopPolling();
}
}
const { startWatchingResource: startPolling, stopWatchingResource: stopPolling } = useResourceWatcher(fetchStatus, {
shortPollingInterval: pollInterval,
enableBackgroundPolling,
});
return { runStatus, isTerminal, startPolling, stopPolling };
}
/**
* Poll status for all tracked storage-operation runs in a history.
* The watcher is shared across all components that call this function with the same historyId, so they will all see the same run statuses and share the polling.
* Polling will only be active when at least one component is using the watcher, and will stop when no components are using it.
*/
export function useStorageHistoryRunsWatcher(historyId: string, options?: StorageRunWatcherOptions) {
const { pollInterval, enableBackgroundPolling } = resolveWatcherOptions(options);
let watcher = sharedHistoryWatchers.get(historyId);
if (!watcher) {
const storageOperationsStore = useStorageOperationsStore();
const userStore = useUserStore();
const quotaUsageStore = useQuotaUsageStore();
const runSummariesByRunId = ref<RunSummaries>({});
const historyReference: HistoryReference = {
id: historyId,
model_class: "History",
};
const watchStorageOperationRuns = async () => {
const runs = storageOperationsStore.getRuns(historyId);
if (!runs.length) {
runSummariesByRunId.value = {};
return;
}
const previous = runSummariesByRunId.value;
const runsToFetch = runs.filter((run) => !isTerminalRunState(run.state));
if (!runsToFetch.length) {
const nextFromPrevious: RunSummaries = {};
runs.forEach((run) => {
const summary = previous[run.run_id];
if (summary) {
nextFromPrevious[run.run_id] = summary;
}
});
runSummariesByRunId.value = nextFromPrevious;
return;
}
const settled = await Promise.allSettled(
runsToFetch.map(async (run) => {
const status = await getStorageOperationRunStatus(historyReference, run.run_id);
return { run_id: run.run_id, summary: status?.run };
}),
);
const next: RunSummaries = {};
let hasNewlyTerminalRun = false;
runs.forEach((run) => {
const existingSummary = previous[run.run_id];
if (existingSummary) {
next[run.run_id] = existingSummary;
}
});
settled.forEach((result, index) => {
const runId = runsToFetch[index]?.run_id;
if (!runId) {
return;
}
if (result.status === "fulfilled" && result.value.summary) {
const summary = result.value.summary;
const previouslyTerminal = isTerminalRunState(runsToFetch[index]?.state ?? summary.state);
next[runId] = summary;
storageOperationsStore.updateRunStatus(runId, {
state: summary.state,
total_count: summary.total_count,
succeeded_count: summary.succeeded_count,
failed_count: summary.failed_count,
skipped_count: summary.skipped_count,
total_bytes_processed: summary.total_bytes_processed,
update_time: summary.update_time,
});
if (!previouslyTerminal && isTerminalRunState(summary.state)) {
hasNewlyTerminalRun = true;
}
}
});
runSummariesByRunId.value = next;
if (hasNewlyTerminalRun) {
void userStore.refreshUser(false);
quotaUsageStore.requestRefreshDebounced(0);
}
};
const { startWatchingResource, stopWatchingResource, isWatchingResource } = useResourceWatcher(
watchStorageOperationRuns,
{
shortPollingInterval: pollInterval,
enableBackgroundPolling,
},
);
watcher = {
runSummariesByRunId,
isPolling: readonly(isWatchingResource),
startPolling: startWatchingResource,
stopPolling: stopWatchingResource,
activePollers: 0,
};
sharedHistoryWatchers.set(historyId, watcher);
}
const sharedWatcher = watcher;
let hasStartedPolling = false;
function startPolling() {
if (hasStartedPolling) {
return;
}
hasStartedPolling = true;
sharedWatcher.activePollers += 1;
if (sharedWatcher.activePollers === 1) {
sharedWatcher.startPolling();
}
}
function stopPolling() {
if (!hasStartedPolling) {
return;
}
hasStartedPolling = false;
sharedWatcher.activePollers = Math.max(0, sharedWatcher.activePollers - 1);
if (sharedWatcher.activePollers === 0) {
sharedWatcher.stopPolling();
}
}
return {
isPolling: sharedWatcher.isPolling,
startPolling,
stopPolling,
runSummariesByRunId: readonly(sharedWatcher.runSummariesByRunId),
};
}
+14
View File
@@ -50,6 +50,8 @@ import HistoryList from "@/components/History/HistoryList.vue";
import HistoryPublished from "@/components/History/HistoryPublished.vue";
import HistoryView from "@/components/History/HistoryView.vue";
import HistoryMultipleView from "@/components/History/Multiple/MultipleView.vue";
import StorageOperationHistoryView from "@/components/History/StorageOperations/StorageOperationHistoryView.vue";
import StorageOperationRunView from "@/components/History/StorageOperations/StorageOperationRunView.vue";
import WorkflowExtractionForm from "@/components/History/WorkflowExtractionForm.vue";
import HistoryImport from "@/components/HistoryImport.vue";
import ZipImportResults from "@/components/ImportData/zip/ZipImportResults.vue";
@@ -424,6 +426,18 @@ export function getRouter(Galaxy) {
component: HistoryInvocations,
props: true,
},
{
path: "histories/:historyId/storage/runs",
component: StorageOperationHistoryView,
props: true,
redirect: redirectAnon(),
},
{
path: "histories/:historyId/storage/runs/:runId",
component: StorageOperationRunView,
props: true,
redirect: redirectAnon(),
},
{
path: "histories/:historyId/extract_workflow",
component: WorkflowExtractionForm,
@@ -0,0 +1,117 @@
import { createPinia, setActivePinia } from "pinia";
import { beforeEach, describe, expect, it, vi } from "vitest";
import { isTerminalRunState } from "@/utils/storageOperations";
import { useStorageOperationsStore } from "./storageOperationsStore";
function createTestRun(runId: string, historyId: string, startedAt?: Date | string) {
const timestamp = startedAt instanceof Date ? startedAt.toISOString() : startedAt || new Date().toISOString();
return {
historyId,
runUrl: `/histories/${historyId}/storage/runs/${runId}`,
run_id: runId,
state: "pending" as const,
mode: "move" as const,
target_object_store_id: "other",
create_time: timestamp,
update_time: timestamp,
total_count: 1,
succeeded_count: 0,
failed_count: 0,
skipped_count: 0,
total_bytes_processed: 0,
};
}
describe("storageOperationsStore", () => {
beforeEach(() => {
setActivePinia(createPinia());
localStorage.clear();
});
it("tracks and clears runs", () => {
const store = useStorageOperationsStore();
store.startRun({
...createTestRun("run_1", "history_1"),
total_count: 2,
});
expect(store.getRuns("history_1")).toHaveLength(1);
expect(store.getActiveRuns("history_1")).toHaveLength(1);
expect(store.getCompletedRuns("history_1")).toHaveLength(0);
expect(store.getActiveRunCount("history_1")).toBe(1);
store.clearRun("run_1");
expect(store.getRuns("history_1")).toHaveLength(0);
});
it("drops expired runs when listing", () => {
vi.useFakeTimers();
const store = useStorageOperationsStore();
const now = new Date("2026-01-01T00:00:00Z");
vi.setSystemTime(now);
store.startRun(createTestRun("fresh_run", "history_1", now));
store.startRun(createTestRun("expired_run", "history_1", new Date(now.getTime() - 26 * 60 * 60 * 1000)));
expect(store.getRuns("history_1")).toHaveLength(1);
expect(store.getRuns("history_1")[0]?.run_id).toBe("fresh_run");
expect(store.getActiveRuns("history_1")).toHaveLength(1);
vi.useRealTimers();
});
it("distinguishes active and completed runs", () => {
vi.useFakeTimers();
const store = useStorageOperationsStore();
const now = new Date("2026-01-01T00:00:00Z");
vi.setSystemTime(now);
// Start a run
store.startRun({
...createTestRun("run_1", "history_1", now),
total_count: 10,
});
expect(store.getActiveRunCount("history_1")).toBe(1);
expect(store.getCompletedRunCount("history_1")).toBe(0);
// Mark as completed
store.updateRunStatus("run_1", {
state: "completed",
succeeded_count: 10,
failed_count: 0,
skipped_count: 0,
total_bytes_processed: 123,
update_time: new Date(now.getTime() + 1000).toISOString(),
});
expect(store.getActiveRunCount("history_1")).toBe(0);
expect(store.getCompletedRunCount("history_1")).toBe(1);
const completedRun = store.getCompletedRuns("history_1")[0];
if (completedRun) {
expect(isTerminalRunState(completedRun.state)).toBe(true);
expect(completedRun.state).toBe("completed");
expect(completedRun.succeeded_count).toBe(10);
expect(completedRun.total_bytes_processed).toBe(123);
}
vi.useRealTimers();
});
it("startRun creates a new run", () => {
const store = useStorageOperationsStore();
store.startRun({
...createTestRun("run_2", "history_2"),
total_count: 3,
});
expect(store.getRuns("history_2")).toHaveLength(1);
expect(store.getRuns("history_2")[0]?.run_id).toBe("run_2");
});
});
+103
View File
@@ -0,0 +1,103 @@
import { defineStore } from "pinia";
import { useUserLocalStorage } from "@/composables/userLocalStorage";
import { isTerminalRunState, type StorageOperationRunSummary, type TrackedStorageRun } from "@/utils/storageOperations";
const ACTIVE_RUN_TTL_MS = 24 * 60 * 60 * 1000;
export type StorageRun = TrackedStorageRun;
export type RunStatusUpdate = {
state: StorageOperationRunSummary["state"];
succeeded_count?: StorageOperationRunSummary["succeeded_count"];
failed_count?: StorageOperationRunSummary["failed_count"];
skipped_count?: StorageOperationRunSummary["skipped_count"];
total_count?: StorageOperationRunSummary["total_count"];
total_bytes_processed?: StorageOperationRunSummary["total_bytes_processed"];
update_time?: StorageOperationRunSummary["update_time"];
};
export const useStorageOperationsStore = defineStore("storageOperationsStore", () => {
const runs = useUserLocalStorage<StorageRun[]>("storage-operations-active-runs", []);
function isExpired(run: StorageRun): boolean {
const startedAtRaw = run.create_time || run.update_time;
if (!startedAtRaw) {
return false;
}
const startedAt = Date.parse(startedAtRaw);
if (!Number.isFinite(startedAt)) {
return false;
}
return Date.now() - startedAt > ACTIVE_RUN_TTL_MS;
}
function pruneExpiredRuns() {
runs.value = runs.value.filter((run) => !isExpired(run));
}
function startRun(run: StorageRun): void {
pruneExpiredRuns();
// Remove any existing run with the same ID
runs.value = runs.value.filter((r) => r.run_id !== run.run_id);
runs.value.push(run);
}
function updateRunStatus(runId: string, update: RunStatusUpdate): void {
pruneExpiredRuns();
runs.value = runs.value.map((run) => {
if (run.run_id !== runId) {
return run;
}
return {
...run,
...update,
};
});
}
function clearRun(runId: string): void {
runs.value = runs.value.filter((run) => run.run_id !== runId);
}
function getRuns(historyId: string): StorageRun[] {
return runs.value.filter((run) => run.historyId === historyId && !isExpired(run));
}
function getActiveRuns(historyId: string): StorageRun[] {
return runs.value.filter(
(run) => run.historyId === historyId && !isTerminalRunState(run.state) && !isExpired(run),
);
}
function getCompletedRuns(historyId: string): StorageRun[] {
return runs.value.filter(
(run) => run.historyId === historyId && isTerminalRunState(run.state) && !isExpired(run),
);
}
function getActiveRunCount(historyId: string): number {
return getActiveRuns(historyId).length;
}
function getCompletedRunCount(historyId: string): number {
return getCompletedRuns(historyId).length;
}
pruneExpiredRuns();
return {
runs,
startRun,
updateRunStatus,
getRuns,
getActiveRuns,
getCompletedRuns,
getActiveRunCount,
getCompletedRunCount,
clearRun,
};
});
+10
View File
@@ -10,14 +10,24 @@ export const useUserFlagsStore = defineStore("userFlagsStore", () => {
"user-flags-store-preferred-form-select",
"none" as PreferredFormSelect,
);
const showStorageOperationsHelperPopover = useUserLocalStorage(
"user-flags-store-show-storage-operations-helper-popover",
true,
);
function ignoreSelectionQueryBreakWarning() {
showSelectionQueryBreakWarning.value = false;
}
function ignoreStorageOperationsHelperPopover() {
showStorageOperationsHelperPopover.value = false;
}
return {
showSelectionQueryBreakWarning,
ignoreSelectionQueryBreakWarning,
preferredFormSelectElement,
showStorageOperationsHelperPopover,
ignoreStorageOperationsHelperPopover,
};
});
+31
View File
@@ -97,6 +97,37 @@
}
}
/* Optional modifier for contexts that need softer option hover/selection contrast. */
.multiselect.multiselect--soft-option-highlight {
.multiselect__option--highlight {
background: $brand-secondary;
color: $text-color;
&::after {
background: unset;
color: inherit;
font-weight: normal;
}
}
.multiselect__option--selected {
background: lighten($brand-secondary, 3%);
color: $text-color;
&::after {
color: inherit;
}
&.multiselect__option--highlight {
background: darken($brand-secondary, 3%);
&::after {
background: unset;
}
}
}
}
.multiselect--active {
z-index: 1500; // Should be on top of everything
}
+108
View File
@@ -0,0 +1,108 @@
import type { components } from "@/api/schema";
export type StorageOperationRunState = components["schemas"]["StorageOperationRunState"];
export type StorageOperationRunSummary = components["schemas"]["StorageOperationRunSummary"];
export type DatasetStorageOperationFailureReasonCode =
components["schemas"]["DatasetStorageOperationFailureReasonCode"];
export type TrackedStorageRun = StorageOperationRunSummary & {
historyId: string;
runUrl: string;
};
export interface IneligibleReasonDescription {
label: string;
description: string;
}
/**
* Mapping of storage operation reason codes to user-friendly labels and descriptions.
* These correspond to the DatasetStorageOperationFailureReasonCode enum defined in the backend API schema.
*/
const REASON_CODE_DESCRIPTIONS: Record<DatasetStorageOperationFailureReasonCode, IneligibleReasonDescription> = {
dataset_not_found: {
label: "Dataset not found",
description: "The dataset could not be found at the time of the operation.",
},
invalid_target_object_store: {
label: "Invalid storage location",
description: "The selected storage location is not available or is invalid.",
},
missing_source_object_store: {
label: "Missing source location",
description: "The dataset does not have a source storage location defined.",
},
already_in_target: {
label: "Already in target location",
description: "The dataset is already stored in the selected location.",
},
target_quota_exceeded: {
label: "Target quota exceeded",
description: "The operation would exceed the target storage quota.",
},
shared_dataset: {
label: "Shared dataset",
description: "This dataset is referenced by histories you do not own and cannot be moved.",
},
insufficient_permissions: {
label: "Insufficient permissions",
description: "You do not have permission to move this dataset to a different storage location.",
},
dataset_in_use: {
label: "Dataset in use",
description: "The dataset is currently being used by an active job and cannot be moved.",
},
target_expiration_imminent: {
label: "Would expire soon in target location",
description:
"Moving this dataset would leave less than 10 days before expiration in the selected target location.",
},
checksum_verification_failed: {
label: "Checksum verification failed",
description:
"The data transfer could not be verified: the source and target checksums do not match. The original data is unchanged.",
},
execution_error: {
label: "Execution error",
description: "An error occurred while attempting to move the dataset.",
},
};
/**
* Get a user-friendly description for a storage operation reason code.
* @param reasonCode - The reason code from the backend
* @returns An object with a human-readable label and description
*/
export function getIneligibleReasonDescription(reasonCode: string): IneligibleReasonDescription {
return (
REASON_CODE_DESCRIPTIONS[reasonCode as DatasetStorageOperationFailureReasonCode] || {
label: "Unknown reason",
description: `Reason code: ${reasonCode}`,
}
);
}
/**
* Check if a storage operation run state is a terminal state (no longer running).
* @param state - The run state (e.g., "pending", "running", "completed", "failed")
* @returns true if the state is terminal (completed or failed)
*/
export function isTerminalRunState(state?: StorageOperationRunState): boolean {
return state === "completed" || state === "failed";
}
export function toTrackedStorageRun(historyId: string, run: StorageOperationRunSummary): TrackedStorageRun {
const createTime = run.create_time || new Date().toISOString();
const updateTime = run.update_time || createTime;
return {
...run,
create_time: createTime,
update_time: updateTime,
historyId,
runUrl: `/histories/${historyId}/storage/runs/${run.run_id}`,
};
}
export function getCompletedAtForRun(run: StorageOperationRunSummary): string | undefined {
return isTerminalRunState(run.state) ? run.update_time : undefined;
}
+50
View File
@@ -744,6 +744,56 @@
:Type: int
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
``bulk_storage_operation_dataset_minimum_days_to_expiration``
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
:Description:
Minimum remaining lifetime, in days, required before a dataset can
be moved into an object store with automatic expiration. This
prevents moving data into a target store where it would be close
to expiring immediately. Defaults to 7 days.
:Default: ``7``
:Type: int
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
``bulk_storage_operation_completed_run_retention_days``
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
:Description:
How long completed bulk dataset storage operation runs are kept in
the database before Galaxy prunes their run records. Snapshot
cleanup is handled separately. Defaults to 30 days.
:Default: ``30``
:Type: int
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
``prune_expired_bulk_storage_operations_interval``
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
:Description:
How many seconds between attempts to prune expired bulk dataset
storage operation snapshots and old completed run records. Set to
0 to disable the periodic pruning task. Defaults to every 24
hours.
:Default: ``86400``
:Type: int
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
``recover_stale_bulk_storage_operation_runs_interval``
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
:Description:
How many seconds between attempts to recover bulk dataset storage
operation runs that were left pending or running after a worker
stopped updating them. Defaults to every hour.
:Default: ``3600``
:Type: int
~~~~~~~~~~~~~~~~~~~~~~~~~~~~
``file_sources_config_file``
~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+8
View File
@@ -41,6 +41,14 @@ galaxy.schema.schema module
:undoc-members:
:show-inheritance:
galaxy.schema.storage\_operations module
----------------------------------------
.. automodule:: galaxy.schema.storage_operations
:members:
:undoc-members:
:show-inheritance:
galaxy.schema.tasks module
--------------------------
+4
View File
@@ -260,8 +260,12 @@ def setup_periodic_tasks(config, celery_app):
beat_schedule: dict[str, dict[str, Any]] = {}
schedule_task("prune_history_audit_table", config.history_audit_table_prune_interval)
schedule_task("prune_expired_bulk_storage_operations", config.prune_expired_bulk_storage_operations_interval)
schedule_task("cleanup_short_term_storage", config.short_term_storage_cleanup_interval)
schedule_task("prune_kombu_sqla_transport", config.kombu_sqla_transport_cleanup_interval)
schedule_task(
"recover_stale_bulk_storage_operation_runs", config.recover_stale_bulk_storage_operation_runs_interval
)
if config.statsd_host:
schedule_task("emit_queue_metrics_task", config.queue_metrics_interval)
+157
View File
@@ -11,6 +11,7 @@ from typing import (
)
from urllib.parse import urlparse
from celery import current_task
from lagom.exceptions import UnresolvableType
from sqlalchemy import (
and_,
@@ -18,6 +19,7 @@ from sqlalchemy import (
delete,
exists,
false,
or_,
select,
text,
update,
@@ -34,6 +36,7 @@ from galaxy.datatypes.registry import Registry as DatatypesRegistry
from galaxy.exceptions import ObjectNotFound
from galaxy.jobs import MinimalJobWrapper
from galaxy.managers.collections import DatasetCollectionManager
from galaxy.managers.dataset_storage_operations import DatasetStorageOperationManager
from galaxy.managers.datasets import (
DatasetAssociationManager,
DatasetManager,
@@ -50,6 +53,8 @@ from galaxy.managers.tool_data import ToolDataImportManager
from galaxy.managers.workflow_completion import WorkflowCompletionManager
from galaxy.metadata.set_metadata import set_metadata_portable
from galaxy.model import (
DatasetStorageOperationRun,
DatasetStorageOperationSnapshot,
Job,
User,
)
@@ -58,6 +63,7 @@ from galaxy.objectstore import BaseObjectStore
from galaxy.objectstore.caching import check_caches
from galaxy.queue_worker import GalaxyQueueWorker
from galaxy.schema.notifications import NotificationCreateRequest
from galaxy.schema.storage_operations import StorageOperationRunState
from galaxy.schema.tasks import (
ComputeDatasetHashTaskRequest,
GenerateHistoryContentDownload,
@@ -308,6 +314,83 @@ def set_metadata(
sa_session.commit()
@galaxy_task(action="execute bulk storage move run")
def bulk_move_storage(
sa_session: galaxy_scoped_session,
dataset_manager: DatasetManager,
notification_manager: NotificationManager,
app: MinimalManagerApp,
run_db_id: int,
task_user_id: int,
notify_on_completion: Optional[bool] = None,
):
run = sa_session.get(DatasetStorageOperationRun, run_db_id)
if run is None:
log.error(f"Bulk storage move run not found: {run_db_id}")
return
current_task_id = getattr(getattr(current_task, "request", None), "id", None)
if current_task_id is not None:
if run.task_id is None:
# Claim ownership for the first worker execution in case run.task_id was not yet persisted.
run.task_id = str(current_task_id)
sa_session.add(run)
sa_session.commit()
elif str(run.task_id) != str(current_task_id):
log.info(
"Skipping bulk storage run %s because task ownership moved from %s to %s",
run.id,
current_task_id,
run.task_id,
)
return
if run.state in (StorageOperationRunState.completed.value, StorageOperationRunState.failed.value):
log.info("Skipping bulk storage run %s because it is already terminal (%s)", run.id, run.state)
return
user = sa_session.get(User, task_user_id)
if user is None:
log.error("Bulk storage run %s user %s not found; marking run as failed", run.id, task_user_id)
run.state = StorageOperationRunState.failed.value
run.failed_count = run.total_count or 0
sa_session.add(run)
sa_session.commit()
return
snapshot = sa_session.get(DatasetStorageOperationSnapshot, run.snapshot_id)
storage_operation_manager = DatasetStorageOperationManager(app.object_store, app.config)
executor = storage_operation_manager.create_run_executor(
sa_session=sa_session,
dataset_manager=dataset_manager,
app=app,
run=run,
user=user,
current_task_id=current_task_id,
)
execution_result = executor.execute_run(snapshot)
if execution_result.state not in (StorageOperationRunState.completed, StorageOperationRunState.failed):
return
run_notify_on_completion = notify_on_completion
if run_notify_on_completion is None:
run_notify_on_completion = run.notify_on_completion
if not run_notify_on_completion:
return
try:
notification_manager.send_storage_operation_notification(
user_id=user.id,
run=run,
execution_result=execution_result,
encode_id=app.security.encode_id,
)
except Exception:
log.exception("Failed to send storage operation notification for run %s", run.id)
def _get_dataset_manager(
hda_manager: HDAManager, ldda_manager: LDDAManager, model_class: str = "HistoryDatasetAssociation"
) -> DatasetAssociationManager:
@@ -646,6 +729,23 @@ def cleanup_expired_notifications(notification_manager: NotificationManager):
)
@galaxy_task(action="prune expired bulk storage operations")
def prune_expired_bulk_storage_operations(
sa_session: galaxy_scoped_session,
object_store: BaseObjectStore,
config: GalaxyAppConfiguration,
):
storage_operation_manager = DatasetStorageOperationManager(object_store, config)
deleted_run_count = storage_operation_manager.prune_completed_runs(sa_session)
deleted_snapshot_count = storage_operation_manager.prune_expired_snapshots(sa_session)
if deleted_run_count or deleted_snapshot_count:
log.info(
"Pruned %s completed storage operation runs and %s expired storage operation snapshots",
deleted_run_count,
deleted_snapshot_count,
)
@galaxy_task(action="prune object store cache directories")
def clean_object_store_caches(object_store: BaseObjectStore):
check_caches(object_store.cache_targets())
@@ -803,3 +903,60 @@ def cleanup_stale_concurrency_slots(
)
session.commit()
log.info(f"Cleaned up {len(stale_task_ids)} stale concurrency tracking rows")
@galaxy_task(action="recover stale storage operation runs")
def recover_stale_bulk_storage_operation_runs(
session: galaxy_scoped_session,
stale_threshold_minutes: int = 30,
):
threshold = datetime.datetime.now() - datetime.timedelta(minutes=stale_threshold_minutes)
stale_runs = (
session.execute(
select(DatasetStorageOperationRun).where(
or_(
DatasetStorageOperationRun.state == StorageOperationRunState.pending.value,
DatasetStorageOperationRun.state == StorageOperationRunState.running.value,
),
DatasetStorageOperationRun.update_time < threshold,
)
)
.scalars()
.all()
)
if not stale_runs:
return
try:
active_response = celery_app.control.inspect().active() or {}
except Exception:
log.warning("Failed to inspect active tasks on workers; skipping stale storage operation recovery")
return
active_task_ids = {task["id"] for tasks in active_response.values() for task in tasks}
recovered_count = 0
for run in stale_runs:
run_task_id = str(run.task_id) if run.task_id is not None else None
if run_task_id is not None and run_task_id in active_task_ids:
continue
try:
task_result = bulk_move_storage.delay(
run_db_id=run.id,
task_user_id=run.user_id,
notify_on_completion=run.notify_on_completion,
)
run.state = StorageOperationRunState.pending.value
run.task_id = str(task_result.id)
session.add(run)
session.commit()
recovered_count += 1
except Exception:
session.rollback()
log.exception("Failed to redispatch stale storage operation run %s", run.id)
if recovered_count:
log.info("Redispatched %s stale storage operation runs", recovered_count)
@@ -742,6 +742,27 @@ galaxy:
# cleaned up in default Celery task configuration.
#short_term_storage_cleanup_interval: 3600
# Minimum remaining lifetime, in days, required before a dataset can
# be moved into an object store with automatic expiration. This
# prevents moving data into a target store where it would be close to
# expiring immediately. Defaults to 7 days.
#bulk_storage_operation_dataset_minimum_days_to_expiration: 7
# How long completed bulk dataset storage operation runs are kept in
# the database before Galaxy prunes their run records. Snapshot
# cleanup is handled separately. Defaults to 30 days.
#bulk_storage_operation_completed_run_retention_days: 30
# How many seconds between attempts to prune expired bulk dataset
# storage operation snapshots and old completed run records. Set to 0
# to disable the periodic pruning task. Defaults to every 24 hours.
#prune_expired_bulk_storage_operations_interval: 86400
# How many seconds between attempts to recover bulk dataset storage
# operation runs that were left pending or running after a worker
# stopped updating them. Defaults to every hour.
#recover_stale_bulk_storage_operation_runs_interval: 3600
# Configured FileSource plugins.
# The value of this option will be resolved with respect to
# <config_dir>.
@@ -548,6 +548,41 @@ mapping:
How many seconds between instances of short term storage being cleaned up in default
Celery task configuration.
bulk_storage_operation_dataset_minimum_days_to_expiration:
type: int
required: false
default: 7
desc: |
Minimum remaining lifetime, in days, required before a dataset can be moved into an
object store with automatic expiration. This prevents moving data into a target store
where it would be close to expiring immediately. Defaults to 7 days.
bulk_storage_operation_completed_run_retention_days:
type: int
required: false
default: 30
desc: |
How long completed bulk dataset storage operation runs are kept in the database before
Galaxy prunes their run records. Snapshot cleanup is handled separately. Defaults to 30 days.
prune_expired_bulk_storage_operations_interval:
type: int
required: false
default: 86400
desc: |
How many seconds between attempts to prune expired bulk dataset storage operation
snapshots and old completed run records. Set to 0 to disable the periodic pruning task.
Defaults to every 24 hours.
recover_stale_bulk_storage_operation_runs_interval:
type: int
required: false
default: 3600
desc: |
How many seconds between attempts to recover bulk dataset storage operation runs that
were left pending or running after a worker stopped updating them.
Defaults to every hour.
file_sources_config_file:
type: str
default: file_sources_conf.yml
@@ -0,0 +1,267 @@
Use this template to customize the HTML email your users will receive when a new notification of category
"storage_operation" is sent to them. Copy the file to {{ templates_dir
}}/mail/notifications/storage_operation-email.html and modify as required. If you are adding URLs, remember that only
absolute URLs (with a domain name) make sense in email! They can be served from any stable location, including your
Galaxy server or GitHub. The following variables are available for inserting into the HTML with Jinja2 syntax, like {{
variable_name }}. They will be rendered into the text before the email is sent: - name The user's name - user_email The
user's email - date Date and time of the notification - hostname Your galaxy's hostname (i.e. usegalaxy.* or the value
in `server_name` from the galaxy config file) - contact_email Your galaxy's contact email - notification_settings_url
The URL to the user's notification settings to manage their subscriptions - variant The notification variant indicates
the level of importance of the notification (i.e. info, warning, urgent) - content The storage operation payload -
subject Notification subject - message Notification body - run_url Link to the run status page - mode Storage operation
mode - state Storage operation state - total_count Total datasets in run - succeeded_count Successful datasets count -
failed_count Failed datasets count - skipped_count Skipped datasets count - galaxy_url The URL to the Galaxy instance
(i.e. https://usegalaxy.*) Template begins here >>>>>>
<!doctype html>
<html lang="en" dir="ltr">
<head>
<meta charset="utf-8" />
<title>[Galaxy] {{ content['subject'] }}</title>
<link href="https://fonts.googleapis.com/css2?family=Roboto&display=swap" rel="stylesheet" />
</head>
<body
style="
margin: 0;
padding: 0;
background-color: #f6f8fb;
font-family: &quot;Roboto&quot;, sans-serif;
color: #24344d;
">
{% set total_count = content['total_count'] or 0 %} {% if total_count > 0 %} {% set succeeded_pct =
((content['succeeded_count'] or 0) * 100 / total_count) | round(1) %} {% set failed_pct =
((content['failed_count'] or 0) * 100 / total_count) | round(1) %} {% set skipped_pct =
((content['skipped_count'] or 0) * 100 / total_count) | round(1) %} {% else %} {% set succeeded_pct = 0 %} {%
set failed_pct = 0 %} {% set skipped_pct = 0 %} {% endif %}
<table
role="presentation"
cellpadding="0"
cellspacing="0"
width="100%"
style="background-color: #f6f8fb; padding: 24px 12px">
<tr>
<td align="center">
<table
role="presentation"
cellpadding="0"
cellspacing="0"
width="100%"
style="
max-width: 680px;
background-color: #ffffff;
border: 1px solid #dbe3f0;
border-radius: 8px;
overflow: hidden;
">
<tr>
<td
style="
background: linear-gradient(90deg, #2c5aa0 0%, #315fbe 100%);
padding: 16px 24px;
color: #ffffff;
">
<p style="margin: 0; font-size: 18px; font-weight: 700">
Galaxy storage operation update
</p>
<p style="margin: 6px 0 0 0; font-size: 12px; opacity: 0.92">
{{ content['subject'] }}
</p>
</td>
</tr>
<tr>
<td style="padding: 24px">
<p style="margin: 0 0 16px 0; font-size: 16px">Hello {{ name }},</p>
<p style="margin: 0 0 20px 0; font-size: 12pt; line-height: 1.5">
{{ content['message'] }}
</p>
<table
role="presentation"
cellpadding="0"
cellspacing="0"
width="100%"
style="
border: 1px solid #dbe3f0;
border-radius: 6px;
background-color: #fbfcfe;
margin: 0 0 20px 0;
">
<tr>
<td
style="
padding: 14px 16px;
border-bottom: 1px solid #e5ebf5;
font-size: 13px;
font-weight: 700;
color: #223a63;
">
Run summary
</td>
</tr>
<tr>
<td style="padding: 14px 16px">
<p style="margin: 0 0 8px 0; font-size: 12pt">
Mode: <strong>{{ content['mode'] }}</strong><br />
State: <strong>{{ content['state'] }}</strong>
</p>
<p style="margin: 0; font-size: 11pt; line-height: 1.6">
Total datasets: {{ content['total_count'] }}<br />
Succeeded:
<span style="color: #1f7a38">{{ content['succeeded_count'] }}</span> ({{
succeeded_pct }}%)<br />
Failed:
<span style="color: #b54747">{{ content['failed_count'] }}</span> ({{
failed_pct }}%)<br />
Skipped:
<span style="color: #8a6f16">{{ content['skipped_count'] }}</span> ({{
skipped_pct }}%)
</p>
<table
role="presentation"
cellpadding="0"
cellspacing="0"
width="100%"
style="margin-top: 12px; border-collapse: separate">
<tr>
{% if total_count > 0 %}
<td
width="{{ succeeded_pct }}%"
style="
height: 12px;
background-color: #2e8748;
font-size: 0;
line-height: 0;
">
&nbsp;
</td>
<td
width="{{ skipped_pct }}%"
style="
height: 12px;
background-color: #d7a100;
font-size: 0;
line-height: 0;
">
&nbsp;
</td>
<td
width="{{ failed_pct }}%"
style="
height: 12px;
background-color: #cc4b4b;
font-size: 0;
line-height: 0;
">
&nbsp;
</td>
{% else %}
<td
style="
width: 100%;
height: 12px;
background-color: #d9e2f2;
font-size: 0;
line-height: 0;
">
&nbsp;
</td>
{% endif %}
</tr>
</table>
</td>
</tr>
</table>
{% if content['failed_count'] and content['failed_count'] > 0 %}
<p
style="
margin: 0 0 20px 0;
font-size: 11pt;
line-height: 1.5;
color: #7c2d2d;
background-color: #fff1f1;
border: 1px solid #f1caca;
border-radius: 6px;
padding: 12px 14px;
">
Some datasets failed during this run. Open the run details to review failure
reasons.
</p>
{% elif content['succeeded_count'] and content['total_count'] and
content['succeeded_count'] == content['total_count'] %}
<p
style="
margin: 0 0 20px 0;
font-size: 11pt;
line-height: 1.5;
color: #195b2e;
background-color: #effaf2;
border: 1px solid #c9e8d1;
border-radius: 6px;
padding: 12px 14px;
">
All datasets were processed successfully.
</p>
{% endif %}
<p style="margin: 0 0 20px 0">
<a
href="{{ content['run_url'] }}"
style="
display: inline-block;
background-color: #2c5aa0;
color: #ffffff;
text-decoration: none;
font-size: 12pt;
font-weight: 700;
border-radius: 6px;
padding: 10px 16px;
">
Open run status
</a>
</p>
<p style="margin: 0 0 14px 0; font-size: 12pt">Thank you for using Galaxy.</p>
<p style="margin: 0; font-size: 12pt; line-height: 1.5">
Regards,<br />
Your Galaxy Team at
<a href="{{ galaxy_url }}" style="color: #2c5aa0">{{ hostname }}</a>
</p>
</td>
</tr>
<tr>
<td
style="
padding: 16px 24px;
border-top: 1px solid #e5ebf5;
background-color: #f8faff;
font-size: 10pt;
line-height: 1.45;
color: #4f5f79;
">
You received this email because you are subscribed to storage operation notifications.
{% if notification_settings_url %} You can manage your notification settings
<a href="{{ notification_settings_url }}" style="color: #2c5aa0">here</a>. {% endif %}
<br />
{% if contact_email %} This is an automated email. Please do not reply directly; contact
us at
<a href="mailto:{{ contact_email }}" style="color: #2c5aa0">{{ contact_email }}</a>. {%
endif %}
</td>
</tr>
<tr>
<td style="padding: 16px 24px 24px 24px; background-color: #f8faff">
<img
style="width: 130px; height: auto; display: block"
src="https://galaxyproject.org/images/galaxy-logos/galaxy_project_logo_square.png"
alt="Galaxy project logo" />
</td>
</tr>
</table>
</td>
</tr>
</table>
</body>
</html>
@@ -0,0 +1,78 @@
Use this template to customize the text email your users will receive
when a new notification of category "storage_operation" is sent to them.
Copy the file to {{ templates_dir }}/mail/notifications/storage_operation-email.txt and modify as required.
If you are adding URLs, remember that only absolute URLs (with
a domain name) make sense in email! They can be served from any stable
location, including your Galaxy server or GitHub.
The following variables are available for inserting into the text like
{{ variable_name }}. They will be rendered into the text before the email is
sent:
- name The user's name
- user_email The user's email
- date Date and time of the notification
- hostname Your galaxy's hostname (i.e. usegalaxy.* or the value in `server_name` from the galaxy config file)
- contact_email Your galaxy's contact email
- notification_settings_url The URL to the user's notification settings to manage their subscriptions
- variant The notification variant indicates the level of importance of the notification (i.e. info, warning, urgent)
- content The storage operation payload
- subject Notification subject
- message Notification body in text
- run_url Link to the run status page
- mode Storage operation mode
- state Storage operation state
- total_count Total datasets in run
- succeeded_count Successful datasets count
- failed_count Failed datasets count
- skipped_count Skipped datasets count
- galaxy_url The URL to the Galaxy instance (i.e. https://usegalaxy.*)
Template begins here>>>>>>
Hello {{ name }},
{{ content['subject'] }}
{{ content['message'] }}
Storage operation summary:
- Mode: {{ content['mode'] }}
- Current state: {{ content['state'] }}
- Total datasets: {{ content['total_count'] }}
- Succeeded: {{ content['succeeded_count'] }}
- Failed: {{ content['failed_count'] }}
- Skipped: {{ content['skipped_count'] }}
{% if content['total_count'] and content['total_count'] > 0 %}
- Success rate: {{ ((content['succeeded_count'] * 100) / content['total_count']) | round(1) }}%
- Failed rate: {{ ((content['failed_count'] * 100) / content['total_count']) | round(1) }}%
- Skipped rate: {{ ((content['skipped_count'] * 100) / content['total_count']) | round(1) }}%
{% endif %}
Run details:
{% if content['failed_count'] and content['failed_count'] > 0 %}
Some datasets failed during this run. Please review the run details for the failure reasons.
{% elif content['succeeded_count'] and content['total_count'] and content['succeeded_count'] == content['total_count'] %}
All datasets were processed successfully.
{% else %}
This run has completed with mixed results. Please review the run details.
{% endif %}
To view the live or final status of this run, open:
{{ content['run_url'] }}
Thank you for using Galaxy!
Regards,
Your Galaxy Team at {{ hostname }}
---
You received this email because you are subscribed to receive storage operation notifications.
{% if notification_settings_url %}
To manage your notification settings, please visit {{ notification_settings_url }}.
{% endif %}
{% if contact_email %}
This is an automated email. If you have any questions or concerns, please do not reply to this email, instead, contact us at {{ contact_email }}.
{% endif %}
File diff suppressed because it is too large Load Diff
+76
View File
@@ -1,4 +1,5 @@
import logging
from collections.abc import Callable
from datetime import datetime
from enum import Enum
from typing import (
@@ -41,6 +42,7 @@ from galaxy.exceptions import (
from galaxy.managers.markdown_util import to_html
from galaxy.managers.sse_dispatch import SSEEventDispatcher
from galaxy.model import (
DatasetStorageOperationRun,
GroupRoleAssociation,
Notification,
User,
@@ -66,11 +68,16 @@ from galaxy.schema.notifications import (
NotificationResponse,
NotificationVariant,
PersonalNotificationCategory,
StorageOperationNotificationContent,
UpdateUserNotificationPreferencesRequest,
UserNotificationPreferences,
UserNotificationUpdateRequest,
)
from galaxy.schema.schema import AsyncTaskResultSummary
from galaxy.schema.storage_operations import (
StorageOperationExecutionResult,
StorageOperationRunState,
)
from galaxy.util import now
log = logging.getLogger(__name__)
@@ -184,6 +191,57 @@ class NotificationManager:
return notification, notifications_sent
def send_storage_operation_notification(
self,
user_id: int,
run: DatasetStorageOperationRun,
execution_result: StorageOperationExecutionResult,
encode_id: Callable[[int], str],
) -> tuple[Optional[Notification], int]:
"""Create and send a storage operation notification to a single user."""
encoded_history_id = encode_id(run.history_id)
encoded_run_id = encode_id(run.id)
relative_run_url = f"/histories/{encoded_history_id}/storage/runs/{encoded_run_id}"
galaxy_url = getattr(self.config, "galaxy_infrastructure_url", None)
run_url = f"{galaxy_url}{relative_run_url}" if galaxy_url else relative_run_url
if execution_result.state == StorageOperationRunState.failed:
variant = NotificationVariant.urgent
elif run.failed_count > 0 or run.skipped_count > 0:
variant = NotificationVariant.warning
else:
variant = NotificationVariant.info
notification_request = NotificationCreateRequest(
recipients=NotificationRecipients.model_construct(user_ids=[user_id]),
notification=NotificationCreateData(
source="galaxy",
category=PersonalNotificationCategory.storage_operation,
variant=variant,
publication_time=None,
expiration_time=None,
content=StorageOperationNotificationContent.model_construct(
subject=(
"Storage operation completed"
if execution_result.state == StorageOperationRunState.completed
else "Storage operation failed"
),
message=execution_result.message,
history_id=encoded_history_id,
run_id=encoded_run_id,
run_url=run_url,
mode=run.mode,
state=execution_result.state,
total_count=run.total_count,
succeeded_count=run.succeeded_count,
failed_count=run.failed_count,
skipped_count=run.skipped_count,
),
),
galaxy_url=galaxy_url,
)
return self.send_notification_to_recipients(notification_request)
def send_notification_internal(
self, request: NotificationCreateRequest, force_sync: bool = False
) -> Union[NotificationCreatedResponse, AsyncTaskResultSummary]:
@@ -810,11 +868,29 @@ class NewSharedItemEmailNotificationTemplateBuilder(EmailNotificationTemplateBui
return f"[Galaxy] New {content.item_type} shared with you: {content.item_name}"
class StorageOperationEmailNotificationTemplateBuilder(EmailNotificationTemplateBuilder):
markdown_to = {
TemplateFormats.HTML: to_html,
TemplateFormats.TXT: lambda x: x,
}
def get_content(self, template_format: TemplateFormats) -> AnyNotificationContent:
content = StorageOperationNotificationContent.model_construct(**self.notification.content) # type: ignore[arg-type]
content.message = self.markdown_to[template_format](content.message)
return content
def get_subject(self) -> str:
content = cast(StorageOperationNotificationContent, self.get_content(TemplateFormats.TXT))
return f"[Galaxy] {content.subject}"
class EmailNotificationChannelPlugin(NotificationChannelPlugin):
# Register the supported email templates here
email_templates_by_category: dict[PersonalNotificationCategory, type[EmailNotificationTemplateBuilder]] = {
PersonalNotificationCategory.message: MessageEmailNotificationTemplateBuilder,
PersonalNotificationCategory.new_shared_item: NewSharedItemEmailNotificationTemplateBuilder,
PersonalNotificationCategory.storage_operation: StorageOperationEmailNotificationTemplateBuilder,
}
def send(self, notification: Notification, user: User):
+59
View File
@@ -12726,6 +12726,65 @@ class CeleryUserActiveTask(Base):
started_at: Mapped[datetime]
class DatasetStorageOperationSnapshot(Base):
"""Immutable snapshot of a resolved storage operation selection."""
__tablename__ = "dataset_storage_operation_snapshot"
id: Mapped[int] = mapped_column(primary_key=True)
history_id: Mapped[int] = mapped_column(ForeignKey("history.id", ondelete="CASCADE"), index=True)
user_id: Mapped[int] = mapped_column(ForeignKey("galaxy_user.id", ondelete="CASCADE"), index=True)
mode: Mapped[str] = mapped_column(String(32), index=True)
target_object_store_id: Mapped[str] = mapped_column(String(255))
resolved_dataset_ids: Mapped[list[int]] = mapped_column(JSONType)
eligible_dataset_ids: Mapped[list[int]] = mapped_column(JSONType)
create_time: Mapped[datetime] = mapped_column(default=now, nullable=True)
update_time: Mapped[datetime] = mapped_column(default=now, onupdate=now, nullable=True)
expires_at: Mapped[datetime] = mapped_column(index=True)
class DatasetStorageOperationRun(Base):
"""Tracks one execution attempt for a storage operation snapshot."""
__tablename__ = "dataset_storage_operation_run"
id: Mapped[int] = mapped_column(primary_key=True)
snapshot_id: Mapped[int] = mapped_column(
ForeignKey("dataset_storage_operation_snapshot.id", ondelete="CASCADE"), index=True
)
history_id: Mapped[int] = mapped_column(ForeignKey("history.id", ondelete="CASCADE"), index=True)
user_id: Mapped[int] = mapped_column(ForeignKey("galaxy_user.id", ondelete="CASCADE"), index=True)
mode: Mapped[str] = mapped_column(String(32), index=True)
target_object_store_id: Mapped[str] = mapped_column(String(255))
state: Mapped[str] = mapped_column(String(32), index=True)
skip_ineligible: Mapped[bool] = mapped_column(Boolean, default=True)
notify_on_completion: Mapped[bool] = mapped_column(Boolean, default=True)
task_id: Mapped[Optional[Union[UUID, str]]] = mapped_column(UUIDType(), index=True)
total_count: Mapped[int] = mapped_column(default=0)
succeeded_count: Mapped[int] = mapped_column(default=0)
failed_count: Mapped[int] = mapped_column(default=0)
skipped_count: Mapped[int] = mapped_column(default=0)
total_bytes_processed: Mapped[int] = mapped_column(default=0)
create_time: Mapped[datetime] = mapped_column(default=now, nullable=True)
update_time: Mapped[datetime] = mapped_column(default=now, onupdate=now, nullable=True)
class DatasetStorageOperationRunItem(Base):
"""Per-dataset status for a storage operation run."""
__tablename__ = "dataset_storage_operation_run_item"
__table_args__ = (UniqueConstraint("run_id", "dataset_id"),)
id: Mapped[int] = mapped_column(primary_key=True)
run_id: Mapped[int] = mapped_column(ForeignKey("dataset_storage_operation_run.id", ondelete="CASCADE"), index=True)
dataset_id: Mapped[int] = mapped_column(ForeignKey("dataset.id", ondelete="CASCADE"), index=True)
state: Mapped[str] = mapped_column(String(32), index=True)
reason_code: Mapped[Optional[str]] = mapped_column(String(64), nullable=True)
bytes_processed: Mapped[int] = mapped_column(default=0)
create_time: Mapped[datetime] = mapped_column(default=now, nullable=True)
update_time: Mapped[datetime] = mapped_column(default=now, onupdate=now, nullable=True)
class UserCredentials(Base):
"""
Represents a credential associated with a user for a specific service.
@@ -0,0 +1,93 @@
"""add storage operation run tables
Revision ID: 6925fe4c8a17
Revises: b75f0f4dbcd4
Create Date: 2026-04-15 16:04:34.330085
"""
from sqlalchemy import (
Boolean,
Column,
DateTime,
ForeignKey,
Integer,
String,
UniqueConstraint,
)
from galaxy.model.custom_types import (
JSONType,
UUIDType,
)
from galaxy.model.migrations.util import (
create_table,
drop_table,
)
# revision identifiers, used by Alembic.
revision = "6925fe4c8a17"
down_revision = "b75f0f4dbcd4"
branch_labels = None
depends_on = None
def upgrade():
create_table(
"dataset_storage_operation_snapshot",
Column("id", Integer, primary_key=True),
Column("history_id", Integer, ForeignKey("history.id", ondelete="CASCADE"), index=True),
Column("user_id", Integer, ForeignKey("galaxy_user.id", ondelete="CASCADE"), index=True, nullable=False),
Column("mode", String(32), index=True, nullable=False),
Column("target_object_store_id", String(255), nullable=False),
Column("resolved_dataset_ids", JSONType, nullable=False),
Column("eligible_dataset_ids", JSONType, nullable=False),
Column("create_time", DateTime, nullable=True),
Column("update_time", DateTime, nullable=True),
Column("expires_at", DateTime, index=True, nullable=False),
)
create_table(
"dataset_storage_operation_run",
Column("id", Integer, primary_key=True),
Column(
"snapshot_id",
Integer,
ForeignKey("dataset_storage_operation_snapshot.id", ondelete="CASCADE"),
index=True,
),
Column("history_id", Integer, ForeignKey("history.id", ondelete="CASCADE"), index=True),
Column("user_id", Integer, ForeignKey("galaxy_user.id", ondelete="CASCADE"), index=True, nullable=False),
Column("mode", String(32), index=True, nullable=False),
Column("target_object_store_id", String(255), nullable=False),
Column("state", String(32), index=True, nullable=False),
Column("skip_ineligible", Boolean, nullable=False, default=True),
Column("notify_on_completion", Boolean, nullable=False, default=True),
Column("task_id", UUIDType(), index=True),
Column("total_count", Integer, nullable=False, default=0),
Column("succeeded_count", Integer, nullable=False, default=0),
Column("failed_count", Integer, nullable=False, default=0),
Column("skipped_count", Integer, nullable=False, default=0),
Column("total_bytes_processed", Integer, nullable=False, default=0),
Column("create_time", DateTime, nullable=True),
Column("update_time", DateTime, nullable=True),
)
create_table(
"dataset_storage_operation_run_item",
Column("id", Integer, primary_key=True),
Column("run_id", Integer, ForeignKey("dataset_storage_operation_run.id", ondelete="CASCADE"), index=True),
Column("dataset_id", Integer, ForeignKey("dataset.id", ondelete="CASCADE"), index=True),
Column("state", String(32), index=True, nullable=False),
Column("reason_code", String(64), nullable=True),
Column("bytes_processed", Integer, nullable=False, default=0),
Column("create_time", DateTime, nullable=True),
Column("update_time", DateTime, nullable=True),
UniqueConstraint("run_id", "dataset_id"),
)
def downgrade():
drop_table("dataset_storage_operation_run_item")
drop_table("dataset_storage_operation_run")
drop_table("dataset_storage_operation_snapshot")
+24
View File
@@ -25,6 +25,7 @@ from galaxy.schema.generics import (
PatchGenericPickle,
)
from galaxy.schema.schema import Model
from galaxy.schema.storage_operations import StorageOperationRunState
from galaxy.schema.types import (
AbsoluteOrRelativeUrl,
OffsetNaiveDatetime,
@@ -61,6 +62,7 @@ class PersonalNotificationCategory(str, Enum):
message = "message"
new_shared_item = "new_shared_item"
storage_operation = "storage_operation"
# TODO: enable this and create content model when we have a hook for completed workflows
# workflow_execution_completed = "workflow_execution_completed"
@@ -115,6 +117,27 @@ class NewSharedItemNotificationContent(Model):
slug: str = Field(..., title="Slug", description="The slug of the shared item. Used for the link to the item.")
class StorageOperationNotificationContent(MessageNotificationContentBase):
category: Literal[PersonalNotificationCategory.storage_operation] = PersonalNotificationCategory.storage_operation
history_id: EncodedDatabaseIdField = Field(..., title="History ID", description="The encoded history ID.")
run_id: EncodedDatabaseIdField = Field(..., title="Run ID", description="The encoded storage operation run ID.")
run_url: AbsoluteOrRelativeUrl = Field(
...,
title="Run URL",
description="Absolute or relative URL to the storage operation run status view.",
)
mode: str = Field(..., title="Mode", description="Storage operation mode.")
state: StorageOperationRunState = Field(
...,
title="State",
description="The current state of the storage operation run when this notification was generated.",
)
total_count: int = Field(..., title="Total Count", description="Total datasets in the run.")
succeeded_count: int = Field(default=0, title="Succeeded Count", description="Succeeded datasets count.")
failed_count: int = Field(default=0, title="Failed Count", description="Failed datasets count.")
skipped_count: int = Field(default=0, title="Skipped Count", description="Skipped datasets count.")
NotificationContentField = Field(
default=...,
discriminator="category",
@@ -126,6 +149,7 @@ AnyUserNotificationContent = Annotated[
Union[
MessageNotificationContent,
NewSharedItemNotificationContent,
StorageOperationNotificationContent,
],
NotificationContentField,
]
+165
View File
@@ -0,0 +1,165 @@
from dataclasses import dataclass
from datetime import datetime
from enum import Enum
from typing import (
Any,
Optional,
)
from pydantic import (
BaseModel,
ConfigDict,
Field,
UUID4,
)
from galaxy.schema.fields import (
DecodedDatabaseIdField,
EncodedDatabaseIdField,
)
CreateTimeField = Field(
title="Create Time",
description="The time and date this item was created.",
)
UpdateTimeField = Field(
title="Update Time",
description="The last time and date this item was updated.",
)
class Model(BaseModel):
model_config = ConfigDict(populate_by_name=True, use_enum_values=True, protected_namespaces=())
class StorageOperationMode(str, Enum):
move = "move"
class StorageOperationRunState(str, Enum):
pending = "pending"
running = "running"
completed = "completed"
failed = "failed"
class StorageOperationRunItemState(str, Enum):
pending = "pending"
running = "running"
succeeded = "succeeded"
failed = "failed"
skipped = "skipped"
class DatasetStorageOperationFailureReasonCode(str, Enum):
dataset_not_found = "dataset_not_found"
invalid_target_object_store = "invalid_target_object_store"
missing_source_object_store = "missing_source_object_store"
already_in_target = "already_in_target"
target_quota_exceeded = "target_quota_exceeded"
shared_dataset = "shared_dataset"
insufficient_permissions = "insufficient_permissions"
dataset_in_use = "dataset_in_use"
target_expiration_imminent = "target_expiration_imminent"
checksum_verification_failed = "checksum_verification_failed"
execution_error = "execution_error"
class StorageOperationPreviewRequest(Model):
mode: StorageOperationMode
target_object_store_id: str
items: Optional[list[dict[str, Any]]] = None
class StorageOperationSelectionCounts(Model):
selected_items_count: int
expanded_leaf_count: int
unique_dataset_count: int
class StorageOperationEligibilityReasonSummary(Model):
reason_code: DatasetStorageOperationFailureReasonCode
count: int
class StorageOperationEligibilitySummary(Model):
eligible_count: int
ineligible_count: int
reasons: list[StorageOperationEligibilityReasonSummary] = Field(default_factory=list)
class StorageOperationQuotaDeltaTransfer(Model):
source_object_store_id: str
target_object_store_id: str
bytes: int = 0
class StorageOperationQuotaProjectionSummary(Model):
projected_usage: int
quota_limit: int
class StorageOperationEstimateSummary(Model):
bytes_to_transfer: int = 0
quota_delta_transfers: list[StorageOperationQuotaDeltaTransfer] = Field(default_factory=list)
quota_projection: Optional[StorageOperationQuotaProjectionSummary] = None
class StorageOperationPreviewResponse(Model):
snapshot_id: EncodedDatabaseIdField
selection_counts: StorageOperationSelectionCounts
eligibility: StorageOperationEligibilitySummary
estimates: StorageOperationEstimateSummary
warnings: list[str] = Field(default_factory=list)
expires_at: datetime
class StorageOperationExecutePolicy(Model):
skip_ineligible: bool = True
max_retries: Optional[int] = None
class StorageOperationExecuteRequest(Model):
snapshot_id: DecodedDatabaseIdField
execution_policy: StorageOperationExecutePolicy = Field(default_factory=StorageOperationExecutePolicy)
notify_on_completion: bool = True
class StorageOperationRunSummary(Model):
run_id: EncodedDatabaseIdField
state: StorageOperationRunState
mode: StorageOperationMode
target_object_store_id: str
create_time: datetime = CreateTimeField
update_time: datetime = UpdateTimeField
total_count: int
succeeded_count: int
failed_count: int
skipped_count: int
total_bytes_processed: int
task_id: Optional[UUID4] = None
class StorageOperationRunItemStatus(Model):
dataset_id: EncodedDatabaseIdField
state: StorageOperationRunItemState
reason_code: Optional[DatasetStorageOperationFailureReasonCode] = None
bytes_processed: int
create_time: datetime = CreateTimeField
update_time: datetime = UpdateTimeField
class StorageOperationExecuteResponse(Model):
run: StorageOperationRunSummary
class StorageOperationRunResponse(Model):
run: StorageOperationRunSummary
items: list[StorageOperationRunItemStatus] = Field(default_factory=list)
@dataclass(frozen=True)
class StorageOperationExecutionResult:
state: StorageOperationRunState
message: str
@@ -57,6 +57,14 @@ from galaxy.schema.schema import (
UpdateHistoryContentsPayload,
WriteStoreToPayload,
)
from galaxy.schema.storage_operations import (
StorageOperationExecuteRequest,
StorageOperationExecuteResponse,
StorageOperationPreviewRequest,
StorageOperationPreviewResponse,
StorageOperationRunItemStatus,
StorageOperationRunResponse,
)
from galaxy.schema.tasks import (
CopyDatasetsPayload,
CopyDatasetsResponse,
@@ -64,7 +72,9 @@ from galaxy.schema.tasks import (
from galaxy.webapps.galaxy.api import (
depends,
DependsOnTrans,
IndexQueryTag,
Router,
search_query_param,
)
from galaxy.webapps.galaxy.api.common import (
get_filter_query_params,
@@ -133,6 +143,33 @@ PurgeQueryParam = Query(
deprecated=True,
)
StorageRunOffsetQueryParam = Query(
default=0,
ge=0,
title="Offset",
description="The offset for paginated per-item run details.",
)
StorageRunLimitQueryParam = Query(
default=50,
ge=1,
le=200,
title="Limit",
description="The maximum number of per-item run details to return.",
)
StorageRunItemsSearchTags = [
IndexQueryTag("state", "Item state."),
IndexQueryTag("reason_code", "Item reason code.", alias="reason"),
IndexQueryTag("dataset_id", "Encoded dataset id.", alias="dataset"),
]
StorageRunSearchQueryParam: Optional[str] = search_query_param(
model_name="Storage operation run item",
tags=StorageRunItemsSearchTags,
free_text_fields=["state", "reason_code", "dataset_id"],
)
RecursiveQueryParam = Query(
default=False,
title="Recursive",
@@ -801,6 +838,68 @@ class FastAPIHistoryContents:
"""
return self.service.bulk_operation(trans, history_id, filter_query_params, payload)
@router.post(
"/api/histories/{history_id}/contents/bulk/storage/preview",
summary="Previews a storage bulk operation for selected history contents.",
)
def bulk_storage_operation_preview(
self,
history_id: HistoryIDPathParam,
trans: ProvidesHistoryContext = DependsOnTrans,
filter_query_params: ValueFilterQueryParams = Depends(get_value_filter_query_params),
payload: StorageOperationPreviewRequest = Body(...),
) -> StorageOperationPreviewResponse:
return self.service.bulk_storage_operation_preview(trans, history_id, filter_query_params, payload)
@router.post(
"/api/histories/{history_id}/contents/bulk/storage/execute",
summary="Executes a previously previewed storage bulk operation snapshot.",
)
def bulk_storage_operation_execute(
self,
history_id: HistoryIDPathParam,
trans: ProvidesHistoryContext = DependsOnTrans,
payload: StorageOperationExecuteRequest = Body(...),
) -> StorageOperationExecuteResponse:
return self.service.bulk_storage_operation_execute(trans, history_id, payload)
@router.get(
"/api/histories/{history_id}/contents/bulk/storage/runs/{run_id}",
summary="Returns run status summary for a storage bulk operation.",
)
def bulk_storage_operation_run(
self,
history_id: HistoryIDPathParam,
run_id: DecodedDatabaseIdField,
trans: ProvidesHistoryContext = DependsOnTrans,
) -> StorageOperationRunResponse:
return self.service.bulk_storage_operation_run(trans, history_id, run_id)
@router.get(
"/api/histories/{history_id}/contents/bulk/storage/runs/{run_id}/items",
summary="Returns paginated per-item details for a storage bulk operation run.",
)
def bulk_storage_operation_run_items(
self,
history_id: HistoryIDPathParam,
run_id: DecodedDatabaseIdField,
response: Response,
trans: ProvidesHistoryContext = DependsOnTrans,
offset: int = StorageRunOffsetQueryParam,
limit: int = StorageRunLimitQueryParam,
search: Optional[str] = StorageRunSearchQueryParam,
) -> list[StorageOperationRunItemStatus]:
run_items, total_matches = self.service.bulk_storage_operation_run_items(
trans,
history_id,
run_id,
offset=offset,
limit=limit,
search=search,
)
response.headers["total_matches"] = str(total_matches)
return run_items
@router.put(
"/api/histories/{history_id}/contents/{id}/validate",
summary="Validates the metadata associated with a dataset within a History.",
+2
View File
@@ -279,6 +279,8 @@ def app_pair(global_conf, load_app_kwds=None, wsgi_preflight=True, **kwargs):
webapp.add_client_route("/histories/{history_id}/graph")
webapp.add_client_route("/histories/{history_id}/pages")
webapp.add_client_route("/histories/{history_id}/pages/{page_id}")
webapp.add_client_route("/histories/{history_id}/storage/runs")
webapp.add_client_route("/histories/{history_id}/storage/runs/{path:.*?}")
webapp.add_client_route("/histories/archived")
webapp.add_client_route("/histories/list_published")
webapp.add_client_route("/histories/list_shared")
@@ -10,19 +10,19 @@ from typing import (
TYPE_CHECKING,
Union,
)
from uuid import UUID
from celery import chain
from pydantic import (
ConfigDict,
Field,
)
from typing_extensions import (
Protocol,
)
from typing_extensions import Protocol
from galaxy import exceptions
from galaxy.celery.helpers import async_task_summary
from galaxy.celery.tasks import (
bulk_move_storage,
change_datatype,
materialize as materialize_task,
prepare_dataset_collection_download,
@@ -30,7 +30,9 @@ from galaxy.celery.tasks import (
touch,
write_history_content_to,
)
from galaxy.config import GalaxyAppConfiguration
from galaxy.managers import (
datasets,
folders,
hdas,
hdcas,
@@ -46,6 +48,7 @@ from galaxy.managers.context import (
ProvidesHistoryContext,
ProvidesUserContext,
)
from galaxy.managers.dataset_storage_operations import DatasetStorageOperationManager
from galaxy.managers.genomes import GenomesManager
from galaxy.managers.history_contents import (
HistoryContentsFilters,
@@ -107,6 +110,14 @@ from galaxy.schema.schema import (
UpdateHistoryContentsBatchPayload,
WriteStoreToPayload,
)
from galaxy.schema.storage_operations import (
StorageOperationExecuteRequest,
StorageOperationExecuteResponse,
StorageOperationPreviewRequest,
StorageOperationPreviewResponse,
StorageOperationRunItemStatus,
StorageOperationRunResponse,
)
from galaxy.schema.tasks import (
CopyDatasetsPayload,
CopyDatasetsResponse,
@@ -262,9 +273,11 @@ class HistoriesContentsService(ServiceBase, ServesExportStores, ConsumesModelSto
def __init__(
self,
security: IdEncodingHelper,
config: GalaxyAppConfiguration,
object_store: BaseObjectStore,
history_manager: histories.HistoryManager,
history_contents_manager: HistoryContentsManager,
dataset_manager: datasets.DatasetManager,
hda_manager: hdas.HDAManager,
hdca_manager: hdcas.HDCAManager,
dataset_collection_manager: DatasetCollectionManager,
@@ -280,6 +293,7 @@ class HistoriesContentsService(ServiceBase, ServesExportStores, ConsumesModelSto
super().__init__(security)
self.history_manager = history_manager
self.history_contents_manager = history_contents_manager
self.dataset_manager = dataset_manager
self.hda_manager = hda_manager
self.hdca_manager = hdca_manager
self.dataset_collection_manager = dataset_collection_manager
@@ -293,6 +307,11 @@ class HistoriesContentsService(ServiceBase, ServesExportStores, ConsumesModelSto
self.short_term_storage_allocator = short_term_storage_allocator
self.genomes_manager = genomes_manager
self.object_store = object_store
self.storage_operation_manager = DatasetStorageOperationManager(
object_store,
config,
hdca_manager=self.hdca_manager,
)
def index(
self,
@@ -734,6 +753,107 @@ class HistoriesContentsService(ServiceBase, ServesExportStores, ConsumesModelSto
success_count = len(contents) - len(errors)
return HistoryContentBulkOperationResult(success_count=success_count, errors=errors)
def bulk_storage_operation_preview(
self,
trans: ProvidesHistoryContext,
history_id: DecodedDatabaseIdField,
filter_query_params: ValueFilterQueryParams,
payload: StorageOperationPreviewRequest,
) -> StorageOperationPreviewResponse:
user = self.get_authenticated_user(trans)
history = self.history_manager.get_mutable(history_id, user, current_history=trans.history)
filters = self.history_contents_filters.parse_query_filters(filter_query_params)
if payload.items:
parsed_items = [HistoryContentItem.model_validate(item) for item in payload.items]
contents = self._get_contents_by_item_list(trans, history, parsed_items)
else:
contents = self.history_contents_manager.contents(history, filters)
return self.storage_operation_manager.build_preview_response(
sa_session=trans.sa_session,
security_agent=trans.app.security_agent,
quota_agent=trans.app.quota_agent,
history_id=history.id,
user=user,
contents=contents,
target_object_store_id=payload.target_object_store_id,
query_based_selection=not payload.items,
)
def bulk_storage_operation_execute(
self,
trans: ProvidesHistoryContext,
history_id: DecodedDatabaseIdField,
payload: StorageOperationExecuteRequest,
) -> StorageOperationExecuteResponse:
user = self.get_authenticated_user(trans)
history = self.history_manager.get_mutable(history_id, user, current_history=trans.history)
snapshot = self.storage_operation_manager.get_snapshot(trans.sa_session, payload.snapshot_id)
self.storage_operation_manager.validate_snapshot_for_history(
snapshot,
history_id=history.id,
user_id=user.id,
)
run, _ = self.storage_operation_manager.create_run_and_summary(
sa_session=trans.sa_session,
snapshot=snapshot,
skip_ineligible=payload.execution_policy.skip_ineligible,
notify_on_completion=payload.notify_on_completion,
)
task_result = bulk_move_storage.delay(
run_db_id=run.id,
task_user_id=user.id,
notify_on_completion=payload.notify_on_completion,
)
run.task_id = UUID(task_result.id)
trans.sa_session.add(run)
trans.sa_session.commit()
return StorageOperationExecuteResponse(run=self.storage_operation_manager.to_run_summary(run))
def bulk_storage_operation_run(
self,
trans: ProvidesHistoryContext,
history_id: DecodedDatabaseIdField,
run_id: DecodedDatabaseIdField,
) -> StorageOperationRunResponse:
user = self.get_authenticated_user(trans)
history = self.history_manager.get_mutable(history_id, user, current_history=trans.history)
run = self.storage_operation_manager.get_run(
sa_session=trans.sa_session,
run_id=run_id,
history_id=history.id,
user_id=user.id,
)
summary = self.storage_operation_manager.to_run_summary(run)
return StorageOperationRunResponse(run=summary)
def bulk_storage_operation_run_items(
self,
trans: ProvidesHistoryContext,
history_id: DecodedDatabaseIdField,
run_id: DecodedDatabaseIdField,
offset: int = 0,
limit: int = 50,
search: Optional[str] = None,
) -> tuple[list[StorageOperationRunItemStatus], int]:
user = self.get_authenticated_user(trans)
history = self.history_manager.get_mutable(history_id, user, current_history=trans.history)
run = self.storage_operation_manager.get_run(
sa_session=trans.sa_session,
run_id=run_id,
history_id=history.id,
user_id=user.id,
)
return self.storage_operation_manager.get_run_items(
sa_session=trans.sa_session,
run=run,
decode_id=self.decode_id,
offset=offset,
limit=limit,
search=search,
)
def validate(self, trans, history_id: DecodedDatabaseIdField, history_content_id: DecodedDatabaseIdField):
"""
Validates the metadata associated with a dataset within a History.
+82
View File
@@ -1504,6 +1504,88 @@ class BaseDatasetPopulator(BasePopulator):
get_storage_response = self._get(storage_url)
return get_storage_response
def bulk_storage_operation_preview_raw(
self, history_id: str, payload: dict[str, Any], expected_status: int = 200
) -> Response:
response = self._post(
f"histories/{history_id}/contents/bulk/storage/preview",
data=payload,
json=True,
)
api_asserts.assert_status_code_is(response, expected_status)
return response
def bulk_storage_operation_preview(
self, history_id: str, payload: dict[str, Any], expected_status: int = 200
) -> dict[str, Any]:
return self.bulk_storage_operation_preview_raw(history_id, payload, expected_status=expected_status).json()
def bulk_storage_operation_execute_raw(
self, history_id: str, payload: dict[str, Any], expected_status: int = 200
) -> Response:
response = self._post(
f"histories/{history_id}/contents/bulk/storage/execute",
data=payload,
json=True,
)
api_asserts.assert_status_code_is(response, expected_status)
return response
def bulk_storage_operation_execute(
self, history_id: str, payload: dict[str, Any], expected_status: int = 200
) -> dict[str, Any]:
return self.bulk_storage_operation_execute_raw(history_id, payload, expected_status=expected_status).json()
def bulk_storage_operation_run_status_raw(
self, history_id: str, run_id: str, expected_status: int = 200
) -> Response:
response = self._get(f"histories/{history_id}/contents/bulk/storage/runs/{run_id}")
api_asserts.assert_status_code_is(response, expected_status)
return response
def bulk_storage_operation_run_status(
self, history_id: str, run_id: str, expected_status: int = 200
) -> dict[str, Any]:
response = self.bulk_storage_operation_run_status_raw(history_id, run_id, expected_status=expected_status)
return response.json()
def bulk_storage_operation_run_items(
self,
history_id: str,
run_id: str,
expected_status: int = 200,
offset: int = 0,
limit: int = 50,
search: Optional[str] = None,
) -> list[dict[str, Any]]:
query: dict[str, Any] = {
"offset": offset,
"limit": limit,
}
if search is not None:
query["search"] = search
response = self._get(f"histories/{history_id}/contents/bulk/storage/runs/{run_id}/items", data=query)
api_asserts.assert_status_code_is(response, expected_status)
return response.json()
def wait_for_bulk_storage_operation_run(
self,
history_id: str,
run_id: str,
include_items_on_terminal: bool = False,
search: Optional[str] = None,
timeout: timeout_type = DEFAULT_TIMEOUT,
) -> dict[str, Any]:
def is_terminal():
run_data = self.bulk_storage_operation_run_status(history_id, run_id)
if run_data["run"]["state"] in ("completed", "failed"):
if include_items_on_terminal:
run_data["items"] = self.bulk_storage_operation_run_items(history_id, run_id, search=search)
return run_data
return None
return wait_on(is_terminal, "bulk storage operation run completion", timeout=timeout)
def get_roles(self) -> list:
using_requirement("admin")
roles_response = self._get("roles", admin=True)
File diff suppressed because it is too large Load Diff
@@ -4,6 +4,7 @@ from datetime import (
)
from typing import (
Any,
cast,
Optional,
)
from unittest.mock import (
@@ -20,6 +21,7 @@ from galaxy.managers.notification import (
NotificationRecipientResolver,
)
from galaxy.model import (
DatasetStorageOperationRun,
Group,
Role,
User,
@@ -39,6 +41,10 @@ from galaxy.schema.notifications import (
UserNotificationPreferences,
UserNotificationUpdateRequest,
)
from galaxy.schema.storage_operations import (
StorageOperationExecutionResult,
StorageOperationRunState,
)
from galaxy.util import now
from .base import BaseTestCase
@@ -256,6 +262,37 @@ class TestUserNotifications(NotificationManagerBaseTestCase):
assert actual_notifications_sent == num_target_users
self._assert_notification_expected(actual_notification, expected_notification)
def test_send_storage_operation_notification(self):
user = self._create_test_user()
run = DatasetStorageOperationRun(
id=7,
history_id=11,
mode="move",
total_count=3,
succeeded_count=3,
failed_count=0,
skipped_count=0,
)
notification, sent_count = self.notification_manager.send_storage_operation_notification(
user_id=user.id,
run=run,
execution_result=StorageOperationExecutionResult(
state=StorageOperationRunState.completed,
message="Bulk move completed.",
),
encode_id=lambda value: f"encoded-{value}",
)
assert sent_count == 1
assert notification is not None
assert notification.category == PersonalNotificationCategory.storage_operation
content = cast(dict[str, Any], notification.content)
assert content["state"] == StorageOperationRunState.completed.value
assert content["history_id"] == "encoded-11"
assert content["run_id"] == "encoded-7"
def test_get_user_notifications(self):
user = self._create_test_user()
user_notifications = self.notification_manager.get_user_notifications(user)