opensource/galaxy
3
0
Fork 0
mirror of https://github.com/galaxyproject/galaxy.git synced 2026-09-24 16:30:27 +08:00
Code Issues Packages Projects Releases Wiki Activity

objectstores storage management (UI+API)

Browse Source
This commit is contained in:
John Chilton
2024-03-27 10:22:59 -04:00
parent 1c07b630be
commit 77ef4a3896
19 changed files with 670 additions and 136 deletions
Download Patch File Download Diff File Expand all files Collapse all files
client/src/api/datasets.ts
+7 -11
View File
@@ -49,25 +49,21 @@ export async function fetchDatasetDetails(params: { id: string }): Promise<Datas
return data as unknown as DatasetDetails;
}
const updateHistoryDataset = fetcher.path("/api/histories/{history_id}/contents/{type}s/{id}").method("put").create();
const updateDataset = fetcher.path("/api/datasets/{dataset_id}").method("put").create();
export async function undeleteHistoryDataset(historyId: string, datasetId: string) {
const { data } = await updateHistoryDataset({
history_id: historyId,
id: datasetId,
export async function undeleteHistoryDataset(datasetId: string) {
const { data } = await updateDataset({
dataset_id: datasetId,
type: "dataset",
deleted: false,
});
return data;
}
const deleteHistoryDataset = fetcher
.path("/api/histories/{history_id}/contents/{type}s/{id}")
.method("delete")
.create();
const deleteDataset = fetcher.path("/api/datasets/{dataset_id}").method("delete").create();
export async function purgeHistoryDataset(historyId: string, datasetId: string) {
const { data } = await deleteHistoryDataset({ history_id: historyId, id: datasetId, type: "dataset", purge: true });
export async function purgeHistoryDataset(datasetId: string) {
const { data } = await deleteDataset({ dataset_id: datasetId, purge: true });
return data;
}
client/src/api/users.ts
+1
View File
@@ -3,6 +3,7 @@ import { fetcher } from "@/api/schema";
export const createApiKey = fetcher.path("/api/users/{user_id}/api_key").method("post").create();
export const deleteUser = fetcher.path("/api/users/{user_id}").method("delete").create();
export const fetchQuotaUsages = fetcher.path("/api/users/{user_id}/usage").method("get").create();
export const fetchObjectStoreUsages = fetcher.path("/api/users/{user_id}/objectstore_usage").method("get").create();
export const recalculateDiskUsage = fetcher.path("/api/users/current/recalculate_disk_usage").method("put").create();
export const recalculateDiskUsageByUserId = fetcher
.path("/api/users/{user_id}/recalculate_disk_usage")
client/src/components/User/DiskUsage/StorageDashboard.vue
+28 -7
View File
@@ -20,14 +20,22 @@ const texts = reactive({
icon: "fas fa-broom fa-6x",
buttonText: localize("Free up disk usage"),
},
explore: {
title: localize("Visually explore your disk usage"),
explore_by_history: {
title: localize("Visually explore your disk usage by history"),
description: localize(
"Want to know what histories or datasets take up the most space in your account? Here you can explore your disk usage in a visual way."
"Want to know what histories or datasets take up the most space in your account? Here you can explore your disk usage in a visual way by history."
),
icon: "fas fa-chart-pie fa-6x",
buttonText: localize("Explore now"),
},
explore_by_objectstore: {
title: localize("Visually explore your disk usage by object store"),
description: localize(
"Want to know how the space in your account is being distributed by object store? Here you can explore your disk usage in a visual way by object store."
),
icon: "fas fa-hdd fa-6x",
buttonText: localize("Explore now"),
},
});
function goToStorageManager() {
@@ -37,6 +45,10 @@ function goToStorageManager() {
function goToHistoriesOverview() {
router.push({ name: "HistoriesOverview" });
}
function goToObjectStoresOverview() {
router.push({ name: "ObjectStoresOverview" });
}
</script>
<template>
@@ -60,10 +72,19 @@ function goToHistoriesOverview() {
<IconCard
class="mx-auto mb-3"
data-description="explore usage card"
:title="texts.explore.title"
:description="texts.explore.description"
:icon="texts.explore.icon"
:button-text="texts.explore.buttonText"
:title="texts.explore_by_history.title"
:description="texts.explore_by_history.description"
:icon="texts.explore_by_history.icon"
:button-text="texts.explore_by_history.buttonText"
@onButtonClick="goToHistoriesOverview" />
<IconCard
class="mx-auto mb-3"
data-description="explore object store usage card"
:title="texts.explore_by_objectstore.title"
:description="texts.explore_by_objectstore.description"
:icon="texts.explore_by_objectstore.icon"
:button-text="texts.explore_by_objectstore.buttonText"
@onButtonClick="goToObjectStoresOverview" />
</div>
</template>
client/src/components/User/DiskUsage/Visualizations/HistoriesStorageOverview.vue
+9 -17
View File
@@ -1,5 +1,5 @@
<script setup lang="ts">
import { onMounted, ref } from "vue";
import { ref } from "vue";
import { useRouter } from "vue-router/composables";
import { useConfirmDialog } from "@/composables/confirmDialog";
@@ -8,13 +8,14 @@ import { useHistoryStore } from "@/stores/historyStore";
import localize from "@/utils/localization";
import type { DataValuePoint } from "./Charts";
import { bytesLabelFormatter, bytesValueFormatter } from "./Charts/formatters";
import { fetchAllHistoriesSizeSummary, type ItemSizeSummary, purgeHistoryById, undeleteHistoryById } from "./service";
import { byteFormattingForChart, useDataLoading } from "./util";
import BarChart from "./Charts/BarChart.vue";
import OverviewPage from "./OverviewPage.vue";
import RecoverableItemSizeTooltip from "./RecoverableItemSizeTooltip.vue";
import SelectedItemActions from "./SelectedItemActions.vue";
import WarnDeletedHistories from "./WarnDeletedHistories.vue";
import LoadingSpan from "@/components/LoadingSpan.vue";
const historyStore = useHistoryStore();
@@ -25,17 +26,16 @@ const { confirm } = useConfirmDialog();
const historiesSizeSummaryMap = new Map<string, ItemSizeSummary>();
const topTenHistoriesBySizeData = ref<DataValuePoint[] | null>(null);
const activeVsArchivedVsDeletedTotalSizeData = ref<DataValuePoint[] | null>(null);
const isLoading = ref(true);
const numberOfHistoriesToDisplayOptions = [10, 20, 50];
const numberOfHistoriesToDisplay = ref(numberOfHistoriesToDisplayOptions[0]);
onMounted(async () => {
isLoading.value = true;
const { isLoading, loadDataOnMount } = useDataLoading();
loadDataOnMount(async () => {
const allHistoriesSizeSummary = await fetchAllHistoriesSizeSummary();
allHistoriesSizeSummary.forEach((history) => historiesSizeSummaryMap.set(history.id, history));
buildGraphsData();
isLoading.value = false;
});
function buildGraphsData() {
@@ -155,13 +155,7 @@ async function onPermanentlyDeleteHistory(historyId: string) {
<p class="text-justify">
Here you can find various graphs displaying the storage size taken by <b>all your histories</b>.
</p>
<p class="text-justify">
Note: these graphs include <b>deleted histories</b>. Remember that, even if you delete histories, they still
take up storage space. However, you can free up the storage space by permanently deleting them from the
<i>Discarded Items</i> section of the
<router-link :to="{ name: 'StorageManager' }"><b>Storage Manager</b></router-link> page or by selecting them
individually in the graph and clicking the <b>Permanently Delete</b> button.
</p>
<WarnDeletedHistories />
<div v-if="isLoading" class="text-center">
<LoadingSpan class="mt-5" :message="localize('Loading your storage data. This may take a while...')" />
@@ -176,8 +170,7 @@ async function onPermanentlyDeleteHistory(historyId: string) {
"
:data="topTenHistoriesBySizeData"
:enable-selection="true"
:label-formatter="bytesLabelFormatter"
:value-formatter="bytesValueFormatter">
v-bind="byteFormattingForChart">
<template v-slot:title>
<b>{{ localize(`Top ${numberOfHistoriesToDisplay} Histories by Size`) }}</b>
<b-form-select
@@ -218,8 +211,7 @@ async function onPermanentlyDeleteHistory(historyId: string) {
)
"
:data="activeVsArchivedVsDeletedTotalSizeData"
:label-formatter="bytesLabelFormatter"
:value-formatter="bytesValueFormatter">
v-bind="byteFormattingForChart">
<template v-slot:tooltip="{ data }">
<RecoverableItemSizeTooltip
v-if="data"
client/src/components/User/DiskUsage/Visualizations/HistoryStorageOverview.vue
+37 -96
View File
@@ -1,30 +1,22 @@
<script setup lang="ts">
import { onMounted, ref } from "vue";
import { ref } from "vue";
import { useRouter } from "vue-router/composables";
import { useConfirmDialog } from "@/composables/confirmDialog";
import { useToast } from "@/composables/toast";
import { useHistoryStore } from "@/stores/historyStore";
import localize from "@/utils/localization";
import type { DataValuePoint } from "./Charts";
import { bytesLabelFormatter, bytesValueFormatter } from "./Charts/formatters";
import {
fetchHistoryContentsSizeSummary,
type ItemSizeSummary,
purgeDatasetById,
undeleteDatasetById,
} from "./service";
import { fetchHistoryContentsSizeSummary, type ItemSizeSummary } from "./service";
import { buildTopNDatasetsBySizeData, byteFormattingForChart, useDataLoading, useDatasetsToDisplay } from "./util";
import BarChart from "./Charts/BarChart.vue";
import OverviewPage from "./OverviewPage.vue";
import RecoverableItemSizeTooltip from "./RecoverableItemSizeTooltip.vue";
import SelectedItemActions from "./SelectedItemActions.vue";
import WarnDeletedDatasets from "./WarnDeletedDatasets.vue";
import LoadingSpan from "@/components/LoadingSpan.vue";
const router = useRouter();
const { success: successToast, error: errorToast } = useToast();
const { confirm } = useConfirmDialog();
const { getHistoryNameById } = useHistoryStore();
interface Props {
@@ -33,40 +25,39 @@ interface Props {
const props = defineProps<Props>();
const datasetsSizeSummaryMap = new Map<string, ItemSizeSummary>();
const topTenDatasetsBySizeData = ref<DataValuePoint[] | null>(null);
const activeVsDeletedTotalSizeData = ref<DataValuePoint[] | null>(null);
const isLoading = ref(true);
const numberOfDatasetsToDisplayOptions = [10, 20, 50];
const numberOfDatasetsToDisplay = ref(numberOfDatasetsToDisplayOptions[0]);
const {
numberOfDatasetsToDisplayOptions,
numberOfDatasetsToDisplay,
numberOfDatasetsLimit,
datasetsSizeSummaryMap,
topNDatasetsBySizeData,
isRecoverableDataPoint,
onUndeleteDataset,
onPermanentlyDeleteDataset,
} = useDatasetsToDisplay();
onMounted(async () => {
isLoading.value = true;
const limit = Math.max(...numberOfDatasetsToDisplayOptions);
const allDatasetsInHistorySizeSummary = await fetchHistoryContentsSizeSummary(props.historyId, limit);
const { isLoading, loadDataOnMount } = useDataLoading();
loadDataOnMount(async () => {
const allDatasetsInHistorySizeSummary = await fetchHistoryContentsSizeSummary(
props.historyId,
numberOfDatasetsLimit
);
allDatasetsInHistorySizeSummary.forEach((dataset) => datasetsSizeSummaryMap.set(dataset.id, dataset));
buildGraphsData();
isLoading.value = false;
});
function buildGraphsData() {
const allDatasetsInHistorySizeSummary = Array.from(datasetsSizeSummaryMap.values());
topTenDatasetsBySizeData.value = buildTopTenDatasetsBySizeData(allDatasetsInHistorySizeSummary);
topNDatasetsBySizeData.value = buildTopNDatasetsBySizeData(
allDatasetsInHistorySizeSummary,
numberOfDatasetsToDisplay.value
);
activeVsDeletedTotalSizeData.value = buildActiveVsDeletedTotalSizeData(allDatasetsInHistorySizeSummary);
}
function buildTopTenDatasetsBySizeData(datasetsSizeSummary: ItemSizeSummary[]): DataValuePoint[] {
const topTenDatasetsBySize = datasetsSizeSummary
.sort((a, b) => b.size - a.size)
.slice(0, numberOfDatasetsToDisplay.value);
return topTenDatasetsBySize.map((dataset) => ({
id: dataset.id,
label: dataset.name,
value: dataset.size,
}));
}
function buildActiveVsDeletedTotalSizeData(datasetsSizeSummary: ItemSizeSummary[]): DataValuePoint[] {
const activeDatasetsSize = datasetsSizeSummary
.filter((dataset) => !dataset.deleted)
@@ -88,14 +79,6 @@ function buildActiveVsDeletedTotalSizeData(datasetsSizeSummary: ItemSizeSummary[
];
}
function isRecoverableDataPoint(dataPoint?: DataValuePoint): boolean {
if (dataPoint) {
const historiesSizeSummary = datasetsSizeSummaryMap.get(dataPoint.id || "");
return historiesSizeSummary?.deleted || dataPoint.id === "deleted";
}
return false;
}
async function onViewDataset(datasetId: string) {
router.push({
name: "DatasetDetails",
@@ -106,45 +89,12 @@ async function onViewDataset(datasetId: string) {
});
}
async function onUndeleteDataset(datasetId: string) {
try {
const result = await undeleteDatasetById(props.historyId, datasetId);
const dataset = datasetsSizeSummaryMap.get(datasetId);
if (dataset && !result.deleted) {
dataset.deleted = result.deleted;
datasetsSizeSummaryMap.set(datasetId, dataset);
successToast(localize("Dataset undeleted successfully."));
buildGraphsData();
}
} catch (error) {
errorToast(`${error}`, localize("An error occurred while undeleting the dataset."));
}
function onPermDelete(datasetId: string) {
onPermanentlyDeleteDataset(buildGraphsData, datasetId);
}
async function onPermanentlyDeleteDataset(datasetId: string) {
const confirmed = await confirm(
localize("Are you sure you want to permanently delete this dataset? This action cannot be undone."),
{
title: localize("Permanently delete dataset?"),
okVariant: "danger",
okTitle: localize("Permanently delete"),
cancelTitle: localize("Cancel"),
}
);
if (!confirmed) {
return;
}
try {
const result = await purgeDatasetById(props.historyId, datasetId);
const dataset = datasetsSizeSummaryMap.get(datasetId);
if (dataset && result) {
datasetsSizeSummaryMap.delete(datasetId);
successToast(localize("Dataset permanently deleted successfully."));
buildGraphsData();
}
} catch (error) {
errorToast(`${error}`, localize("An error occurred while permanently deleting the dataset."));
}
function onUndelete(datasetId: string) {
onUndeleteDataset(buildGraphsData, datasetId);
}
</script>
<template>
@@ -157,36 +107,28 @@ async function onPermanentlyDeleteDataset(datasetId: string) {
<router-link :to="{ name: 'HistoriesOverview' }"><b>Histories Storage Overview</b></router-link> page to see
the storage taken by <b>all your histories</b>.
</p>
<p class="text-justify">
Note: these graphs include <b>deleted datasets</b>. Remember that, even if you delete datasets, they still
take up storage space. However, you can free up the storage space by permanently deleting them from the
<i>Discarded Items</i> section of the
<router-link :to="{ name: 'StorageManager' }"><b>Storage Manager</b></router-link> page or by selecting them
individually in the graph and clicking the <b>Permanently Delete</b> button.
</p>
<WarnDeletedDatasets />
<div v-if="isLoading" class="text-center">
<LoadingSpan class="mt-5" :message="localize('Loading your storage data. This may take a while...')" />
</div>
<div v-else>
<BarChart
v-if="topTenDatasetsBySizeData"
v-if="topNDatasetsBySizeData"
:description="
localize(
`These are the ${numberOfDatasetsToDisplay} datasets that take the most space in this history. Click on a bar to see more information about the dataset.`
)
"
:data="topNDatasetsBySizeData"
:enable-selection="true"
:data="topTenDatasetsBySizeData"
:label-formatter="bytesLabelFormatter"
:value-formatter="bytesValueFormatter">
v-bind="byteFormattingForChart">
<template v-slot:title>
<b>{{ localize(`Top ${numberOfDatasetsToDisplay} Datasets by Size`) }}</b>
<b-form-select
v-model="numberOfDatasetsToDisplay"
:options="numberOfDatasetsToDisplayOptions"
:disabled="isLoading"
title="Number of histories to show"
title="Number of datasets to show"
class="float-right w-auto"
size="sm"
@change="buildGraphsData()">
@@ -204,8 +146,8 @@ async function onPermanentlyDeleteDataset(datasetId: string) {
item-type="dataset"
:is-recoverable="isRecoverableDataPoint(data)"
@view-item="onViewDataset"
@undelete-item="onUndeleteDataset"
@permanently-delete-item="onPermanentlyDeleteDataset" />
@undelete-item="onUndelete"
@permanently-delete-item="onPermDelete" />
</template>
</BarChart>
@@ -218,8 +160,7 @@ async function onPermanentlyDeleteDataset(datasetId: string) {
)
"
:data="activeVsDeletedTotalSizeData"
:label-formatter="bytesLabelFormatter"
:value-formatter="bytesValueFormatter">
v-bind="byteFormattingForChart">
<template v-slot:tooltip="{ data }">
<RecoverableItemSizeTooltip
v-if="data"
client/src/components/User/DiskUsage/Visualizations/ObjectStoreActions.vue
+48
View File
@@ -0,0 +1,48 @@
<script setup lang="ts">
import { library } from "@fortawesome/fontawesome-svg-core";
import { faChartBar } from "@fortawesome/free-solid-svg-icons";
import { FontAwesomeIcon } from "@fortawesome/vue-fontawesome";
import { BButton } from "bootstrap-vue";
import { computed } from "vue";
import localize from "@/utils/localization";
import type { DataValuePoint } from "./Charts";
interface Props {
data: DataValuePoint;
}
const props = defineProps<Props>();
library.add(faChartBar);
const label = computed(() => props.data.id);
const viewDetailsIcon = computed(() => "chart-bar");
const emit = defineEmits<{
(e: "view-item", itemId: string): void;
}>();
function onViewItem() {
emit("view-item", props.data.id);
}
</script>
<template>
<div class="selected-item-info">
<div class="h-md mx-2">
<b>{{ label }}</b>
</div>
<div class="my-2">
<BButton
variant="outline-primary"
size="sm"
class="mx-2"
:title="localize(`Go to the details of this object store`)"
@click="onViewItem">
<FontAwesomeIcon :icon="viewDetailsIcon" />
</BButton>
</div>
</div>
</template>
client/src/components/User/DiskUsage/Visualizations/ObjectStoreStorageOverview.vue
+123
View File
@@ -0,0 +1,123 @@
<script setup lang="ts">
import { useRouter } from "vue-router/composables";
import { fetchObjectStoreContentsSizeSummary } from "./service";
import { buildTopNDatasetsBySizeData, byteFormattingForChart, useDataLoading, useDatasetsToDisplay } from "./util";
import BarChart from "./Charts/BarChart.vue";
import OverviewPage from "./OverviewPage.vue";
import RecoverableItemSizeTooltip from "./RecoverableItemSizeTooltip.vue";
import SelectedItemActions from "./SelectedItemActions.vue";
import WarnDeletedDatasets from "./WarnDeletedDatasets.vue";
import LoadingSpan from "@/components/LoadingSpan.vue";
interface Props {
objectStoreId: string;
}
const props = defineProps<Props>();
const router = useRouter();
const {
numberOfDatasetsToDisplayOptions,
numberOfDatasetsToDisplay,
numberOfDatasetsLimit,
datasetsSizeSummaryMap,
topNDatasetsBySizeData,
isRecoverableDataPoint,
onUndeleteDataset,
onPermanentlyDeleteDataset,
} = useDatasetsToDisplay();
const { isLoading, loadDataOnMount } = useDataLoading();
loadDataOnMount(async () => {
const allDatasetsInObjectStoreSizeSummary = await fetchObjectStoreContentsSizeSummary(
props.objectStoreId,
numberOfDatasetsLimit
);
allDatasetsInObjectStoreSizeSummary.forEach((dataset) => datasetsSizeSummaryMap.set(dataset.id, dataset));
buildGraphsData();
});
function buildGraphsData() {
const allDatasetsInObjectStoreSizeSummary = Array.from(datasetsSizeSummaryMap.values());
topNDatasetsBySizeData.value = buildTopNDatasetsBySizeData(
allDatasetsInObjectStoreSizeSummary,
numberOfDatasetsToDisplay.value
);
}
async function onViewDataset(datasetId: string) {
router.push({
name: "DatasetDetails",
params: {
datasetId: datasetId,
},
});
}
function onPermDelete(datasetId: string) {
onPermanentlyDeleteDataset(buildGraphsData, datasetId);
}
function onUndelete(datasetId: string) {
onUndeleteDataset(buildGraphsData, datasetId);
}
</script>
<template>
<OverviewPage title="Object Store Storage Overview">
<p class="text-justify">
Here you will find some Graphs displaying the storage taken by datasets in the object store:
<b>{{ objectStoreId }}</b
>. You can use these graphs to identify the datasets that take the most space in this object store.
</p>
<WarnDeletedDatasets />
<div v-if="isLoading" class="text-center">
<LoadingSpan class="mt-5" :message="localize('Loading your storage data. This may take a while...')" />
</div>
<div v-else>
<BarChart
v-if="topNDatasetsBySizeData"
:description="
localize(
`These are the ${numberOfDatasetsToDisplay} datasets that take the most space in this history. Click on a bar to see more information about the dataset.`
)
"
v-bind="byteFormattingForChart"
:enable-selection="true"
:data="topNDatasetsBySizeData">
<template v-slot:title>
<b>{{ localize(`Top ${numberOfDatasetsToDisplay} Datasets by Size`) }}</b>
<b-form-select
v-model="numberOfDatasetsToDisplay"
:options="numberOfDatasetsToDisplayOptions"
:disabled="isLoading"
title="Number of datasets to show"
class="float-right w-auto"
size="sm"
@change="buildGraphsData()">
</b-form-select>
</template>
<template v-slot:tooltip="{ data }">
<RecoverableItemSizeTooltip
v-if="data"
:data="data"
:is-recoverable="isRecoverableDataPoint(data)" />
</template>
<template v-slot:selection="{ data }">
<SelectedItemActions
:data="data"
item-type="dataset"
:is-recoverable="isRecoverableDataPoint(data)"
@view-item="onViewDataset"
@permanently-delete-item="onPermDelete"
@undelete-item="onUndelete" />
</template>
</BarChart>
</div>
</OverviewPage>
</template>
client/src/components/User/DiskUsage/Visualizations/ObjectStoresStorageOverview.vue
+71
View File
@@ -0,0 +1,71 @@
<script setup lang="ts">
import { ref } from "vue";
import { useRouter } from "vue-router/composables";
import { fetchObjectStoreUsages } from "@/api/users";
import localize from "@/utils/localization";
import type { DataValuePoint } from "./Charts";
import { byteFormattingForChart, useDataLoading } from "./util";
import BarChart from "./Charts/BarChart.vue";
import ObjectStoreActions from "./ObjectStoreActions.vue";
import OverviewPage from "./OverviewPage.vue";
import ShowObjectStore from "./ShowObjectStore.vue";
import WarnDeletedHistories from "./WarnDeletedHistories.vue";
import LoadingSpan from "@/components/LoadingSpan.vue";
const router = useRouter();
const objectStoresBySizeData = ref<DataValuePoint[] | null>(null);
const { isLoading, loadDataOnMount } = useDataLoading();
loadDataOnMount(async () => {
const { data } = await fetchObjectStoreUsages({ user_id: "current" });
const objectStoresBySize = data.sort((a, b) => b.total_disk_usage - a.total_disk_usage);
objectStoresBySizeData.value = objectStoresBySize.map((objectStore) => ({
id: objectStore.object_store_id,
label: objectStore.object_store_id,
value: objectStore.total_disk_usage,
}));
});
function onViewObjectStore(objectStoreId: string) {
router.push({ name: "ObjectStoreOverview", params: { objectStoreId } });
}
</script>
<template>
<OverviewPage title="Object Stores Storage Overview">
<p class="text-justify">
Here you can find various graphs displaying the storage size taken by all your histories grouped by object
store.
</p>
<WarnDeletedHistories />
<div v-if="isLoading" class="text-center">
<LoadingSpan class="mt-5" :message="localize('Loading your storage data. This may take a while...')" />
</div>
<div v-else>
<BarChart
v-if="objectStoresBySizeData"
:description="
localize(
`This graph displays how your Galaxy data is stored sorted into the the object store is stored in. Click on a bar to see more information about the object store.`
)
"
:data="objectStoresBySizeData"
:enable-selection="true"
v-bind="byteFormattingForChart">
<template v-slot:title>
<b>{{ localize(`Object Stores by Usage`) }}</b>
</template>
<template v-slot:tooltip="{ data }">
<ShowObjectStore v-if="data" :object-store-id="data.id" />
</template>
<template v-slot:selection="{ data }">
<ObjectStoreActions :data="data" @view-item="onViewObjectStore" />
</template>
</BarChart>
</div>
</OverviewPage>
</template>
client/src/components/User/DiskUsage/Visualizations/ShowObjectStore.vue
+50
View File
@@ -0,0 +1,50 @@
<script setup lang="ts">
import { ref, watch } from "vue";
import { getObjectStoreDetails } from "@/api/objectStores";
import type { ConcreteObjectStoreModel } from "@/components/ObjectStore/types";
import { errorMessageAsString } from "@/utils/simple-error";
import LoadingSpan from "@/components/LoadingSpan.vue";
import DescribeObjectStore from "@/components/ObjectStore/DescribeObjectStore.vue";
interface Props {
objectStoreId: string;
}
const props = defineProps<Props>();
const objectStore = ref<ConcreteObjectStoreModel | null>(null);
const loading = ref(true);
const error = ref<string | null>(null);
async function fetch() {
loading.value = true;
try {
objectStore.value = await getObjectStoreDetails(props.objectStoreId);
} catch (e) {
error.value = errorMessageAsString(e);
} finally {
loading.value = false;
}
}
watch(
() => props.objectStoreId,
async () => {
fetch();
}
);
fetch();
const loadingMessage = "Loading object store details";
const forWhat = "This object store is";
</script>
<template>
<div style="width: 300px">
<LoadingSpan v-if="loading" :message="loadingMessage | localize" />
<DescribeObjectStore v-else-if="objectStore != null" :what="forWhat" :storage-info="objectStore">
</DescribeObjectStore>
<b-alert v-else-if="error" show variant="danger">{{ error }}</b-alert>
</div>
</template>
client/src/components/User/DiskUsage/Visualizations/WarnDeletedDatasets.vue
+9
View File
@@ -0,0 +1,9 @@
<template>
<p class="text-justify">
Note: these graphs include <b>deleted datasets</b>. Remember that, even if you delete datasets, they still take
up storage space. However, you can free up the storage space by permanently deleting them from the
<i>Discarded Items</i> section of the
<router-link :to="{ name: 'StorageManager' }"><b>Storage Manager</b></router-link> page or by selecting them
individually in the graph and clicking the <b>Permanently Delete</b> button.
</p>
</template>
client/src/components/User/DiskUsage/Visualizations/WarnDeletedHistories.vue
+9
View File
@@ -0,0 +1,9 @@
<template>
<p class="text-justify">
Note: these graphs include <b>deleted histories</b>. Remember that, even if you delete histories, they still
take up storage space. However, you can free up the storage space by permanently deleting them from the
<i>Discarded Items</i> section of the
<router-link :to="{ name: 'StorageManager' }"><b>Storage Manager</b></router-link> page or by selecting them
individually in the graph and clicking the <b>Permanently Delete</b> button.
</p>
</template>
client/src/components/User/DiskUsage/Visualizations/service.ts
+15 -4
View File
@@ -45,6 +45,17 @@ export async function fetchHistoryContentsSizeSummary(historyId: string, limit =
return response.data as unknown as ItemSizeSummary[];
}
export async function fetchObjectStoreContentsSizeSummary(objectStoreId: string, limit = 5000) {
const response = await datasetsFetcher({
keys: itemSizeSummaryFields,
limit,
order: "size-dsc",
q: ["purged", "history_content_type", "object_store_id"],
qv: ["false", "dataset", objectStoreId],
});
return response.data as unknown as ItemSizeSummary[];
}
export async function undeleteHistoryById(historyId: string): Promise<ItemSizeSummary> {
const response = await undeleteHistory({ history_id: historyId });
return response.data as unknown as ItemSizeSummary;
@@ -55,12 +66,12 @@ export async function purgeHistoryById(historyId: string): Promise<PurgeableItem
return response.data as unknown as PurgeableItemSizeSummary;
}
export async function undeleteDatasetById(historyId: string, datasetId: string): Promise<ItemSizeSummary> {
const data = await undeleteHistoryDataset(historyId, datasetId);
export async function undeleteDatasetById(datasetId: string): Promise<ItemSizeSummary> {
const data = await undeleteHistoryDataset(datasetId);
return data as unknown as ItemSizeSummary;
}
export async function purgeDatasetById(historyId: string, datasetId: string): Promise<PurgeableItemSizeSummary> {
const data = await purgeHistoryDataset(historyId, datasetId);
export async function purgeDatasetById(datasetId: string): Promise<PurgeableItemSizeSummary> {
const data = await purgeHistoryDataset(datasetId);
return data as unknown as PurgeableItemSizeSummary;
}
client/src/components/User/DiskUsage/Visualizations/util.ts
+119
View File
@@ -0,0 +1,119 @@
import { onMounted, ref } from "vue";
import { useConfirmDialog } from "@/composables/confirmDialog";
import { useToast } from "@/composables/toast";
import localize from "@/utils/localization";
import type { DataValuePoint } from "./Charts";
import { bytesLabelFormatter, bytesValueFormatter } from "./Charts/formatters";
import { type ItemSizeSummary, purgeDatasetById, undeleteDatasetById } from "./service";
interface DataLoader {
(): Promise<undefined>;
}
interface DataReload {
(): void;
}
export function useDatasetsToDisplay() {
const numberOfDatasetsToDisplayOptions = [10, 20, 50];
const numberOfDatasetsToDisplay = ref<number>(numberOfDatasetsToDisplayOptions[0] || 10);
const numberOfDatasetsLimit = Math.max(...numberOfDatasetsToDisplayOptions);
const datasetsSizeSummaryMap = new Map<string, ItemSizeSummary>();
const topNDatasetsBySizeData = ref<DataValuePoint[] | null>(null);
function isRecoverableDataPoint(dataPoint?: DataValuePoint): boolean {
if (dataPoint) {
const datasetSizeSummary = datasetsSizeSummaryMap.get(dataPoint.id || "");
return datasetSizeSummary?.deleted || dataPoint.id === "deleted";
}
return false;
}
const { success: successToast, error: errorToast } = useToast();
const { confirm } = useConfirmDialog();
async function onUndeleteDataset(reloadData: DataReload, datasetId: string) {
try {
const result = await undeleteDatasetById(datasetId);
const dataset = datasetsSizeSummaryMap.get(datasetId);
if (dataset && !result.deleted) {
dataset.deleted = result.deleted;
datasetsSizeSummaryMap.set(datasetId, dataset);
successToast(localize("Dataset undeleted successfully."));
reloadData();
}
} catch (error) {
errorToast(`${error}`, localize("An error occurred while undeleting the dataset."));
}
}
async function onPermanentlyDeleteDataset(reloadData: DataReload, datasetId: string) {
const confirmed = await confirm(
localize("Are you sure you want to permanently delete this dataset? This action cannot be undone."),
{
title: localize("Permanently delete dataset?"),
okVariant: "danger",
okTitle: localize("Permanently delete"),
cancelTitle: localize("Cancel"),
}
);
if (!confirmed) {
return;
}
try {
const result = await purgeDatasetById(datasetId);
const dataset = datasetsSizeSummaryMap.get(datasetId);
if (dataset && result) {
datasetsSizeSummaryMap.delete(datasetId);
successToast(localize("Dataset permanently deleted successfully."));
reloadData();
}
} catch (error) {
errorToast(`${error}`, localize("An error occurred while permanently deleting the dataset."));
}
}
return {
numberOfDatasetsToDisplayOptions,
numberOfDatasetsToDisplay,
numberOfDatasetsLimit,
datasetsSizeSummaryMap,
topNDatasetsBySizeData,
isRecoverableDataPoint,
onUndeleteDataset,
onPermanentlyDeleteDataset,
};
}
export function useDataLoading() {
const isLoading = ref(true);
const loadDataOnMount = (dataLoader: DataLoader) => {
onMounted(async () => {
isLoading.value = true;
await dataLoader();
isLoading.value = false;
});
};
return {
isLoading,
loadDataOnMount,
};
}
export function buildTopNDatasetsBySizeData(datasetsSizeSummary: ItemSizeSummary[], n: number): DataValuePoint[] {
const topTenDatasetsBySize = datasetsSizeSummary.sort((a, b) => b.size - a.size).slice(0, n);
return topTenDatasetsBySize.map((dataset) => ({
id: dataset.id,
label: dataset.name,
value: dataset.size,
}));
}
export const byteFormattingForChart = {
"enable-selection": true,
labelFormatter: bytesLabelFormatter,
valueFormatter: bytesValueFormatter,
};
client/src/entry/analysis/routes/storageDashboardRoutes.ts
+14
View File
@@ -2,6 +2,8 @@ import StorageManager from "@/components/User/DiskUsage/Management/StorageManage
import StorageDashboard from "@/components/User/DiskUsage/StorageDashboard.vue";
import HistoriesStorageOverview from "@/components/User/DiskUsage/Visualizations/HistoriesStorageOverview.vue";
import HistoryStorageOverview from "@/components/User/DiskUsage/Visualizations/HistoryStorageOverview.vue";
import ObjectStoresStorageOverview from "@/components/User/DiskUsage/Visualizations/ObjectStoresStorageOverview.vue";
import ObjectStoreStorageOverview from "@/components/User/DiskUsage/Visualizations/ObjectStoreStorageOverview.vue";
import Base from "@/entry/analysis/modules/Base.vue";
export default [
@@ -32,6 +34,18 @@ export default [
component: HistoryStorageOverview,
props: true,
},
{
path: "objectstores",
name: "ObjectStoresOverview",
component: ObjectStoresStorageOverview,
props: true,
},
{
path: "objectstores/:objectStoreId",
name: "ObjectStoreOverview",
component: ObjectStoreStorageOverview,
props: true,
},
{
path: "*",
redirect: { name: "StorageDashboard" },
lib/galaxy/model/__init__.py
+39
View File
@@ -657,6 +657,35 @@ WHERE user_id = :id AND quota_source_label IS NOT NULL
return statements
UNIQUE_DATASET_USER_USAGE_PER_OBJECTSTORE = """
WITH per_user_histories AS
(
SELECT id
FROM history
WHERE user_id = :id
AND NOT purged
),
per_hist_hdas AS (
SELECT DISTINCT dataset_id
FROM history_dataset_association
WHERE NOT purged
AND history_id IN (SELECT id FROM per_user_histories)
)
SELECT COALESCE(SUM(COALESCE(dataset.total_size, dataset.file_size, 0)), 0), dataset.object_store_id
FROM dataset
LEFT OUTER JOIN library_dataset_dataset_association ON dataset.id = library_dataset_dataset_association.dataset_id
WHERE dataset.id IN (SELECT dataset_id FROM per_hist_hdas)
AND library_dataset_dataset_association.id IS NULL
GROUP BY dataset.object_store_id
"""
def calculate_disk_usage_per_objectstore(sa_session, user_id: str):
statement = UNIQUE_DATASET_USER_USAGE_PER_OBJECTSTORE
params = {"id": user_id}
return sa_session.execute(statement, params).all()
# move these to galaxy.schema.schema once galaxy-data depends on
# galaxy-schema.
class UserQuotaBasicUsage(BaseModel):
@@ -670,6 +699,11 @@ class UserQuotaUsage(UserQuotaBasicUsage):
quota: Optional[str] = None
class UserObjectstoreUsage(BaseModel):
object_store_id: str
total_disk_usage: float
class User(Base, Dictifiable, RepresentById):
"""
Data for a Galaxy user or admin and relations to their
@@ -1141,6 +1175,11 @@ ON CONFLICT
with transaction(session):
session.commit()
def dictify_objectstore_usage(self) -> List[UserObjectstoreUsage]:
session = object_session(self)
rows = calculate_disk_usage_per_objectstore(session, self.id)
return [UserObjectstoreUsage(object_store_id=r[1], total_disk_usage=r[0]) for r in rows if r[1]]
def dictify_usage(self, object_store=None) -> List[UserQuotaBasicUsage]:
"""Include object_store to include empty/unused usage info."""
used_labels: Set[Union[str, None]] = set()
lib/galaxy/webapps/galaxy/api/history_contents.py
+56
View File
@@ -810,6 +810,28 @@ class FastAPIHistoryContents:
trans, history_id, id, payload.model_dump(exclude_unset=True), serialization_params, contents_type=type
)
@router.put(
"/api/datasets/{dataset_id}",
summary="Updates the values for the history dataset (HDA) item with the given ``ID``.",
operation_id="datasets__update_dataset",
)
def update_dataset(
self,
dataset_id: HistoryItemIDPathParam,
trans: ProvidesHistoryContext = DependsOnTrans,
serialization_params: SerializationParams = Depends(query_serialization_params),
payload: UpdateHistoryContentsPayload = Body(...),
) -> AnyHistoryContentItem:
"""Updates the values for the history content item with the given ``ID``."""
return self.service.update(
trans,
None,
dataset_id,
payload.model_dump(exclude_unset=True),
serialization_params,
contents_type=HistoryContentType.dataset,
)
@router.put(
"/api/histories/{history_id}/contents/{id}",
summary="Updates the values for the history content item with the given ``ID`` and query specified type. ``/api/histories/{history_id}/contents/{type}s/{id}`` should be used instead.",
@@ -903,6 +925,40 @@ class FastAPIHistoryContents:
payload,
)
@router.delete(
"/api/datasets/{dataset_id}",
summary="Delete the history dataset content with the given ``ID``.",
responses=CONTENT_DELETE_RESPONSES,
operation_id="datasets__delete",
)
def delete_dataset(
self,
response: Response,
dataset_id: HistoryItemIDPathParam,
trans: ProvidesHistoryContext = DependsOnTrans,
serialization_params: SerializationParams = Depends(query_serialization_params),
purge: Optional[bool] = PurgeQueryParam,
recursive: Optional[bool] = RecursiveQueryParam,
stop_job: Optional[bool] = StopJobQueryParam,
payload: DeleteHistoryContentPayload = Body(None),
):
"""
Delete the history content with the given ``ID`` and path specified type.
**Note**: Currently does not stop any active jobs for which this dataset is an output.
"""
return self._delete(
response,
trans,
dataset_id,
HistoryContentType.dataset,
serialization_params,
purge,
recursive,
stop_job,
payload,
)
def _delete(
self,
response: Response,
lib/galaxy/webapps/galaxy/api/users.py
+16
View File
@@ -38,6 +38,7 @@ from galaxy.model import (
HistoryDatasetAssociation,
Role,
UserAddress,
UserObjectstoreUsage,
UserQuotaUsage,
)
from galaxy.model.base import transaction
@@ -304,6 +305,21 @@ class FastAPIUsers:
else:
return []
@router.get(
"/api/users/{user_id}/objectstore_usage",
name="get_user_objectstore_usage",
summary="Return the user's object store usage summary broken down by object store ID",
)
def objectstore_usage(
self,
trans: ProvidesUserContext = DependsOnTrans,
user_id: FlexibleUserIdType = FlexibleUserIdPathParam,
) -> List[UserObjectstoreUsage]:
if user := self.service.get_user_full(trans, user_id, False):
return user.dictify_objectstore_usage()
else:
return []
@router.get(
"/api/users/{user_id}/usage/{label}",
name="get_user_usage_for_label",
lib/galaxy/webapps/galaxy/services/history_contents.py
+5 -1
View File
@@ -610,7 +610,7 @@ class HistoriesContentsService(ServiceBase, ServesExportStores, ConsumesModelSto
def update(
self,
trans,
history_id: DecodedDatabaseIdField,
history_id: Optional[DecodedDatabaseIdField],
id: DecodedDatabaseIdField,
payload: Dict[str, Any],
serialization_params: SerializationParams,
@@ -627,6 +627,10 @@ class HistoriesContentsService(ServiceBase, ServesExportStores, ConsumesModelSto
:returns: an error object if an error occurred or a dictionary containing
any values that were different from the original and, therefore, updated
"""
if history_id is None:
hda = self.hda_manager.get_owned(id, trans.user, current_history=trans.history)
history_id = hda.history.id
history = self.history_manager.get_mutable(history_id, trans.user, current_history=trans.history)
if contents_type == HistoryContentType.dataset:
return self.__update_dataset(trans, history, id, payload, serialization_params)
test/unit/data/test_quota.py
+14
View File
@@ -142,6 +142,20 @@ class TestCalculateUsage(BaseModelTestCase):
u.calculate_and_set_disk_usage(object_store)
self._refresh_user_and_assert_disk_usage_is(10)
def test_calculate_objectstore_usage(self):
# not strictly a quota check but such similar code and ideas...
u = self.u
self._add_dataset(10, "not_tracked")
self._add_dataset(15, "tracked")
usage = u.dictify_objectstore_usage()
assert len(usage) == 2
usage_dict = dict([(u.object_store_id, u.total_disk_usage) for u in usage])
assert int(usage_dict["not_tracked"]) == 10
assert int(usage_dict["tracked"]) == 15
def test_calculate_usage_disabled_quota(self):
u = self.u