diff --git a/client/src/components/DataDialog/DataDialog.vue b/client/src/components/DataDialog/DataDialog.vue index 913bd8b12c2..bedead028e3 100644 --- a/client/src/components/DataDialog/DataDialog.vue +++ b/client/src/components/DataDialog/DataDialog.vue @@ -1,11 +1,14 @@ + + + + + + + {{ title }} + + — {{ targetHistoryName }} + + + + + + + + + + + {{ error }} + + + + Uploading + {{ Math.round(uploadProgress) }}% + + + + + + + + + + + + + + + Cancel + + Start + + + + + + + diff --git a/client/src/components/Panels/Upload/UploadMethodView.vue b/client/src/components/Panels/Upload/UploadMethodView.vue index fcf73bc8705..f5e907d5ba6 100644 --- a/client/src/components/Panels/Upload/UploadMethodView.vue +++ b/client/src/components/Panels/Upload/UploadMethodView.vue @@ -4,6 +4,7 @@ import { computed, ref, watch } from "vue"; import { useRouter } from "vue-router/composables"; import { useTargetHistoryUploadState } from "@/composables/history/useTargetHistoryUploadState"; +import { useUploadSubmission } from "@/composables/upload/useUploadSubmission"; import { useHistoryStore } from "@/stores/historyStore"; import type { UploadMethod, UploadMethodComponent } from "./types"; @@ -25,6 +26,8 @@ const router = useRouter(); const uploadMethodRef = ref(null); const canUpload = ref(false); +const { submitPreparedUpload } = useUploadSubmission(); + const historyStore = useHistoryStore(); const { currentHistoryId } = storeToRefs(historyStore); @@ -75,7 +78,13 @@ function handleStart() { if (!canStartUpload.value) { return; } - uploadMethodRef.value?.startUpload(); + const prepared = uploadMethodRef.value?.prepareUpload(); + if (!prepared) { + return; + } + // Fire-and-forget: progress is tracked in uploadState, visible in the progress view + void submitPreparedUpload(targetHistoryId.value, prepared); + uploadMethodRef.value?.reset?.(); router.push("/upload/progress"); } diff --git a/client/src/components/Panels/Upload/UploadProgress.vue b/client/src/components/Panels/Upload/UploadProgress.vue index ec72067996c..efa2bd08ff5 100644 --- a/client/src/components/Panels/Upload/UploadProgress.vue +++ b/client/src/components/Panels/Upload/UploadProgress.vue @@ -1,7 +1,7 @@ diff --git a/client/src/components/Panels/Upload/methods/DataLibraryUpload.vue b/client/src/components/Panels/Upload/methods/DataLibraryUpload.vue index 23ea26ed8b6..0662849b36c 100644 --- a/client/src/components/Panels/Upload/methods/DataLibraryUpload.vue +++ b/client/src/components/Panels/Upload/methods/DataLibraryUpload.vue @@ -15,17 +15,18 @@ import { getFolderContents, getLibraries, isLibraryFile } from "@/api/libraries" import type { BreadcrumbItem } from "@/components/Common"; import type { TableField } from "@/components/Common/GTable.types"; import { Model } from "@/components/FilesDialog/model"; +import { selectionToArray } from "@/components/FilesDialog/utilities"; import type { SelectionItem } from "@/components/SelectionDialog/selectionTypes"; import { useCollectionCreation } from "@/composables/upload/collectionCreation"; import { useUploadReadyState } from "@/composables/upload/uploadReadyState"; import { useUploadStaging } from "@/composables/upload/useUploadStaging"; -import { useUploadQueue } from "@/composables/uploadQueue"; import { useUrlTracker } from "@/composables/urlTracker"; import { errorMessageAsString } from "@/utils/simple-error"; +import { buildPreparedUpload } from "@/utils/upload"; import { mapToLibraryDatasetUpload } from "@/utils/upload/itemMappers"; import { bytesToString } from "@/utils/utils"; -import type { UploadMethodComponent, UploadMethodConfig } from "../types"; +import type { PreparedUpload, UploadMethodComponent, UploadMethodConfig } from "../types"; import type { LibraryDatasetItem } from "../types/uploadItem"; import CollectionCreationConfig from "../CollectionCreationConfig.vue"; @@ -37,23 +38,34 @@ import UtcDate from "@/components/UtcDate.vue"; interface Props { method: UploadMethodConfig; + /** History ID where uploaded datasets will be added. */ targetHistoryId: string; + /** Allow creating dataset collections from selected library datasets. */ + allowCollections?: boolean; + /** When false, restrict selection to a single dataset. */ + multiple?: boolean; + /** When true, do not persist staging to the shared store (modal use). */ + transient?: boolean; } -const props = defineProps(); +const props = withDefaults(defineProps(), { + allowCollections: true, + multiple: true, + transient: false, +}); const emit = defineEmits<{ (e: "ready", ready: boolean): void; }>(); -const uploadQueue = useUploadQueue(); - const tableContainerRef = ref(null); const collectionConfigComponent = ref | null>(null); const libraryDatasetItems = ref([]); -const { clear: clearStaging } = useUploadStaging(props.method.id, libraryDatasetItems); +const { clear: clearStaging } = useUploadStaging(props.method.id, libraryDatasetItems, { + disableStore: props.transient, +}); -const { collectionState, handleCollectionStateChange, buildCollectionConfig, resetCollection } = +const { buildCollectionConfig, collectionState, handleCollectionStateChange, resetCollection } = useCollectionCreation(collectionConfigComponent); let nextId = 1; @@ -106,7 +118,8 @@ function createLibraryDatasetItem(item: AnyLibraryFolderItem, libraryId: string, const showBrowser = ref(true); -const selectionModel = ref(new Model({ multiple: true })); +const isSingleMode = computed(() => props.multiple === false); +const selectionModel = ref(new Model({ multiple: !isSingleMode.value })); const selectionCount = ref(0); // Navigation state @@ -174,6 +187,10 @@ const hasPagination = computed(() => { const hasItems = computed(() => libraryDatasetItems.value.length > 0); const hasSelection = computed(() => selectionCount.value > 0); +const addMoreDatasetsTitle = computed(() => + isSingleMode.value ? "Change selected dataset" : "Add more datasets to the upload list", +); +const addMoreDatasetsLabel = computed(() => (isSingleMode.value ? "Change selected dataset" : "Add More Datasets")); const searchTitle = computed(() => { if (!currentLibrary.value) { @@ -420,7 +437,12 @@ async function navigateToBreadcrumb(index: number) { } function addSelectedDatasets() { - const selectedItems = selectionModel.value.finalize() as SelectionItem[]; + let selectedItems = selectionToArray(selectionModel.value.finalize()); + + if (isSingleMode.value) { + selectedItems = selectedItems.slice(0, 1); + libraryDatasetItems.value = []; + } // Filter out any items that already exist in libraryDatasetItems const existingUrls = new Set(libraryDatasetItems.value.map((item) => item.url)); @@ -435,7 +457,7 @@ function addSelectedDatasets() { } // Clear selection and switch to table view - selectionModel.value = new Model({ multiple: true }); + selectionModel.value = new Model({ multiple: !isSingleMode.value }); selectionCount.value = 0; showBrowser.value = false; scrollToBottom(); @@ -460,18 +482,24 @@ function scrollToBottom() { function removeItem(id: number) { libraryDatasetItems.value = libraryDatasetItems.value.filter((item) => item.id !== id); + + if (libraryDatasetItems.value.length === 0) { + showBrowser.value = true; + resetCollection(); + } } -function clearAll() { +function reset() { libraryDatasetItems.value = []; resetCollection(); - selectionModel.value = new Model({ multiple: true }); + selectionModel.value = new Model({ multiple: !isSingleMode.value }); selectionCount.value = 0; searchQuery.value = ""; urlTracker.reset(); currentPage.value = 1; loadLibraries(); showBrowser.value = true; + clearStaging(); } function clearSearch() { @@ -482,21 +510,14 @@ function updateSearchQuery(newQuery: string) { searchQuery.value = newQuery; } -function startUpload() { +function prepareUpload(): PreparedUpload | null { + if (libraryDatasetItems.value.length === 0) { + return null; + } + const uploads = libraryDatasetItems.value.map((item) => mapToLibraryDatasetUpload(item, props.targetHistoryId)); - const collectionConfig = buildCollectionConfig(props.targetHistoryId); - uploadQueue.enqueue(uploads, collectionConfig); - - // Reset state - libraryDatasetItems.value = []; - clearStaging(); - showBrowser.value = true; - resetCollection(); - selectionModel.value = new Model({ multiple: true }); - selectionCount.value = 0; - clearSearch(); - urlTracker.reset(); + return buildPreparedUpload(uploads, buildCollectionConfig(props.targetHistoryId)); } const libraryFields: TableField[] = [ @@ -694,7 +715,7 @@ onMounted(async () => { } }); -defineExpose({ startUpload }); +defineExpose({ prepareUpload, reset }); @@ -771,7 +792,7 @@ defineExpose({ startUpload }); @row-click="onBrowserRowClick"> @@ -888,6 +909,7 @@ defineExpose({ startUpload }); @@ -897,10 +919,10 @@ defineExpose({ startUpload }); color="grey" tooltip tooltip-placement="top" - title="Add more datasets to the upload list" + :title="addMoreDatasetsTitle" @click="showFolderBrowser"> - Add More Datasets + {{ addMoreDatasetsLabel }} ({ startUpload }); tooltip tooltip-placement="top" title="Remove all datasets from the upload list" - @click="clearAll"> + @click="reset"> Clear All diff --git a/client/src/components/Panels/Upload/methods/LocalFileUpload.vue b/client/src/components/Panels/Upload/methods/LocalFileUpload.vue index e4670a9b409..631cf86a892 100644 --- a/client/src/components/Panels/Upload/methods/LocalFileUpload.vue +++ b/client/src/components/Panels/Upload/methods/LocalFileUpload.vue @@ -12,11 +12,11 @@ import { useUploadDefaults } from "@/composables/upload/uploadDefaults"; import { useUploadItemValidation } from "@/composables/upload/uploadItemValidation"; import { useUploadReadyState } from "@/composables/upload/uploadReadyState"; import { useUploadStaging } from "@/composables/upload/useUploadStaging"; -import { useUploadQueue } from "@/composables/uploadQueue"; +import { buildPreparedUpload } from "@/utils/upload"; import { mapToLocalFileUpload } from "@/utils/upload/itemMappers"; import { bytesToString } from "@/utils/utils"; -import type { UploadMethodComponent, UploadMethodConfig } from "../types"; +import type { PreparedUpload, UploadMethodComponent, UploadMethodConfig } from "../types"; import type { LocalFileItem } from "../types/uploadItem"; import CollectionCreationConfig from "../CollectionCreationConfig.vue"; @@ -32,10 +32,24 @@ import GTable from "@/components/Common/GTable.vue"; interface Props { method: UploadMethodConfig; + /** History ID where uploaded datasets will be added. */ targetHistoryId: string; + /** Allow creating dataset collections from selected files. */ + allowCollections?: boolean; + /** Optional list of allowed formats to constrain selectable extensions. */ + formats?: string[]; + /** When false, restrict selection to a single file. */ + multiple?: boolean; + /** When true, do not persist staging to the shared store (modal use). */ + transient?: boolean; } -const props = defineProps(); +const props = withDefaults(defineProps(), { + allowCollections: true, + formats: undefined, + multiple: true, + transient: false, +}); const emit = defineEmits<{ (e: "ready", ready: boolean): void; @@ -43,20 +57,21 @@ const emit = defineEmits<{ const { advancedMode } = useUploadAdvancedMode(); -const uploadQueue = useUploadQueue(); - -const { effectiveExtensions, listDbKeys, configurationsReady, createItemDefaults } = useUploadDefaults(); +const { effectiveExtensions, listDbKeys, configurationsReady, createItemDefaults } = useUploadDefaults(props.formats); const selectedFiles = ref([]); -const { clear: clearStaging } = useUploadStaging(props.method.id, selectedFiles); +const { clear: clearStaging } = useUploadStaging(props.method.id, selectedFiles, { + disableStore: props.transient, +}); const uploadFile = ref(null); const dropZoneElement = ref(null); const collectionConfigComponent = ref | null>(null); -const { collectionState, handleCollectionStateChange, buildCollectionConfig, resetCollection } = +const { buildCollectionConfig, collectionState, handleCollectionStateChange, resetCollection } = useCollectionCreation(collectionConfigComponent); const hasFiles = computed(() => selectedFiles.value.length > 0); +const canAddMoreFiles = computed(() => props.multiple !== false || selectedFiles.value.length === 0); const totalSize = computed(() => { const bytes = selectedFiles.value.reduce((sum, item) => sum + item.file.size, 0); return bytesToString(bytes); @@ -141,7 +156,14 @@ function addFiles(files: FileList | File[] | null) { const fileArray = Array.from(files); const defaults = createItemDefaults(); - for (const file of fileArray) { + // Enforce single file limit if multiple is false + const filesToAdd = props.multiple === false ? fileArray.slice(0, 1) : fileArray; + + for (const file of filesToAdd) { + // If multiple is false, replace existing files + if (props.multiple === false) { + selectedFiles.value = []; + } selectedFiles.value.push({ file, name: file.name, @@ -166,22 +188,22 @@ function handleBrowse() { uploadFile.value?.click(); } -function clearAll() { +function reset() { selectedFiles.value = []; resetCollection(); -} - -function startUpload() { - const uploads = selectedFiles.value.map((item) => mapToLocalFileUpload(item, props.targetHistoryId)); - const collectionConfig = buildCollectionConfig(props.targetHistoryId); - - uploadQueue.enqueue(uploads, collectionConfig); - selectedFiles.value = []; clearStaging(); - resetCollection(); } -defineExpose({ startUpload }); +function prepareUpload(): PreparedUpload | null { + if (selectedFiles.value.length === 0) { + return null; + } + + const uploads = selectedFiles.value.map((item) => mapToLocalFileUpload(item, props.targetHistoryId)); + return buildPreparedUpload(uploads, buildCollectionConfig(props.targetHistoryId)); +} + +defineExpose({ prepareUpload, reset }); @@ -300,6 +322,7 @@ defineExpose({ startUpload }); @@ -309,7 +332,12 @@ defineExpose({ startUpload }); color="grey" tooltip tooltip-placement="top" - title="Browse and add more files to the upload list" + :disabled="!canAddMoreFiles" + :title=" + canAddMoreFiles + ? 'Browse and add more files to the upload list' + : 'Only one file can be selected' + " @click="handleBrowse"> Add More Files @@ -320,7 +348,7 @@ defineExpose({ startUpload }); tooltip tooltip-placement="top" title="Remove all files from the upload list" - @click="clearAll"> + @click="reset"> Clear All @@ -340,7 +368,7 @@ defineExpose({ startUpload }); id="local-file-input" ref="uploadFile" type="file" - multiple + :multiple="props.multiple !== false" class="d-none" @change="addFileFromInput($event.target)" /> diff --git a/client/src/components/Panels/Upload/methods/PasteContentUpload.vue b/client/src/components/Panels/Upload/methods/PasteContentUpload.vue index d7d2e0ccce0..7b6ab2536d5 100644 --- a/client/src/components/Panels/Upload/methods/PasteContentUpload.vue +++ b/client/src/components/Panels/Upload/methods/PasteContentUpload.vue @@ -11,11 +11,11 @@ import { useUploadDefaults } from "@/composables/upload/uploadDefaults"; import { useUploadItemValidation } from "@/composables/upload/uploadItemValidation"; import { useUploadReadyState } from "@/composables/upload/uploadReadyState"; import { useUploadStaging } from "@/composables/upload/useUploadStaging"; -import { useUploadQueue } from "@/composables/uploadQueue"; +import { buildPreparedUpload } from "@/utils/upload"; import { mapToPasteContentUpload } from "@/utils/upload/itemMappers"; import { bytesToString } from "@/utils/utils"; -import type { UploadMethodComponent, UploadMethodConfig } from "../types"; +import type { PreparedUpload, UploadMethodComponent, UploadMethodConfig } from "../types"; import type { PasteContentItem } from "../types/uploadItem"; import CollectionCreationConfig from "../CollectionCreationConfig.vue"; @@ -31,10 +31,24 @@ import GTable from "@/components/Common/GTable.vue"; interface Props { method: UploadMethodConfig; + /** History ID where uploaded datasets will be added. */ targetHistoryId: string; + /** Allow creating dataset collections from pasted datasets. */ + allowCollections?: boolean; + /** Optional list of allowed formats to constrain selectable extensions. */ + formats?: string[]; + /** When false, restrict to a single pasted dataset. */ + multiple?: boolean; + /** When true, do not persist staging to the shared store (modal use). */ + transient?: boolean; } -const props = defineProps(); +const props = withDefaults(defineProps(), { + allowCollections: true, + formats: undefined, + multiple: true, + transient: false, +}); const emit = defineEmits<{ (e: "ready", ready: boolean): void; @@ -42,14 +56,12 @@ const emit = defineEmits<{ const { advancedMode } = useUploadAdvancedMode(); -const uploadQueue = useUploadQueue(); - -const { effectiveExtensions, listDbKeys, configurationsReady, createItemDefaults } = useUploadDefaults(); +const { effectiveExtensions, listDbKeys, configurationsReady, createItemDefaults } = useUploadDefaults(props.formats); const tableContainerRef = ref(null); const collectionConfigComponent = ref | null>(null); -const { collectionState, handleCollectionStateChange, buildCollectionConfig, resetCollection } = +const { buildCollectionConfig, collectionState, handleCollectionStateChange, resetCollection } = useCollectionCreation(collectionConfigComponent); let nextId = 1; @@ -66,7 +78,11 @@ function createPasteContentItem(id: number, name: string): PasteContentItem { const pasteItems = ref([createPasteContentItem(nextId++, "Pasted Dataset 1")]); const expandedItemIds = ref>(new Set()); const rowToggleMap = ref void>>(new Map()); -const { clear: clearStaging } = useUploadStaging(props.method.id, pasteItems); +const { clear: clearStaging } = useUploadStaging(props.method.id, pasteItems, { + disableStore: props.transient, +}); + +const isSingleMode = computed(() => props.multiple === false); const hasItems = computed(() => pasteItems.value.some((item) => item.content.trim().length > 0)); @@ -111,6 +127,16 @@ function removeItem(id: number) { expandedItemIds.value = nextExpanded; } +function reset() { + clearStaging(); + // Reset to single empty item + const newItemId = nextId++; + rowToggleMap.value = new Map(); + expandedItemIds.value = new Set(); + pasteItems.value = [createPasteContentItem(newItemId, "Pasted Dataset 1")]; + resetCollection(); +} + function getItemSize(content: string) { return bytesToString(new Blob([content]).size); } @@ -252,28 +278,17 @@ onMounted(() => { }); }); -function startUpload() { +function prepareUpload(): PreparedUpload | null { const validItems = pasteItems.value.filter((item) => item.content.trim().length > 0); if (validItems.length === 0) { - return; + return null; } const uploads = validItems.map((item) => mapToPasteContentUpload(item, props.targetHistoryId)); - const collectionConfig = buildCollectionConfig(props.targetHistoryId); - - uploadQueue.enqueue(uploads, collectionConfig); - - // Reset to single empty item - const newItemId = nextId++; - pasteItems.value = [createPasteContentItem(newItemId, "Pasted Dataset 1")]; - rowToggleMap.value = new Map(); - expandedItemIds.value = new Set(); - nextTick(() => expandRow(newItemId)); - clearStaging(); - resetCollection(); + return buildPreparedUpload(uploads, buildCollectionConfig(props.targetHistoryId)); } -defineExpose({ startUpload }); +defineExpose({ prepareUpload, reset }); @@ -454,12 +469,14 @@ defineExpose({ startUpload }); (); +const props = withDefaults(defineProps(), { + allowCollections: true, + formats: undefined, + multiple: true, + transient: false, +}); const emit = defineEmits<{ (e: "ready", ready: boolean): void; @@ -43,14 +57,12 @@ const emit = defineEmits<{ const { advancedMode } = useUploadAdvancedMode(); -const uploadQueue = useUploadQueue(); - -const { effectiveExtensions, listDbKeys, configurationsReady, createItemDefaults } = useUploadDefaults(); +const { effectiveExtensions, listDbKeys, configurationsReady, createItemDefaults } = useUploadDefaults(props.formats); const tableContainerRef = ref(null); const collectionConfigComponent = ref | null>(null); -const { collectionState, handleCollectionStateChange, buildCollectionConfig, resetCollection } = +const { buildCollectionConfig, collectionState, handleCollectionStateChange, resetCollection } = useCollectionCreation(collectionConfigComponent); let nextId = 1; @@ -68,11 +80,18 @@ function createPasteUrlItem(id: number, url: string, name: string): PasteUrlItem const urlItems = ref([]); const urlText = ref(""); const showInputArea = ref(true); -const { clear: clearStaging } = useUploadStaging(props.method.id, urlItems); +const { clear: clearStaging } = useUploadStaging(props.method.id, urlItems, { + disableStore: props.transient, +}); const placeholder = "https://example.org/data1.txt\nhttps://example.org/data2.txt"; const hasItems = computed(() => urlItems.value.length > 0); +const isSingleMode = computed(() => props.multiple === false); +const addMoreUrlsTitle = computed(() => + isSingleMode.value ? "Change selected URL" : "Add more URLs to the upload list", +); +const addMoreUrlsLabel = computed(() => (isSingleMode.value ? "Change selected URL" : "Add More URLs")); const { isNameValid, restoreOriginalName } = useUploadItemValidation(); @@ -103,7 +122,14 @@ function addUrlsFromText() { .map((u) => u.trim()) .filter((u) => u.length > 0); - for (const url of urls) { + // Enforce single URL when multiple is false + const urlsToAdd = isSingleMode.value ? urls.slice(0, 1) : urls; + + if (isSingleMode.value) { + urlItems.value = []; + } + + for (const url of urlsToAdd) { urlItems.value.push(createPasteUrlItem(nextId++, url, extractNameFromUrl(url))); } @@ -131,6 +157,11 @@ function scrollToBottom() { function removeItem(id: number) { urlItems.value = urlItems.value.filter((item) => item.id !== id); + + if (urlItems.value.length === 0) { + showInputArea.value = true; + resetCollection(); + } } const tableFields: TableField[] = [ @@ -176,33 +207,25 @@ const tableFields: TableField[] = [ }, ]; -function clearAll() { +function reset() { urlItems.value = []; urlText.value = ""; + clearStaging(); showInputArea.value = true; resetCollection(); } -function startUpload() { +function prepareUpload(): PreparedUpload | null { const validItems = urlItems.value.filter((item) => item.url.trim().length > 0); if (validItems.length === 0) { - return; + return null; } const uploads = validItems.map((item) => mapToPasteUrlUpload(item, props.targetHistoryId)); - const collectionConfig = buildCollectionConfig(props.targetHistoryId); - - uploadQueue.enqueue(uploads, collectionConfig); - - // Reset state - urlItems.value = []; - clearStaging(); - urlText.value = ""; - showInputArea.value = true; - resetCollection(); + return buildPreparedUpload(uploads, buildCollectionConfig(props.targetHistoryId)); } -defineExpose({ startUpload }); +defineExpose({ prepareUpload, reset }); @@ -347,19 +370,15 @@ defineExpose({ startUpload }); - + - Add More URLs + {{ addMoreUrlsLabel }} ({ startUpload }); tooltip tooltip-placement="top" title="Remove all URLs from the upload list" - @click="clearAll"> + @click="reset"> Clear All diff --git a/client/src/components/Panels/Upload/methods/RemoteFilesUpload.vue b/client/src/components/Panels/Upload/methods/RemoteFilesUpload.vue index 03fec89884f..c06f058178b 100644 --- a/client/src/components/Panels/Upload/methods/RemoteFilesUpload.vue +++ b/client/src/components/Panels/Upload/methods/RemoteFilesUpload.vue @@ -1,12 +1,17 @@ - - - View {{ remoteFileItems.length }} staged file(s) + + + + + Browse a File Source below or + + + Connect New Remote Source + + + + + + + + + + + + + {{ errorMessage }} + Retry + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + {{ item.label }} + + + + + + + + {{ item.details }} + + + + + + + + + + No {{ urlTracker.isAtRoot.value ? "file sources" : "files or folders" }} match your search "{{ + searchQuery + }}" + + + No file sources available. Connect a new remote source to get started. + + This directory is empty. + + + + + + + + + + + + View Selected Files ({{ remoteFileItems.length }}) + + + + Add Selected Files ({{ selectionCount }}) - @@ -318,22 +854,20 @@ defineExpose({ startUpload }); - - + @@ -343,10 +877,10 @@ defineExpose({ startUpload }); color="grey" tooltip tooltip-placement="top" - title="Add more remote files to the upload list" + :title="addMoreFilesTitle" @click="showFileBrowser"> - Add More Files + {{ addMoreFilesLabel }} ({ startUpload }); tooltip tooltip-placement="top" title="Remove all files from the upload list" - @click="clearAll"> + @click="reset"> Clear All @@ -379,11 +913,43 @@ defineExpose({ startUpload }); flex-direction: column; overflow: hidden; min-height: 0; +} - .browser-staged-notice { - flex-shrink: 0; +.browser-header { + flex-shrink: 0; + @include upload-list-header; +} + +.search-bar-container { + flex-shrink: 0; +} + +.browser-table-container { + flex: 1; + overflow: auto; + min-height: 0; + + :deep(.browser-table thead) { + @include upload-table-header; + } + + :deep(tbody tr) { + cursor: pointer; + + &:hover { + background-color: rgba($brand-primary, 0.05); + } } } + +.browser-actions { + flex-shrink: 0; + display: flex; + gap: 0.5rem; + padding-top: 1rem; + border-top: 1px solid $border-color; +} + .file-list { width: 100%; height: 100%; diff --git a/client/src/components/Panels/Upload/types.ts b/client/src/components/Panels/Upload/types.ts index 50894b3e383..743b8308036 100644 --- a/client/src/components/Panels/Upload/types.ts +++ b/client/src/components/Panels/Upload/types.ts @@ -1,5 +1,9 @@ import type { IconDefinition } from "@fortawesome/fontawesome-svg-core"; +import type { UploadCollectionConfig } from "@/composables/upload/collectionTypes"; +import type { NewUploadItem } from "@/composables/upload/uploadItemTypes"; +import type { ApiUploadItem } from "@/utils/upload"; + export type UploadMethod = | "local-file" | "paste-content" @@ -100,6 +104,38 @@ export interface UploadMethodConfig { tips?: string[]; } +/** + * Upload data prepared by an upload method component, ready for API submission. + * Returned by `UploadMethodComponent.prepareUpload()` and consumed by `useUploadSubmission`. + */ +export interface PreparedUpload { + /** Items formatted for the Galaxy upload API. */ + apiItems: ApiUploadItem[]; + /** Optional dataset collection configuration for atomic collection creation. */ + collectionConfig?: UploadCollectionConfig; + /** Items used for progress tracking in the upload state store. */ + uploadItems?: NewUploadItem[]; + /** Optional upload behavior flags forwarded to uploadDatasets. */ + uploadOptions?: { + /** Treat multiple API items as one composite dataset. */ + composite?: boolean; + /** Display name for the composite dataset. */ + compositeName?: string; + }; +} + +/** + * Interface that upload method components must implement to be used in the upload panel. + */ export interface UploadMethodComponent { - startUpload: () => void; + /** + * Returns prepared upload data for API submission, or null when there is + * nothing to upload (e.g. no files selected or no content entered). + */ + prepareUpload: () => PreparedUpload | null; + /** + * Clears transient UI state without remounting the component. + * Called by the modal when it reopens so stale input is not carried over. + */ + reset?: () => void; } diff --git a/client/src/components/Panels/Upload/uploadMethodRegistry.ts b/client/src/components/Panels/Upload/uploadMethodRegistry.ts index 74eb23910c1..a557c109a3b 100644 --- a/client/src/components/Panels/Upload/uploadMethodRegistry.ts +++ b/client/src/components/Panels/Upload/uploadMethodRegistry.ts @@ -11,11 +11,13 @@ import { faSitemap, faTable, } from "@fortawesome/free-solid-svg-icons"; -import { computed, type ComputedRef, defineAsyncComponent } from "vue"; +import { computed, type ComputedRef, defineAsyncComponent, type Ref } from "vue"; +import { useConfig } from "@/composables/config"; import { useUploadAdvancedMode } from "@/composables/upload/uploadAdvancedMode"; import type { UploadMethod, UploadMethodConfig } from "./types"; +import type { UploadModalConfig } from "./uploadModalTypes"; export const uploadMethodRegistry: Record = { "local-file": { @@ -203,3 +205,27 @@ export function useAllUploadMethods(): ComputedRef { Object.values(uploadMethodRegistry).filter((method) => !method.requiresAdvancedMode || advancedMode.value), ); } + +/** + * Reactive list of upload methods filtered by modal config and Galaxy config requirements. + */ +export function useFilteredUploadMethods(config: Ref): ComputedRef { + const allMethods = useAllUploadMethods(); + const { config: galaxyConfig, isConfigLoaded } = useConfig(); + + return computed(() => { + const allowedMethods = config.value.allowedMethods; + + return allMethods.value.filter((method) => { + if (allowedMethods && !allowedMethods.some((allowedMethod) => allowedMethod === method.id)) { + return false; + } + + if (!isConfigLoaded.value || !method.requiresConfig) { + return true; + } + + return method.requiresConfig.every((configKey) => Boolean(galaxyConfig.value[configKey])); + }); + }); +} diff --git a/client/src/components/Panels/Upload/uploadModalTypes.ts b/client/src/components/Panels/Upload/uploadModalTypes.ts new file mode 100644 index 00000000000..0c3f7052689 --- /dev/null +++ b/client/src/components/Panels/Upload/uploadModalTypes.ts @@ -0,0 +1,57 @@ +import type { DataOption } from "@/components/Form/Elements/FormData/types"; +import type { UploadedDataset } from "@/composables/upload/uploadItemTypes"; + +import type { UploadMethod } from "./types"; + +export type { UploadedDataset } from "@/composables/upload/uploadItemTypes"; + +/** + * Subset of UploadMethod that directly creates new datasets in the history. + * These are the basic upload methods available in the upload modal. + */ +export type DatasetUploadMethod = Extract< + UploadMethod, + "local-file" | "paste-content" | "paste-links" | "remote-files" | "data-library" +>; + +/** + * Configuration options for the upload modal. + */ +export interface UploadModalConfig { + /** List of upload methods that users can select from. */ + allowedMethods?: DatasetUploadMethod[]; + /** Whether to allow creating dataset collections. */ + allowCollections?: boolean; + /** Allowed file format extensions. */ + formats?: string[]; + /** Whether multiple files can be uploaded at once. */ + multiple?: boolean; + /** Target history ID where datasets will be uploaded. */ + targetHistoryId?: string; + /** Custom title for the upload modal. */ + title?: string; + /** Whether to hide user tips/help text. */ + hideTips?: boolean; +} + +/** + * Result returned when the upload modal is closed. + */ +export interface UploadModalResult { + /** List of datasets that were uploaded. */ + datasets: UploadedDataset[]; + /** Whether the user cancelled the upload operation. */ + cancelled: boolean; + /** Function to convert uploaded datasets to form data options. */ + toDataOptions: () => DataOption[]; +} + +/** + * Promise resolvers for handling the upload modal result. + */ +export interface UploadModalResolvers { + /** Resolve the promise with the upload result. */ + resolve: (result: UploadModalResult) => void; + /** Reject the promise if the upload fails. */ + reject: (reason?: unknown) => void; +} diff --git a/client/src/components/Panels/Upload/uploadState.test.ts b/client/src/components/Panels/Upload/uploadState.test.ts index b74fe159190..6651e67fcf0 100644 --- a/client/src/components/Panels/Upload/uploadState.test.ts +++ b/client/src/components/Panels/Upload/uploadState.test.ts @@ -1,8 +1,8 @@ import { suppressExpectedErrorMessages } from "@tests/vitest/helpers"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import type { UploadCollectionConfig } from "@/composables/upload/collectionTypes"; import type { NewUploadItem } from "@/composables/upload/uploadItemTypes"; -import type { CollectionConfig } from "@/composables/uploadQueue"; import { useUploadState } from "./uploadState"; @@ -24,7 +24,7 @@ function makePastedItem(name = "file.txt", content = "hello world"): NewUploadIt }; } -const BATCH_CONFIG: CollectionConfig = { +const BATCH_CONFIG: UploadCollectionConfig = { name: "My Collection", type: "list", hideSourceItems: false, diff --git a/client/src/components/Panels/Upload/uploadState.ts b/client/src/components/Panels/Upload/uploadState.ts index 0b5a15e2435..5f5200c22ee 100644 --- a/client/src/components/Panels/Upload/uploadState.ts +++ b/client/src/components/Panels/Upload/uploadState.ts @@ -46,6 +46,15 @@ export interface UploadFileListItem extends UploadListItemBase { /** Union of all upload list UI items */ export type UploadListItem = UploadBatchListItem | UploadFileListItem; +/** + * Subset of the upload state API used by upload tracking utilities. + * Provides only the functions needed for tracking and managing upload progress. + */ +export type UploadStateTrackingApi = Pick< + ReturnType, + "addBatch" | "addUploadItem" | "getBatch" | "setStatus" | "updateProgress" | "setError" +>; + /** Collection batch state tracking */ export interface CollectionBatchState { /** Unique batch identifier */ diff --git a/client/src/components/Upload/DefaultBox.vue b/client/src/components/Upload/DefaultBox.vue index e57b13941ec..ce145e52d80 100644 --- a/client/src/components/Upload/DefaultBox.vue +++ b/client/src/components/Upload/DefaultBox.vue @@ -46,6 +46,7 @@ interface Props { disableFooter?: boolean; emitUploaded?: boolean; size?: ComponentSize; + showBetaUpload?: boolean; } const props = withDefaults(defineProps(), { @@ -54,6 +55,7 @@ const props = withDefaults(defineProps(), { lazyLoad: 150, size: "medium", isCollection: false, + showBetaUpload: true, }); const emit = defineEmits<{ @@ -495,6 +497,7 @@ defineExpose({ }"> void } | null>) { const collectionState = ref({ @@ -27,9 +25,9 @@ export function useCollectionCreation(collectionConfigComponent?: Ref<{ reset: ( } /** - * Builds a CollectionConfig for the upload queue if collection creation is active + * Builds an UploadCollectionConfig when collection creation is active. */ - function buildCollectionConfig(targetHistoryId: string): CollectionConfig | undefined { + function buildCollectionConfig(targetHistoryId: string): UploadCollectionConfig | undefined { if (!collectionState.value.config) { return undefined; } diff --git a/client/src/composables/upload/collectionElements.ts b/client/src/composables/upload/collectionElements.ts new file mode 100644 index 00000000000..ce1edeee292 --- /dev/null +++ b/client/src/composables/upload/collectionElements.ts @@ -0,0 +1,92 @@ +import type { CollectionElementIdentifiers } from "@/api"; +import { + COMMON_FILTERS, + DEFAULT_FILTER, + guessInitialFilterType, + guessNameForPair, +} from "@/components/Collections/pairing"; + +import type { SupportedCollectionType } from "./collectionTypes"; + +interface CollectionUploadItem { + name: string; +} + +/** + * Builds collection element identifiers based on collection type. + * Uses the shared pairing abstractions from @/components/Collections/pairing for + * pair name extraction (synchronized with the backend via auto_pairing_spec.yml). + * + * @param items - Upload items with dataset IDs + * @param datasetIds - Array of created dataset IDs (in upload order) + * @param collectionType - Type of collection to create + * @returns Collection element identifiers ready for API + */ +export function buildCollectionElements( + items: CollectionUploadItem[], + datasetIds: string[], + collectionType: SupportedCollectionType, +): CollectionElementIdentifiers { + if (collectionType === "list") { + // Simple list: one element per dataset + return items.map((item, index) => ({ + name: item.name || `element_${index + 1}`, + src: "hda" as const, + id: datasetIds[index], + })); + } + // List of pairs: group consecutive files into pairs + const pairs: CollectionElementIdentifiers = []; + const usedNames = new Set(); + + // Use the shared filter detection to determine forward/reverse naming convention + const filterType = guessInitialFilterType(items) ?? DEFAULT_FILTER; + const [forwardFilter, reverseFilter] = COMMON_FILTERS[filterType]; + + for (let i = 0; i < items.length; i += 2) { + if (i + 1 >= items.length) { + // Odd number of files - skip last one or handle as error + console.warn(`Skipping unpaired file at index ${i}: ${items[i]?.name}`); + break; + } + + const item1 = items[i]; + const item2 = items[i + 1]; + + if (!item1 || !item2) { + continue; + } + + const basePairName = + guessNameForPair(item1, item2, forwardFilter, reverseFilter, true) || `pair_${Math.floor(i / 2) + 1}`; + let pairName = basePairName; + + // Ensure unique pair names by adding suffix if needed + let counter = 1; + while (usedNames.has(pairName)) { + pairName = `${basePairName}_${counter}`; + counter++; + } + usedNames.add(pairName); + + pairs.push({ + collection_type: "paired", + src: "new_collection" as const, + name: pairName, + element_identifiers: [ + { + name: "forward", + src: "hda" as const, + id: datasetIds[i], + }, + { + name: "reverse", + src: "hda" as const, + id: datasetIds[i + 1], + }, + ], + }); + } + + return pairs; +} diff --git a/client/src/composables/upload/collectionTypes.ts b/client/src/composables/upload/collectionTypes.ts index 3868565a360..ae6b26b95db 100644 --- a/client/src/composables/upload/collectionTypes.ts +++ b/client/src/composables/upload/collectionTypes.ts @@ -18,6 +18,16 @@ export interface CollectionCreationInput { type: SupportedCollectionType; } +/** + * Collection configuration used by upload orchestration. + */ +export interface UploadCollectionConfig extends CollectionCreationInput { + /** Whether to hide source datasets after collection creation */ + hideSourceItems: boolean; + /** Target history ID where the collection will be created */ + historyId: string; +} + /** * Validation state for a collection configuration */ diff --git a/client/src/composables/upload/testHelpers/uploadFixtures.ts b/client/src/composables/upload/testHelpers/uploadFixtures.ts new file mode 100644 index 00000000000..7d0263aefda --- /dev/null +++ b/client/src/composables/upload/testHelpers/uploadFixtures.ts @@ -0,0 +1,102 @@ +import type { UploadCollectionConfig } from "@/composables/upload/collectionTypes"; +import type { + LibraryDatasetUploadItem, + LocalFileUploadItem, + PastedContentUploadItem, + RemoteFileUploadItem, + UrlUploadItem, +} from "@/composables/upload/uploadItemTypes"; + +export function makePastedItem(overrides: Partial = {}): PastedContentUploadItem { + return { + uploadMode: "paste-content", + name: "file.txt", + content: "hello world", + size: 11, + targetHistoryId: "hist_1", + dbkey: "?", + extension: "auto", + spaceToTab: false, + toPosixLines: false, + deferred: false, + ...overrides, + }; +} + +export function makeUrlItem(overrides: Partial = {}): UrlUploadItem { + return { + uploadMode: "paste-links", + name: "file.txt", + url: "http://example.com/file.txt", + size: 0, + targetHistoryId: "hist_1", + dbkey: "?", + extension: "auto", + spaceToTab: false, + toPosixLines: false, + deferred: false, + ...overrides, + }; +} + +export function makeRemoteFilesItem(overrides: Partial = {}): RemoteFileUploadItem { + return { + uploadMode: "remote-files", + name: "file.txt", + url: "ftp://server/file.txt", + size: 0, + targetHistoryId: "hist_1", + dbkey: "?", + extension: "auto", + spaceToTab: false, + toPosixLines: false, + deferred: false, + ...overrides, + }; +} + +export function makeLibraryItem(overrides: Partial = {}): LibraryDatasetUploadItem { + return { + uploadMode: "data-library", + name: "library.txt", + size: 0, + targetHistoryId: "hist_1", + dbkey: "?", + extension: "auto", + spaceToTab: false, + toPosixLines: false, + deferred: false, + libraryId: "lib_1", + folderId: "folder_1", + lddaId: "ldda_1", + url: "/api/libraries/lib_1/datasets/ldda_1", + ...overrides, + }; +} + +export function makeLocalFileItem(overrides: Partial = {}): LocalFileUploadItem { + const file = new File(["content"], "test.txt"); + return { + uploadMode: "local-file", + name: "test.txt", + size: file.size, + targetHistoryId: "hist_1", + dbkey: "?", + extension: "auto", + spaceToTab: false, + toPosixLines: false, + deferred: false, + fileData: file, + ...overrides, + }; +} + +export function makeCollectionConfig(overrides: Partial = {}): UploadCollectionConfig { + return { + name: "My Collection", + type: "list", + historyId: "hist_1", + hideSourceItems: false, + ...overrides, + }; +} diff --git a/client/src/composables/upload/uploadDefaults.ts b/client/src/composables/upload/uploadDefaults.ts index c78baa2f0c1..74bc5aa3428 100644 --- a/client/src/composables/upload/uploadDefaults.ts +++ b/client/src/composables/upload/uploadDefaults.ts @@ -8,9 +8,9 @@ import { computed } from "vue"; import type { BaseUploadItem } from "@/components/Panels/Upload/types/uploadItem"; import { useUploadConfigurations } from "@/composables/uploadConfigurations"; -export function useUploadDefaults() { +export function useUploadDefaults(formats?: string[]) { const { configOptions, effectiveExtensions, compositeExtensions, listDbKeys, ready } = - useUploadConfigurations(undefined); + useUploadConfigurations(formats); const defaultExtension = computed(() => configOptions.value?.defaultExtension || "auto"); const defaultDbKey = computed(() => configOptions.value?.defaultDbKey || "?"); diff --git a/client/src/composables/upload/uploadItemTypes.test.ts b/client/src/composables/upload/uploadItemTypes.test.ts new file mode 100644 index 00000000000..0c1493191cf --- /dev/null +++ b/client/src/composables/upload/uploadItemTypes.test.ts @@ -0,0 +1,77 @@ +import { describe, expect, it } from "vitest"; + +import { + makeLibraryItem, + makeLocalFileItem, + makePastedItem, + makeRemoteFilesItem, + makeUrlItem, +} from "@/composables/upload/testHelpers/uploadFixtures"; +import type { NewUploadItem } from "@/composables/upload/uploadItemTypes"; + +import { validateUploadItem } from "./uploadItemTypes"; + +describe("validateUploadItem", () => { + it.each([ + ["paste-content", makePastedItem()], + ["paste-links", makeUrlItem()], + ["remote-files", makeRemoteFilesItem()], + ["data-library", makeLibraryItem()], + ["local-file", makeLocalFileItem()], + ] as [string, NewUploadItem][])("accepts a valid %s item", (_mode, item) => { + expect(validateUploadItem(item)).toBeUndefined(); + }); + + it("rejects paste-content with empty content", () => { + expect(validateUploadItem(makePastedItem({ content: " " }))).toMatch(/No content provided/); + }); + + it("rejects paste-links with missing URL", () => { + expect(validateUploadItem(makeUrlItem({ url: "" }))).toMatch(/No URL provided/); + }); + + it("rejects remote-files with missing URL", () => { + expect(validateUploadItem(makeRemoteFilesItem({ url: " " }))).toMatch(/No URL provided/); + }); + + it("rejects data-library with no lddaId", () => { + expect(validateUploadItem(makeLibraryItem({ lddaId: "" }))).toMatch(/No library dataset ID/); + }); + + it("rejects local-file with no file data", () => { + const item: NewUploadItem = { + uploadMode: "local-file", + name: "missing.txt", + size: 0, + targetHistoryId: "hist_1", + dbkey: "?", + extension: "auto", + spaceToTab: false, + toPosixLines: false, + deferred: false, + }; + expect(validateUploadItem(item)).toMatch(/No file selected/); + }); + + it("rejects local-file with an empty file", () => { + const emptyFile = new File([], "empty.txt"); + const item: NewUploadItem = { + uploadMode: "local-file", + name: "empty.txt", + size: 0, + targetHistoryId: "hist_1", + dbkey: "?", + extension: "auto", + spaceToTab: false, + toPosixLines: false, + deferred: false, + fileData: emptyFile, + }; + expect(validateUploadItem(item)).toMatch(/is empty/); + }); + + it("rejects an unknown upload mode", () => { + const item = { ...makePastedItem(), uploadMode: "unknown-mode" } as unknown as NewUploadItem; + expect(validateUploadItem(item)).toMatch(/Unknown upload mode/); + }); +}); diff --git a/client/src/composables/upload/uploadItemTypes.ts b/client/src/composables/upload/uploadItemTypes.ts index 17ecbba50a3..ddb6a95006e 100644 --- a/client/src/composables/upload/uploadItemTypes.ts +++ b/client/src/composables/upload/uploadItemTypes.ts @@ -3,6 +3,7 @@ * These types represent items in various stages of the upload lifecycle. */ +import type { HistoryContentSource } from "@/api/datasets"; import type { FetchDatasetHash } from "@/api/tools"; import type { UploadMethod } from "@/components/Panels/Upload/types"; @@ -123,3 +124,60 @@ export interface UploadState { /** Upload item with state tracking (used in active upload queue) */ export type UploadItem = NewUploadItem & UploadState; + +/** Sources returned for uploaded history contents. */ +export type UploadedDatasetSource = Extract; + +/** + * Validates a UI upload item before submission. + * Returns an error message if invalid, undefined if valid. + */ +export function validateUploadItem(item: NewUploadItem): string | undefined { + switch (item.uploadMode) { + case "local-file": + if (!item.fileData) { + return `No file selected for "${item.name}"`; + } + if (item.fileData.size === 0) { + return `File "${item.name}" is empty`; + } + break; + + case "paste-content": + if (!item.content || item.content.trim().length === 0) { + return `No content provided for "${item.name}"`; + } + break; + + case "paste-links": + case "remote-files": + if (!item.url || item.url.trim().length === 0) { + return `No URL provided for "${item.name}"`; + } + break; + + case "data-library": + if (!item.lddaId) { + return `No library dataset ID provided for "${item.name}"`; + } + break; + + default: + return `Unknown upload mode: ${(item as NewUploadItem).uploadMode}`; + } + return undefined; +} + +/** + * Represents a dataset that was successfully uploaded. + */ +export interface UploadedDataset { + /** Unique identifier for the dataset. */ + id: string; + /** Display name of the dataset. */ + name: string; + /** History item ID (sequential index). */ + hid?: number; + /** Type of dataset: history dataset (hda) or history dataset collection (hdca). */ + src: UploadedDatasetSource; +} diff --git a/client/src/composables/upload/uploadResponse.ts b/client/src/composables/upload/uploadResponse.ts new file mode 100644 index 00000000000..7c27592e136 --- /dev/null +++ b/client/src/composables/upload/uploadResponse.ts @@ -0,0 +1,91 @@ +import type { FetchDataResponse } from "@/api/tools"; + +import type { UploadedDataset } from "./uploadItemTypes"; + +/** + * Raw dataset data structure from API response. + */ +interface UploadResponseData { + id: string; + name?: string; + label?: string; + hid?: number; + src?: string; +} + +function isUploadResponseData(value: unknown): value is UploadResponseData { + if (!value || typeof value !== "object") { + return false; + } + return "id" in value && typeof value.id === "string"; +} + +function toUploadedDataset(output: unknown): UploadedDataset | null { + if (!isUploadResponseData(output) || !output.id) { + return null; + } + + return { + id: output.id, + name: output.name ?? output.label ?? output.id, + hid: output.hid, + src: output.src === "hdca" ? "hdca" : "hda", + }; +} + +function isPlainObject(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function collectUploadedDatasets(responseOutputs: unknown, datasets: UploadedDataset[]): void { + if (!responseOutputs) { + return; + } + + if (Array.isArray(responseOutputs)) { + responseOutputs.forEach((item) => collectUploadedDatasets(item, datasets)); + return; + } + + const converted = toUploadedDataset(responseOutputs); + if (converted) { + datasets.push(converted); + return; + } + + if (isPlainObject(responseOutputs)) { + Object.values(responseOutputs).forEach((nested) => collectUploadedDatasets(nested, datasets)); + } +} + +/** + * Extracts uploaded datasets from a fetch response, recursively searching through outputs. + * @param response - The fetch response containing outputs + * @returns Array of unique uploaded datasets + */ +export function datasetsFromFetchResponse(response: FetchDataResponse): UploadedDataset[] { + if (!response.outputs) { + return []; + } + + const datasets: UploadedDataset[] = []; + collectUploadedDatasets(response.outputs, datasets); + + const seen = new Set(); + return datasets.filter((dataset) => { + if (seen.has(dataset.id)) { + return false; + } + seen.add(dataset.id); + return true; + }); +} + +/** + * Extracts dataset IDs from a fetch response. + * @param response - The fetch response containing outputs + * @returns Array of dataset IDs + */ +export function datasetIdsFromFetchResponse(response: FetchDataResponse): string[] { + return datasetsFromFetchResponse(response).map((dataset) => dataset.id); +} diff --git a/client/src/composables/upload/uploadTracking.ts b/client/src/composables/upload/uploadTracking.ts new file mode 100644 index 00000000000..c605c943e96 --- /dev/null +++ b/client/src/composables/upload/uploadTracking.ts @@ -0,0 +1,145 @@ +import type { UploadStateTrackingApi } from "@/components/Panels/Upload/uploadState"; + +import type { UploadCollectionConfig } from "./collectionTypes"; +import type { LibraryDatasetUploadItem, NewUploadItem } from "./uploadItemTypes"; + +/** + * Represents an upload item being tracked through its lifecycle. + * @template T - The specific type of upload item (NewUploadItem or subtypes) + */ +export interface TrackedUpload { + /** The upload item configuration */ + item: T; + /** Unique identifier for this tracked upload */ + id: string; +} + +/** + * Result of initializing tracked uploads. + * Contains the tracked uploads and optional batch ID for collection uploads. + */ +export interface InitializedUploads { + /** Array of tracked uploads created from the input items */ + trackedUploads: TrackedUpload[]; + /** Batch ID if uploads were initialized as part of a collection */ + batchId?: string; +} + +/** + * Options for initializing tracked uploads. + */ +interface InitializeTrackedUploadsOptions { + /** Collection configuration if uploads should be part of a collection */ + collectionConfig?: UploadCollectionConfig; + /** Whether to use direct HDCA creation (no separate collection creation step) */ + directCreation?: boolean; + /** Whether to immediately start uploading (sets status to "uploading") */ + startUploading?: boolean; +} + +/** + * Initializes tracked uploads for a set of upload items. + * Creates upload items in the upload state and optionally creates a collection batch. + * + * @param uploadState - Upload state tracking API + * @param items - Upload items to initialize (file, URL, paste uploads, etc.) + * @param options - Initialization options + * @returns Initialized uploads with tracked items and optional batch ID + */ +export function initializeTrackedUploads( + uploadState: UploadStateTrackingApi, + items: NewUploadItem[] | undefined, + options: InitializeTrackedUploadsOptions = {}, +): InitializedUploads { + if (!items || items.length === 0) { + return { trackedUploads: [] }; + } + + const batchId = options.collectionConfig + ? uploadState.addBatch(options.collectionConfig, [], options.directCreation ?? false) + : undefined; + + const trackedUploads = items.map((item) => ({ + item, + id: uploadState.addUploadItem(item, batchId), + })); + + if (batchId) { + const batch = uploadState.getBatch(batchId); + if (batch) { + batch.uploadIds = trackedUploads.map((tracked) => tracked.id); + } + } + + if (options.startUploading) { + trackedUploads.forEach((tracked) => uploadState.setStatus(tracked.id, "uploading")); + } + + return { trackedUploads, batchId }; +} + +/** + * Updates the progress of multiple tracked uploads. + * + * @param uploadState - Upload state tracking API + * @param ids - Upload item IDs to update + * @param percentage - Progress percentage (0-100) + */ +export function updateTrackedProgress(uploadState: UploadStateTrackingApi, ids: string[], percentage: number): void { + ids.forEach((id) => uploadState.updateProgress(id, percentage)); +} + +/** + * Marks multiple tracked uploads as completed. + * Sets progress to 100% and status to "completed". + */ +export function markTrackedCompleted(uploadState: UploadStateTrackingApi, ids: string[]): void { + ids.forEach((id) => { + uploadState.updateProgress(id, 100); + uploadState.setStatus(id, "completed"); + }); +} + +/** + * Marks multiple tracked uploads as failed with an error message. + * + * @param uploadState - Upload state tracking API + * @param trackedUploads - Tracked uploads to mark as failed + * @param message - Error message describing the failure + */ +export function markTrackedError( + uploadState: UploadStateTrackingApi, + trackedUploads: Array>, + message: string, +): void { + trackedUploads.forEach((tracked) => uploadState.setError(tracked.id, message)); +} + +function isTrackedLibraryUpload(tracked: TrackedUpload): tracked is TrackedUpload { + return tracked.item.uploadMode === "data-library"; +} + +/** + * Splits tracked uploads by their type (API uploads vs library uploads). + * Useful for handling different upload modes after initialization. + * + * @param trackedUploads - Tracked uploads to split + * @returns Object containing arrays of API upload IDs and library uploads + */ +export function splitTrackedUploadsByType(trackedUploads: TrackedUpload[]): { + apiIds: string[]; + libraryUploads: TrackedUpload[]; +} { + const apiIds: string[] = []; + const libraryUploads: TrackedUpload[] = []; + + trackedUploads.forEach((tracked) => { + if (isTrackedLibraryUpload(tracked)) { + libraryUploads.push(tracked); + } else { + apiIds.push(tracked.id); + } + }); + + return { apiIds, libraryUploads }; +} diff --git a/client/src/composables/upload/useUploadBatchOperations.test.ts b/client/src/composables/upload/useUploadBatchOperations.test.ts new file mode 100644 index 00000000000..60404ee7c36 --- /dev/null +++ b/client/src/composables/upload/useUploadBatchOperations.test.ts @@ -0,0 +1,107 @@ +import { suppressExpectedErrorMessages } from "@tests/vitest/helpers"; +import flushPromises from "flush-promises"; +import { http, HttpResponse } from "msw"; +import { createPinia, setActivePinia } from "pinia"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +import { useServerMock } from "@/api/client/__mocks__"; +import { useUploadState } from "@/components/Panels/Upload/uploadState"; +import { makeCollectionConfig, makePastedItem, makeUrlItem } from "@/composables/upload/testHelpers/uploadFixtures"; + +import { useUploadBatchOperations } from "./useUploadBatchOperations"; + +const { server } = useServerMock(); + +describe("useUploadBatchOperations", () => { + beforeEach(() => { + setActivePinia(createPinia()); + useUploadState().clearAll(); + }); + + afterEach(() => { + vi.restoreAllMocks(); + useUploadState().clearAll(); + }); + + it("processes the direct collection path atomically", async () => { + server.use( + http.get("/api/configuration", () => HttpResponse.json({ chunk_upload_size: 42 })), + http.post("/api/tools/fetch", async ({ request }) => { + const body = await request.json(); + expect(body).toMatchObject({ + history_id: "hist_1", + targets: [ + { + destination: { type: "hdca" }, + collection_type: "list", + name: "My Collection", + }, + ], + }); + return HttpResponse.json({ outputs: [{ id: "hdca_1", src: "hdca" }] }); + }), + ); + + const state = useUploadState(); + const operations = useUploadBatchOperations({ autoRecover: false }); + const first = makeUrlItem({ name: "a.txt" }); + const second = makeUrlItem({ name: "b.txt" }); + const batchId = state.addBatch(makeCollectionConfig(), [], true); + const id1 = state.addUploadItem(first, batchId); + const id2 = state.addUploadItem(second, batchId); + state.getBatch(batchId)!.uploadIds = [id1, id2]; + + await operations.processDirectBatch(batchId, [id1, id2], [first, second]); + + expect(state.getBatch(batchId)?.status).toBe("completed"); + expect(state.activeItems.value.every((item) => item.status === "completed")).toBe(true); + }); + + it("retries collection creation after an earlier two-step failure", async () => { + suppressExpectedErrorMessages(["Temporary error"]); + server.use(http.post("/api/dataset_collections", () => HttpResponse.json({ id: "col_retried" }))); + + const state = useUploadState(); + const operations = useUploadBatchOperations({ autoRecover: false }); + const uploadId = state.addUploadItem(makePastedItem()); + state.updateProgress(uploadId, 100); + + const batchId = state.addBatch(makeCollectionConfig(), [uploadId], false); + state.addBatchDatasetId(batchId, "ds_1"); + state.setBatchError(batchId, "Temporary error"); + + const item = state.activeItems.value.find((entry) => entry.id === uploadId); + if (item) { + item.error = "Uploaded successfully, but collection creation failed"; + } + + await operations.retryCollectionCreation(batchId); + + expect(state.getBatch(batchId)?.status).toBe("completed"); + expect(state.getBatch(batchId)?.collectionId).toBe("col_retried"); + expect(state.activeItems.value.find((entry) => entry.id === uploadId)?.error).toBeUndefined(); + }); + + it("recovers interrupted two-step collection creation from persisted state", async () => { + server.use(http.post("/api/dataset_collections", () => HttpResponse.json({ id: "col_recovered" }))); + + const state = useUploadState(); + const itemId = state.addUploadItem(makePastedItem({ name: "recovered.txt" })); + state.setStatus(itemId, "uploading"); + state.updateProgress(itemId, 100); + + const batchId = state.addBatch( + { name: "Recovery Collection", type: "list", hideSourceItems: false, historyId: "hist_1" }, + [itemId], + false, + ); + state.addBatchDatasetId(batchId, "ds_recovered"); + + const operations = useUploadBatchOperations({ autoRecover: false }); + operations.recoverIncompleteBatches(); + await flushPromises(); + + expect(state.getBatch(batchId)?.collectionId).toBe("col_recovered"); + expect(state.getBatch(batchId)?.status).toBe("completed"); + }); +}); diff --git a/client/src/composables/upload/useUploadBatchOperations.ts b/client/src/composables/upload/useUploadBatchOperations.ts new file mode 100644 index 00000000000..741bbcf1efc --- /dev/null +++ b/client/src/composables/upload/useUploadBatchOperations.ts @@ -0,0 +1,255 @@ +import { createHistoryDatasetCollectionInstanceFull } from "@/api/datasetCollections"; +import { useUploadState } from "@/components/Panels/Upload/uploadState"; +import { buildCollectionElements } from "@/composables/upload/collectionElements"; +import type { NewUploadItem, UploadItem } from "@/composables/upload/uploadItemTypes"; +import { validateUploadItem } from "@/composables/upload/uploadItemTypes"; +import { useHistoryStore } from "@/stores/historyStore"; +import { getHistoryUploadActionErrorMessage, getHistoryUploadBlockReason } from "@/utils/historyUpload"; +import { errorMessageAsString } from "@/utils/simple-error"; +import { toApiUploadItem, uploadCollectionDatasets } from "@/utils/upload"; + +interface UploadBatchOperationsOptions { + autoRecover?: boolean; +} + +let didRecoverIncompleteBatches = false; + +export function useUploadBatchOperations(options: UploadBatchOperationsOptions = {}) { + const uploadState = useUploadState(); + + function findUploadItem(id: string): UploadItem | undefined { + return uploadState.activeItems.value.find((item) => item.id === id); + } + + async function validateTargetHistory(targetHistoryId: string): Promise { + const historyStore = useHistoryStore(); + let history = historyStore.getHistoryById(targetHistoryId, false) ?? null; + if (!history) { + await historyStore.loadHistoryById(targetHistoryId); + history = historyStore.getHistoryById(targetHistoryId, false) ?? null; + } + if (!history) { + return null; + } + const blockReason = getHistoryUploadBlockReason(history); + return blockReason ? getHistoryUploadActionErrorMessage(blockReason) : null; + } + + /** + * Creates a dataset collection from uploaded datasets. + * + * @param batchId - Batch ID in upload state + * @throws {Error} If the batch is not found, has missing data, or collection creation fails + */ + async function createCollection(batchId: string): Promise { + const batch = uploadState.getBatch(batchId); + if (!batch) { + const errorMsg = `Batch not found: ${batchId}`; + console.error(errorMsg); + throw new Error(errorMsg); + } + + if (batch.collectionId) { + return; + } + + if (!batch.datasetIds || batch.datasetIds.length === 0) { + const errorMsg = "No dataset IDs available for collection creation"; + uploadState.setBatchError(batchId, errorMsg); + throw new Error(errorMsg); + } + + const items = batch.uploadIds + .map((id) => findUploadItem(id)) + .filter((item): item is UploadItem => item !== undefined); + + if (items.length === 0) { + const errorMsg = "No upload items available for collection creation"; + uploadState.setBatchError(batchId, errorMsg); + throw new Error(errorMsg); + } + + if (items.length !== batch.uploadIds.length) { + const errorMsg = `Cannot create collection: only ${items.length} of ${batch.uploadIds.length} upload items found. This can happen after a page refresh. Please re-upload the files or manually create the collection.`; + uploadState.setBatchError(batchId, errorMsg); + throw new Error(errorMsg); + } + + uploadState.updateBatchStatus(batchId, "creating-collection"); + + try { + const elementIdentifiers = buildCollectionElements(items, batch.datasetIds, batch.type); + + if (elementIdentifiers.length === 0) { + throw new Error("No valid collection elements to create"); + } + + const response = await createHistoryDatasetCollectionInstanceFull({ + name: batch.name, + collection_type: batch.type, + element_identifiers: elementIdentifiers, + history_id: batch.historyId, + hide_source_items: batch.hideSourceItems, + instance_type: "history", + copy_elements: true, + fields: "auto", + }); + + uploadState.setBatchCollectionId(batchId, response.id); + uploadState.updateBatchStatus(batchId, "completed"); + } catch (err) { + const errorMsg = errorMessageAsString(err); + uploadState.setBatchError(batchId, errorMsg); + + const currentBatch = uploadState.getBatch(batchId); + currentBatch?.uploadIds.forEach((id) => { + const item = findUploadItem(id); + if (item && !item.error) { + item.error = "Uploaded successfully, but collection creation failed"; + } + }); + + throw new Error(errorMsg); + } + } + + /** + * Processes a collection batch using the direct HDCA creation path. + * All items are uploaded together in one /api/tools/fetch request that + * creates the collection atomically — no separate collection creation step. + * + * Used for non-library batches where all items can be fed to the upload API directly. + */ + async function processDirectBatch(batchId: string, ids: string[], items: NewUploadItem[]): Promise { + const batch = uploadState.getBatch(batchId); + if (!batch) { + console.error(`Batch not found: ${batchId}`); + return; + } + + ids.forEach((id) => uploadState.setStatus(id, "uploading")); + uploadState.updateBatchStatus(batchId, "uploading"); + + try { + const historyError = await validateTargetHistory(batch.historyId); + if (historyError) { + throw new Error(historyError); + } + + for (const item of items) { + const validationError = validateUploadItem(item); + if (validationError) { + throw new Error(validationError); + } + } + + const apiItems = items.map((item) => toApiUploadItem(item)); + + await uploadCollectionDatasets( + apiItems, + { + collectionName: batch.name, + collectionType: batch.type, + }, + { + progress: (percentage) => { + ids.forEach((id) => uploadState.updateProgress(id, percentage)); + }, + success: () => { + ids.forEach((id) => uploadState.updateProgress(id, 100)); + uploadState.updateBatchStatus(batchId, "completed"); + }, + error: (err) => { + const errorMsg = errorMessageAsString(err); + ids.forEach((id) => uploadState.setError(id, errorMsg)); + uploadState.setBatchError(batchId, errorMsg); + }, + }, + ); + } catch (err) { + const errorMsg = errorMessageAsString(err); + ids.forEach((id) => uploadState.setError(id, errorMsg)); + uploadState.setBatchError(batchId, errorMsg); + } + } + + async function retryCollectionCreation(batchId: string): Promise { + const batch = uploadState.getBatch(batchId); + if (!batch) { + console.error(`Batch not found: ${batchId}`); + return; + } + + batch.error = undefined; + uploadState.updateBatchStatus(batchId, "uploading"); + + batch.uploadIds.forEach((id) => { + const item = findUploadItem(id); + if (item?.error?.includes("collection creation failed")) { + item.error = undefined; + } + }); + + try { + await createCollection(batchId); + } catch (err) { + uploadState.setBatchError(batchId, `Retry failed: ${errorMessageAsString(err)}`); + } + } + + function recoverIncompleteBatches(): void { + uploadState.activeBatches.value.forEach((batch) => { + if (batch.collectionId || batch.status === "error" || batch.directCreation) { + return; + } + + const allCompleted = batch.uploadIds.every((uploadId) => { + const upload = findUploadItem(uploadId); + return upload?.status === "completed"; + }); + + if (allCompleted && batch.uploadIds.length > 0 && batch.datasetIds.length > 0) { + const availableItems = batch.uploadIds.filter((uploadId) => findUploadItem(uploadId) !== undefined); + if (availableItems.length !== batch.uploadIds.length) { + uploadState.setBatchError( + batch.id, + "Collection creation failed: upload data lost after page refresh. Please re-upload the files to create the collection or manually create the collection.", + ); + return; + } + + createCollection(batch.id).catch((err) => { + console.error("Recovery failed:", err); + uploadState.setBatchError(batch.id, "Collection creation interrupted. Please retry manually."); + }); + } else if (allCompleted && batch.uploadIds.length > 0) { + uploadState.setBatchError( + batch.id, + "Collection creation interrupted and cannot be recovered. Dataset IDs not available. Please create the collection manually.", + ); + } + }); + } + + function clearCompleted(): void { + uploadState.clearCompleted(); + } + + function clearAll(): void { + uploadState.clearAll(); + } + + if (options.autoRecover !== false && !didRecoverIncompleteBatches) { + recoverIncompleteBatches(); + didRecoverIncompleteBatches = true; + } + + return { + clearAll, + clearCompleted, + createCollection, + processDirectBatch, + recoverIncompleteBatches, + retryCollectionCreation, + }; +} diff --git a/client/src/composables/upload/useUploadMethodModal.ts b/client/src/composables/upload/useUploadMethodModal.ts new file mode 100644 index 00000000000..7c331f33d9c --- /dev/null +++ b/client/src/composables/upload/useUploadMethodModal.ts @@ -0,0 +1,159 @@ +import Vue, { h, reactive } from "vue"; + +import type { DataOption } from "@/components/Form/Elements/FormData/types"; +import type { + DatasetUploadMethod, + UploadedDataset, + UploadModalConfig, + UploadModalResolvers, + UploadModalResult, +} from "@/components/Panels/Upload/uploadModalTypes"; + +import UploadMethodModal from "@/components/Panels/Upload/UploadMethodModal.vue"; + +interface ModalState { + modalVisible: boolean; + modalConfig: UploadModalConfig; +} + +interface UploadMethodModalHostInstance extends Vue { + state: ModalState; + render: () => ReturnType; +} + +const DEFAULT_ALLOWED_METHODS: DatasetUploadMethod[] = [ + "local-file", + "paste-content", + "paste-links", + "remote-files", + "data-library", +]; + +let hostElement: HTMLDivElement | null = null; +let modalVm: UploadMethodModalHostInstance | null = null; +let pendingResolvers: UploadModalResolvers | null = null; + +function toDataOptions(datasets: UploadedDataset[]): DataOption[] { + return datasets.map((dataset) => ({ + id: dataset.id, + name: dataset.name, + hid: dataset.hid, + src: dataset.src, + batch: false, + keep: true, + tags: [], + })); +} + +function applyConfigDefaults(config?: UploadModalConfig): UploadModalConfig { + return { + allowedMethods: config?.allowedMethods ?? DEFAULT_ALLOWED_METHODS, + allowCollections: config?.allowCollections ?? false, + formats: config?.formats, + multiple: config?.multiple ?? true, + targetHistoryId: config?.targetHistoryId, + title: config?.title, + hideTips: config?.hideTips ?? false, + }; +} + +function buildResult(datasets: UploadedDataset[], cancelled: boolean): UploadModalResult { + return { + datasets, + cancelled, + toDataOptions: () => toDataOptions(datasets), + }; +} + +function resolveAndCleanup(resolvers: UploadModalResolvers | null, result: UploadModalResult): void { + if (resolvers) { + resolvers.resolve(result); + } +} + +function ensureMounted(): UploadMethodModalHostInstance { + if (modalVm) { + return modalVm; + } + + hostElement = document.createElement("div"); + hostElement.id = "upload-method-modal-host"; + document.body.appendChild(hostElement); + + const state: ModalState = reactive({ + modalVisible: false, + modalConfig: applyConfigDefaults(), + }); + + const finishUploaded = (datasets: UploadedDataset[]) => { + const resolvers = pendingResolvers; + pendingResolvers = null; + state.modalVisible = false; + resolveAndCleanup(resolvers, buildResult(datasets, false)); + }; + + const finishCancelled = () => { + const resolvers = pendingResolvers; + pendingResolvers = null; + state.modalVisible = false; + resolveAndCleanup(resolvers, buildResult([], true)); + }; + + const render = () => + h(UploadMethodModal, { + props: { + show: state.modalVisible, + config: state.modalConfig, + hideTips: state.modalConfig.hideTips ?? false, + }, + on: { + "update:show": (show: boolean) => { + state.modalVisible = show; + }, + uploaded: finishUploaded, + cancelled: finishCancelled, + }, + }); + + const instance = new Vue({ + name: "UploadMethodModalHost", + setup() { + return { + state, + finishUploaded, + finishCancelled, + render, + }; + }, + render(): ReturnType { + return this.render(); + }, + }).$mount(hostElement); + + modalVm = instance as UploadMethodModalHostInstance; + return modalVm; +} + +export function useUploadMethodModal() { + async function openUploadModal(config?: UploadModalConfig): Promise { + const vm = ensureMounted(); + + if (pendingResolvers) { + return Promise.reject(new Error("An upload modal is already open.")); + } + + const defaultedConfig = applyConfigDefaults(config); + vm.state.modalConfig = defaultedConfig; + vm.state.modalVisible = true; + + return new Promise((resolve, reject) => { + pendingResolvers = { resolve, reject }; + }); + } + + return { + openUploadModal, + }; +} + +export { toDataOptions }; diff --git a/client/src/composables/upload/useUploadStaging.ts b/client/src/composables/upload/useUploadStaging.ts index eebf4bb4c67..f47d0a3d469 100644 --- a/client/src/composables/upload/useUploadStaging.ts +++ b/client/src/composables/upload/useUploadStaging.ts @@ -10,6 +10,13 @@ interface UseUploadStagingReturn { clear: () => void; } +interface UploadStagingOptions { + /** Whether to disable the upload staging store. + * This is useful for short-term staging sessions that don't require persistence. + */ + disableStore?: boolean; +} + /** * Composable to synchronize a local reactive items ref with the upload staging store. * @@ -21,9 +28,19 @@ interface UseUploadStagingReturn { export function useUploadStaging( mode: UploadMethod, items: Ref, + options: UploadStagingOptions = {}, ): UseUploadStagingReturn { const stagingStore = useUploadStagingStore(); + // When disabled, act as a no-op passthrough using only the provided ref + if (options.disableStore) { + return { + clear() { + items.value = []; + }, + }; + } + // Restore staged items on mount onMounted(() => { const staged = stagingStore.getItems(mode); diff --git a/client/src/composables/upload/useUploadSubmission.test.ts b/client/src/composables/upload/useUploadSubmission.test.ts new file mode 100644 index 00000000000..faeddd40504 --- /dev/null +++ b/client/src/composables/upload/useUploadSubmission.test.ts @@ -0,0 +1,316 @@ +import { getLocalVue, suppressExpectedErrorMessages } from "@tests/vitest/helpers"; +import { mount } from "@vue/test-utils"; +import flushPromises from "flush-promises"; +import { http, HttpResponse } from "msw"; +import { createPinia, setActivePinia } from "pinia"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { defineComponent, ref } from "vue"; + +import { useServerMock } from "@/api/client/__mocks__"; +import type { PreparedUpload } from "@/components/Panels/Upload/types"; +import { useUploadState } from "@/components/Panels/Upload/uploadState"; +import { makeCollectionConfig, makeLibraryItem, makeUrlItem } from "@/composables/upload/testHelpers/uploadFixtures"; +import { buildPreparedUpload } from "@/utils/upload"; + +import { useUploadSubmission } from "./useUploadSubmission"; + +const SELECTORS = { + RUN: "[data-test-id='run']", + RESULT: "[data-test-id='result']", + ERROR: "[data-test-id='error']", +}; + +const localVue = getLocalVue(); +const { server } = useServerMock(); + +function mountHarness(prepared: PreparedUpload) { + const Harness = defineComponent({ + setup() { + const { submitPreparedUpload } = useUploadSubmission(); + const result = ref(""); + const error = ref(""); + + async function run() { + try { + const uploaded = await submitPreparedUpload("hist_1", prepared); + result.value = JSON.stringify(uploaded); + } catch (err) { + error.value = String(err); + } + } + + return { error, result, run }; + }, + template: ` + + run + {{ result }} + {{ error }} + + `, + }); + + return mount(Harness, { localVue, pinia: createPinia() }); +} + +function makeSubmissionCollectionConfig() { + return makeCollectionConfig({ + name: "Uploaded Collection", + hideSourceItems: true, + }); +} + +describe("useUploadSubmission", () => { + beforeEach(() => { + setActivePinia(createPinia()); + useUploadState().clearAll(); + + server.use(http.get("/api/configuration", () => HttpResponse.json({ chunk_upload_size: 42 }))); + }); + + afterEach(() => { + useUploadState().clearAll(); + }); + + it("submits mixed uploads, tracks completion, and flattens nested fetch outputs", async () => { + server.use( + http.post("/api/tools/fetch", () => + HttpResponse.json({ + jobs: [{ id: "job_1" }], + outputs: { + first: { id: "hda_1", name: "api dataset", hid: 1, src: "hda" }, + nested: [ + { duplicate: { id: "hda_1", name: "duplicate", hid: 1, src: "hda" } }, + { id: "hdca_1", name: "api collection", src: "hdca" }, + ], + }, + }), + ), + http.post("/api/histories/hist_1/contents/datasets", async ({ request }) => { + const body = await request.json(); + expect(body).toMatchObject({ + content: "ldda_1", + source: "library", + type: "dataset", + }); + return HttpResponse.json({ id: "hda_2", name: "copied library", hid: 2 }); + }), + ); + + const apiItem = makeUrlItem({ name: "remote.txt", url: "https://example.org/remote.txt" }); + const apiPrepared = buildPreparedUpload([apiItem]); + const wrapper = mountHarness({ + apiItems: apiPrepared.apiItems, + uploadItems: [apiItem, makeLibraryItem()], + }); + await flushPromises(); + + await wrapper.find(SELECTORS.RUN).trigger("click"); + await flushPromises(); + + expect(wrapper.find(SELECTORS.RESULT).text()).toContain('"id":"hda_1"'); + expect(wrapper.find(SELECTORS.RESULT).text()).toContain('"id":"hdca_1"'); + expect(wrapper.find(SELECTORS.RESULT).text()).toContain('"id":"hda_2"'); + + const state = useUploadState(); + const pastedEntry = state.activeItems.value.find((item) => item.name === "remote.txt"); + const libraryEntry = state.activeItems.value.find((item) => item.name === "library.txt"); + + expect(pastedEntry?.status).toBe("completed"); + expect(pastedEntry?.progress).toBe(100); + expect(libraryEntry?.status).toBe("completed"); + expect(libraryEntry?.progress).toBe(100); + }); + + it("marks all tracked uploads as errored when the fetch request fails", async () => { + server.use( + http.post("/api/tools/fetch", () => HttpResponse.json({ err_msg: "upload failed" }, { status: 500 })), + ); + + const apiItem = makeUrlItem({ name: "remote.txt", url: "https://example.org/broken.txt" }); + const apiPrepared = buildPreparedUpload([apiItem]); + const wrapper = mountHarness({ + apiItems: apiPrepared.apiItems, + uploadItems: [apiItem, makeLibraryItem()], + }); + await flushPromises(); + + await wrapper.find(SELECTORS.RUN).trigger("click"); + await flushPromises(); + + expect(wrapper.find(SELECTORS.ERROR).text()).toContain("upload failed"); + + const state = useUploadState(); + expect(state.activeItems.value).toHaveLength(2); + for (const item of state.activeItems.value) { + expect(item.status).toBe("error"); + expect(item.error).toBe("upload failed"); + } + }); + + it("falls back to the staged library item name when the copy response omits metadata", async () => { + server.use( + http.post("/api/histories/hist_1/contents/datasets", async ({ request }) => { + const body = await request.json(); + expect(body).toMatchObject({ + content: "ldda_3", + source: "library", + type: "dataset", + }); + return HttpResponse.json({ id: "hda_3" }); + }), + ); + + const wrapper = mountHarness({ + apiItems: [], + uploadItems: [makeLibraryItem({ name: "fallback-name.txt", lddaId: "ldda_3" })], + }); + await flushPromises(); + + await wrapper.find(SELECTORS.RUN).trigger("click"); + await flushPromises(); + + expect(wrapper.find(SELECTORS.RESULT).text()).toContain('"name":"fallback-name.txt"'); + expect(wrapper.find(SELECTORS.RESULT).text()).toContain('"id":"hda_3"'); + }); + + it("groups direct collection uploads into a batch in upload state", async () => { + server.use( + http.post("/api/tools/fetch", async ({ request }) => { + const body = await request.json(); + + expect(body).toMatchObject({ + history_id: "hist_1", + targets: [ + { + destination: { type: "hdca" }, + collection_type: "list", + name: "Uploaded Collection", + }, + ], + }); + + return HttpResponse.json({ + jobs: [{ id: "job_1" }], + outputs: [{ id: "hdca_2", name: "Uploaded Collection", src: "hdca" }], + }); + }), + ); + + const firstItem = makeUrlItem({ name: "1.bed", url: "https://example.org/1.bed" }); + const secondItem = makeUrlItem({ name: "2.bed", url: "https://example.org/2.bed" }); + const prepared = buildPreparedUpload([firstItem, secondItem], makeSubmissionCollectionConfig()); + const wrapper = mountHarness(prepared); + await flushPromises(); + + await wrapper.find(SELECTORS.RUN).trigger("click"); + await flushPromises(); + + expect(wrapper.find(SELECTORS.RESULT).text()).toContain('"id":"hdca_2"'); + + const state = useUploadState(); + const batch = state.activeBatches.value[0]; + + expect(batch?.name).toBe("Uploaded Collection"); + expect(batch?.status).toBe("completed"); + expect(batch?.collectionId).toBe("hdca_2"); + expect(batch?.uploadIds).toHaveLength(2); + expect(state.standaloneUploads.value).toHaveLength(0); + expect(state.orderedUploadItems.value[0]?.type).toBe("batch"); + expect(state.activeItems.value.every((item) => item.batchId === batch?.id)).toBe(true); + }); + + it("creates a two-step collection for library-only uploads", async () => { + server.use( + http.post("/api/histories/hist_1/contents/datasets", () => HttpResponse.json({ id: "hda_lib_1", hid: 3 })), + http.post("/api/dataset_collections", async ({ request }) => { + const body = await request.json(); + expect(body).toMatchObject({ + history_id: "hist_1", + collection_type: "list", + name: "Uploaded Collection", + element_identifiers: [{ id: "hda_lib_1", src: "hda" }], + }); + return HttpResponse.json({ id: "hdca_lib_1" }); + }), + ); + + const prepared = buildPreparedUpload([makeLibraryItem()], makeSubmissionCollectionConfig()); + const wrapper = mountHarness(prepared); + await flushPromises(); + + await wrapper.find(SELECTORS.RUN).trigger("click"); + await flushPromises(); + + expect(wrapper.find(SELECTORS.RESULT).text()).toContain('"id":"hda_lib_1"'); + + const batch = useUploadState().activeBatches.value[0]; + expect(batch?.status).toBe("completed"); + expect(batch?.collectionId).toBe("hdca_lib_1"); + expect(batch?.datasetIds).toEqual(["hda_lib_1"]); + }); + + it("creates a two-step collection for mixed api and library uploads", async () => { + server.use( + http.post("/api/tools/fetch", () => + HttpResponse.json({ + outputs: [{ id: "hda_api_1", name: "api dataset", hid: 1, src: "hda" }], + }), + ), + http.post("/api/histories/hist_1/contents/datasets", () => HttpResponse.json({ id: "hda_lib_2", hid: 2 })), + http.post("/api/dataset_collections", async ({ request }) => { + const body = await request.json(); + expect(body).toMatchObject({ + history_id: "hist_1", + collection_type: "list", + name: "Uploaded Collection", + element_identifiers: [ + { id: "hda_api_1", src: "hda" }, + { id: "hda_lib_2", src: "hda" }, + ], + }); + return HttpResponse.json({ id: "hdca_mixed_1" }); + }), + ); + + const apiItem = makeUrlItem({ name: "api-first.txt" }); + const libraryItem = makeLibraryItem({ name: "library-second.txt", lddaId: "ldda_2" }); + const wrapper = mountHarness(buildPreparedUpload([apiItem, libraryItem], makeSubmissionCollectionConfig())); + await flushPromises(); + + await wrapper.find(SELECTORS.RUN).trigger("click"); + await flushPromises(); + + expect(wrapper.find(SELECTORS.RESULT).text()).toContain('"id":"hda_api_1"'); + expect(wrapper.find(SELECTORS.RESULT).text()).toContain('"id":"hda_lib_2"'); + + const batch = useUploadState().activeBatches.value[0]; + expect(batch?.status).toBe("completed"); + expect(batch?.collectionId).toBe("hdca_mixed_1"); + expect(batch?.datasetIds).toEqual(["hda_api_1", "hda_lib_2"]); + }); + + it("surfaces two-step collection creation failures after uploads succeed", async () => { + suppressExpectedErrorMessages(["Collection error"]); + server.use( + http.post("/api/histories/hist_1/contents/datasets", () => HttpResponse.json({ id: "hda_lib_3", hid: 4 })), + http.post("/api/dataset_collections", () => + HttpResponse.json({ err_msg: "Collection error" }, { status: 500 }), + ), + ); + + const wrapper = mountHarness( + buildPreparedUpload([makeLibraryItem({ lddaId: "ldda_3" })], makeSubmissionCollectionConfig()), + ); + await flushPromises(); + + await wrapper.find(SELECTORS.RUN).trigger("click"); + await flushPromises(); + + expect(wrapper.find(SELECTORS.ERROR).text()).toContain("Collection error"); + + const batch = useUploadState().activeBatches.value[0]; + expect(batch?.status).toBe("error"); + expect(batch?.collectionId).toBeUndefined(); + }); +}); diff --git a/client/src/composables/upload/useUploadSubmission.ts b/client/src/composables/upload/useUploadSubmission.ts new file mode 100644 index 00000000000..510b8ad9c14 --- /dev/null +++ b/client/src/composables/upload/useUploadSubmission.ts @@ -0,0 +1,206 @@ +import { copyDataset } from "@/api/datasets"; +import type { PreparedUpload } from "@/components/Panels/Upload/types"; +import { useUploadState } from "@/components/Panels/Upload/uploadState"; +import { useConfig } from "@/composables/config"; +import type { LibraryDatasetUploadItem, UploadedDataset } from "@/composables/upload/uploadItemTypes"; +import { datasetsFromFetchResponse } from "@/composables/upload/uploadResponse"; +import type { InitializedUploads, TrackedUpload } from "@/composables/upload/uploadTracking"; +import { + initializeTrackedUploads, + markTrackedCompleted, + markTrackedError, + splitTrackedUploadsByType, + updateTrackedProgress, +} from "@/composables/upload/uploadTracking"; +import { useUploadBatchOperations } from "@/composables/upload/useUploadBatchOperations"; +import { errorMessageAsString } from "@/utils/simple-error"; +import type { UploadDatasetsConfig } from "@/utils/upload"; +import { isFetchApiCompatible, uploadCollectionDatasets, uploadDatasets } from "@/utils/upload"; + +/** + * Composable that provides a centralized handler for submitting a prepared upload + * to the Galaxy API. + */ +export function useUploadSubmission() { + const uploadState = useUploadState(); + const uploadBatchOperations = useUploadBatchOperations({ autoRecover: false }); + const { config: galaxyConfig } = useConfig(); + + function initializeUploads(prepared: PreparedUpload): InitializedUploads { + const collectionConfig = prepared.collectionConfig; + const directCreation = isDirectCollectionCreation(prepared); + + return initializeTrackedUploads(uploadState, prepared.uploadItems, { + collectionConfig, + directCreation, + startUploading: true, + }); + } + + /** + * Determines if the prepared upload is for direct collection creation. + * Data library items are not compatible with direct collection creation because + * they require copying datasets rather than uploading files, so the presence of any + * data library items means we cannot do direct collection creation. + */ + function isDirectCollectionCreation(prepared: PreparedUpload): boolean { + return Boolean(prepared.collectionConfig && prepared.uploadItems?.every(isFetchApiCompatible)); + } + + /** + * Process API-based uploads with progress tracking. + * + * This function handles the upload of file-based items through the Galaxy API, + * supporting both regular dataset uploads and direct collection creation. Progress + * is tracked via callbacks and the upload state store. + */ + async function processApiUploads( + prepared: PreparedUpload, + apiIds: string[], + datasets: UploadedDataset[], + trackedUploads: TrackedUpload[], + batchId?: string, + directCollectionCreation?: boolean, + onProgress?: (percentage: number) => void, + ): Promise { + if (prepared.apiItems.length === 0) { + return; + } + + return new Promise((resolve, reject) => { + const config: UploadDatasetsConfig = { + chunkSize: galaxyConfig.value.chunk_upload_size as number, + success: (response) => { + const uploadedDatasets = datasetsFromFetchResponse(response); + + markTrackedCompleted(uploadState, apiIds); + datasets.push(...uploadedDatasets); + + if (batchId) { + if (directCollectionCreation) { + const createdCollection = uploadedDatasets.find((dataset) => dataset.src === "hdca"); + if (createdCollection) { + uploadState.setBatchCollectionId(batchId, createdCollection.id); + } + uploadState.updateBatchStatus(batchId, "completed"); + } else { + uploadedDatasets + .filter((dataset) => dataset.src === "hda") + .forEach((dataset) => uploadState.addBatchDatasetId(batchId, dataset.id)); + } + } + + resolve(); + }, + error: (uploadError) => { + const errorMessage = errorMessageAsString(uploadError); + + markTrackedError(uploadState, trackedUploads, errorMessage); + if (batchId) { + uploadState.setBatchError(batchId, errorMessage); + } + reject(uploadError); + }, + progress: (percentage) => { + onProgress?.(percentage); + updateTrackedProgress(uploadState, apiIds, percentage); + }, + }; + + if (prepared.collectionConfig && directCollectionCreation) { + uploadCollectionDatasets( + prepared.apiItems, + { + collectionName: prepared.collectionConfig.name, + collectionType: prepared.collectionConfig.type, + }, + config, + ); + } else { + uploadDatasets(prepared.apiItems, { + ...config, + composite: prepared.uploadOptions?.composite, + compositeName: prepared.uploadOptions?.compositeName, + }); + } + }); + } + + /** + * Process library dataset uploads with progress tracking. + * + * This function copies datasets from data libraries into the current history. + * Each library dataset is processed sequentially, with progress tracking for each item. + * + * @param libraryUploads - Array of tracked library upload items to process + * @param historyId - The target history ID to copy datasets into + * @param datasets - Array to collect successfully copied datasets + * @param batchId - Optional batch ID for batch upload tracking + * @returns Promise that resolves when all library uploads complete + */ + async function processLibraryUploads( + libraryUploads: TrackedUpload[], + historyId: string, + datasets: UploadedDataset[], + batchId?: string, + ): Promise { + for (const tracked of libraryUploads) { + uploadState.updateProgress(tracked.id, 50); + const copied = await copyDataset(tracked.item.lddaId, historyId, "dataset", "library"); + if (copied && "id" in copied && copied.id) { + const copiedName = + "name" in copied && typeof copied.name === "string" ? copied.name : tracked.item.name; + const copiedHid = "hid" in copied && typeof copied.hid === "number" ? copied.hid : undefined; + + datasets.push({ + id: copied.id, + name: copiedName, + hid: copiedHid, + src: "hda", + }); + if (batchId) { + uploadState.addBatchDatasetId(batchId, copied.id); + } + } + markTrackedCompleted(uploadState, [tracked.id]); + } + } + + /** + * Submit a prepared upload to Galaxy and return the resulting datasets. + * + * Progress is tracked automatically in the upload state store so the + * progress panel reflects upload status. An optional `onProgress` callback + * can be used by the caller to update its own local progress indicator + * (e.g. the modal progress bar). + */ + async function submitPreparedUpload( + historyId: string, + prepared: PreparedUpload, + onProgress?: (percentage: number) => void, + ): Promise { + const datasets: UploadedDataset[] = []; + const directCollectionCreation = isDirectCollectionCreation(prepared); + const { trackedUploads, batchId } = initializeUploads(prepared); + const { apiIds, libraryUploads } = splitTrackedUploadsByType(trackedUploads); + + await processApiUploads( + prepared, + apiIds, + datasets, + trackedUploads, + batchId, + directCollectionCreation, + onProgress, + ); + await processLibraryUploads(libraryUploads, historyId, datasets, batchId); + + if (batchId && prepared.collectionConfig && !directCollectionCreation) { + await uploadBatchOperations.createCollection(batchId); + } + + return datasets; + } + + return { submitPreparedUpload }; +} diff --git a/client/src/composables/uploadQueue.test.ts b/client/src/composables/uploadQueue.test.ts deleted file mode 100644 index 9fe78f5129f..00000000000 --- a/client/src/composables/uploadQueue.test.ts +++ /dev/null @@ -1,405 +0,0 @@ -import { suppressExpectedErrorMessages } from "@tests/vitest/helpers"; -import assert from "assert"; -import flushPromises from "flush-promises"; -import { http, HttpResponse } from "msw"; -import { createPinia, setActivePinia } from "pinia"; -import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; - -import { useServerMock } from "@/api/client/__mocks__"; -import { useUploadState } from "@/components/Panels/Upload/uploadState"; -import type { NewUploadItem } from "@/composables/upload/uploadItemTypes"; - -import { type CollectionConfig, useUploadQueue, validateUploadItem } from "./uploadQueue"; - -// TUS upload is a non-HTTP protocol — mock it so tests never attempt real TUS connections. -vi.mock("@/utils/tusUpload", () => ({ - createTusUpload: vi.fn(), - NamedBlob: class {}, -})); - -const { server } = useServerMock(); - -/** Creates a paste-content upload item. */ -function makePastedItem(overrides: Partial = {}): NewUploadItem { - return { - uploadMode: "paste-content", - name: "file.txt", - content: "hello world", - size: 11, - targetHistoryId: "hist_1", - dbkey: "?", - extension: "auto", - spaceToTab: false, - toPosixLines: false, - deferred: false, - ...overrides, - } as NewUploadItem; -} - -/** Creates a paste-links (URL) upload item. URL items use direct fetchDatasets — no TUS involved. */ -function makeUrlItem(overrides: Partial = {}): NewUploadItem { - return { - uploadMode: "paste-links", - name: "file.txt", - url: "http://example.com/file.txt", - size: 0, - targetHistoryId: "hist_1", - dbkey: "?", - extension: "auto", - spaceToTab: false, - toPosixLines: false, - deferred: false, - ...overrides, - } as NewUploadItem; -} - -/** Creates a remote-files upload item. Same shape as paste-links but with a different uploadMode. */ -function makeRemoteFilesItem(overrides: Partial = {}): NewUploadItem { - return { - uploadMode: "remote-files", - name: "file.txt", - url: "ftp://server/file.txt", - size: 0, - targetHistoryId: "hist_1", - dbkey: "?", - extension: "auto", - spaceToTab: false, - toPosixLines: false, - deferred: false, - ...overrides, - } as NewUploadItem; -} - -/** Creates a data-library upload item for testing the two-step collection path. */ -function makeLibraryItem(overrides: Partial = {}): NewUploadItem { - return { - uploadMode: "data-library", - name: "library.txt", - size: 0, - targetHistoryId: "hist_1", - dbkey: "?", - extension: "auto", - spaceToTab: false, - toPosixLines: false, - deferred: false, - libraryId: "lib_1", - folderId: "folder_1", - lddaId: "ldda_1", - url: "/api/libraries/lib_1/datasets/ldda_1", - ...overrides, - } as NewUploadItem; -} - -function makeCollectionConfig(overrides: Partial = {}): CollectionConfig { - return { - name: "My Collection", - type: "list", - historyId: "hist_1", - hideSourceItems: false, - ...overrides, - }; -} - -/** Creates a local-file upload item with a real File object. */ -function makeLocalFileItem(overrides: Partial = {}): NewUploadItem { - const file = new File(["content"], "test.txt"); - return { - uploadMode: "local-file", - name: "test.txt", - size: file.size, - targetHistoryId: "hist_1", - dbkey: "?", - extension: "auto", - spaceToTab: false, - toPosixLines: false, - deferred: false, - fileData: file, - ...overrides, - } as NewUploadItem; -} - -describe("validateUploadItem", () => { - it.each([ - ["paste-content", makePastedItem()], - ["paste-links", makeUrlItem()], - ["remote-files", makeRemoteFilesItem()], - ["data-library", makeLibraryItem()], - ["local-file", makeLocalFileItem()], - ] as [string, NewUploadItem][])("accepts a valid %s item", (_mode, item) => { - expect(validateUploadItem(item)).toBeUndefined(); - }); - - it("rejects paste-content with empty content", () => { - const item = makePastedItem({ content: " " }); - expect(validateUploadItem(item)).toMatch(/No content provided/); - }); - - it("rejects paste-links with missing URL", () => { - expect(validateUploadItem(makeUrlItem({ url: "" }))).toMatch(/No URL provided/); - }); - - it("rejects remote-files with missing URL", () => { - expect(validateUploadItem(makeRemoteFilesItem({ url: " " }))).toMatch(/No URL provided/); - }); - - it("rejects data-library with no lddaId", () => { - expect(validateUploadItem(makeLibraryItem({ lddaId: "" }))).toMatch(/No library dataset ID/); - }); - - it("rejects local-file with no file data", () => { - const item: NewUploadItem = { - uploadMode: "local-file", - name: "missing.txt", - size: 0, - targetHistoryId: "hist_1", - dbkey: "?", - extension: "auto", - spaceToTab: false, - toPosixLines: false, - deferred: false, - }; - expect(validateUploadItem(item)).toMatch(/No file selected/); - }); - - it("rejects local-file with an empty file", () => { - const emptyFile = new File([], "empty.txt"); - const item: NewUploadItem = { - uploadMode: "local-file", - name: "empty.txt", - size: 0, - targetHistoryId: "hist_1", - dbkey: "?", - extension: "auto", - spaceToTab: false, - toPosixLines: false, - deferred: false, - fileData: emptyFile, - }; - expect(validateUploadItem(item)).toMatch(/is empty/); - }); - - it("rejects an unknown upload mode", () => { - const item = { ...makePastedItem(), uploadMode: "unknown-mode" } as unknown as NewUploadItem; - expect(validateUploadItem(item)).toMatch(/Unknown upload mode/); - }); -}); - -describe("useUploadQueue", () => { - let queue: ReturnType; - - beforeEach(() => { - // Fresh Pinia instance for each test to prevent cross-test store state. - setActivePinia(createPinia()); - // Reset the shared upload state singleton before creating a new queue. - useUploadState().clearAll(); - // Each call creates fresh local queue/batch state and re-runs recoverIncompleteBatches. - queue = useUploadQueue(); - }); - - afterEach(() => { - vi.restoreAllMocks(); - }); - - describe("enqueue — single item", () => { - it("marks the item as completed after a successful upload", async () => { - server.use(http.post("/api/tools/fetch", () => HttpResponse.json({}))); - - const [id] = queue.enqueue([makeUrlItem()]); - await flushPromises(); - - expect(queue.state.activeItems.value.find((i) => i.id === id)?.status).toBe("completed"); - }); - - it("all items reach completed status when multiple items are enqueued", async () => { - server.use(http.post("/api/tools/fetch", () => HttpResponse.json({}))); - - const ids = queue.enqueue([ - makeUrlItem({ name: "a.txt" }), - makeUrlItem({ name: "b.txt" }), - makeUrlItem({ name: "c.txt" }), - ]); - await flushPromises(); - - for (const id of ids) { - expect(queue.state.activeItems.value.find((i) => i.id === id)?.status).toBe("completed"); - } - }); - - it("marks the item as error when the upload endpoint returns an error", async () => { - server.use( - http.post("/api/tools/fetch", () => - HttpResponse.json({ err_msg: "Server unavailable" }, { status: 500 }), - ), - ); - - const [id] = queue.enqueue([makeUrlItem()]); - await flushPromises(); - - const item = queue.state.activeItems.value.find((i) => i.id === id); - expect(item?.status).toBe("error"); - expect(item?.error).toBeTruthy(); - }); - - it("marks item as error when enqueued with invalid content (no server call needed)", async () => { - const [id] = queue.enqueue([makePastedItem({ content: " " })]); - await flushPromises(); - - const item = queue.state.activeItems.value.find((i) => i.id === id); - expect(item?.status).toBe("error"); - expect(item?.error).toMatch(/No content provided/); - }); - }); - - describe("enqueue — direct collection batch", () => { - it("marks the batch as completed after a successful collection upload", async () => { - server.use(http.post("/api/tools/fetch", () => HttpResponse.json({}))); - - queue.enqueue([makeUrlItem({ name: "a.txt" }), makeUrlItem({ name: "b.txt" })], makeCollectionConfig()); - await flushPromises(); - - const batch = queue.state.activeBatches.value[0]; - expect(batch?.status).toBe("completed"); - }); - - it("marks the batch as error when the upload endpoint fails", async () => { - suppressExpectedErrorMessages(["Upload failed"]); - server.use( - http.post("/api/tools/fetch", () => HttpResponse.json({ err_msg: "Upload failed" }, { status: 500 })), - ); - - queue.enqueue([makeUrlItem()], makeCollectionConfig()); - await flushPromises(); - - const batch = queue.state.activeBatches.value[0]; - expect(batch?.status).toBe("error"); - }); - }); - - // ── Two-step collection batch (data-library items) ─────────────────────── - // - // data-library items cannot use direct HDCA creation (they use copyDataset instead - // of /api/tools/fetch). Each item is uploaded individually; after all items complete, - // the collection is created via POST /api/dataset_collections. - - describe("enqueue — two-step collection batch (data-library)", () => { - it("creates the collection after all library items are successfully copied", async () => { - server.use( - http.post("/api/histories/:historyId/contents/:type", () => HttpResponse.json({ id: "ds_lib_1" })), - http.post("/api/dataset_collections", () => HttpResponse.json({ id: "col_1" })), - ); - - queue.enqueue([makeLibraryItem()], makeCollectionConfig()); - await flushPromises(); - - const batch = queue.state.activeBatches.value[0]; - expect(batch?.status).toBe("completed"); - expect(batch?.collectionId).toBe("col_1"); - }); - - it("marks the batch as error when collection creation fails after successful uploads", async () => { - suppressExpectedErrorMessages(["Collection creation failed:", "Collection error"]); - server.use( - http.post("/api/histories/:historyId/contents/:type", () => HttpResponse.json({ id: "ds_lib_1" })), - http.post("/api/dataset_collections", () => - HttpResponse.json({ err_msg: "Collection error" }, { status: 500 }), - ), - ); - - queue.enqueue([makeLibraryItem()], makeCollectionConfig()); - await flushPromises(); - - const batch = queue.state.activeBatches.value[0]; - expect(batch?.status).toBe("error"); - expect(batch?.collectionId).toBeUndefined(); - }); - }); - - describe("retryCollectionCreation", () => { - it("re-attempts collection creation and resolves the batch after a previous failure", async () => { - suppressExpectedErrorMessages(["Collection creation failed:", "Temporary error"]); - - // Phase 1: uploads succeed, collection creation fails. - server.use( - http.post("/api/histories/:historyId/contents/:type", () => HttpResponse.json({ id: "ds_lib_1" })), - http.post("/api/dataset_collections", () => - HttpResponse.json({ err_msg: "Temporary error" }, { status: 500 }), - ), - ); - - queue.enqueue([makeLibraryItem()], makeCollectionConfig()); - await flushPromises(); - - const batchId = queue.state.activeBatches.value[0]?.id; - assert(batchId, "Expected a batch to be created"); - expect(queue.state.getBatch(batchId)?.status).toBe("error"); - - // Phase 2: collection creation now succeeds. - server.use(http.post("/api/dataset_collections", () => HttpResponse.json({ id: "col_retried" }))); - - await queue.retryCollectionCreation(batchId); - await flushPromises(); - - expect(queue.state.getBatch(batchId)?.status).toBe("completed"); - expect(queue.state.getBatch(batchId)?.collectionId).toBe("col_retried"); - }); - }); - - describe("clearCompleted", () => { - it("removes completed uploads from state", async () => { - server.use(http.post("/api/tools/fetch", () => HttpResponse.json({}))); - - const [completedId] = queue.enqueue([makeUrlItem()]); - await flushPromises(); - - expect(queue.state.activeItems.value.find((i) => i.id === completedId)?.status).toBe("completed"); - - queue.clearCompleted(); - - expect(queue.state.activeItems.value.find((i) => i.id === completedId)).toBeUndefined(); - }); - }); - - describe("clearAll", () => { - it("empties all items, batches, and internal queue state", async () => { - server.use(http.post("/api/tools/fetch", () => HttpResponse.json({}))); - - queue.enqueue([makeUrlItem(), makeUrlItem()], makeCollectionConfig()); - await flushPromises(); - - queue.clearAll(); - - expect(queue.state.activeItems.value).toHaveLength(0); - expect(queue.state.activeBatches.value).toHaveLength(0); - }); - }); - - // On initialization, the queue checks for two-step batches where uploads completed - // in a previous session but collection creation did not finish (e.g. page was refreshed). - describe("recoverIncompleteBatches", () => { - it("automatically creates the collection for a two-step batch with completed uploads and dataset IDs", async () => { - const uploadState = useUploadState(); - - // Simulate state left from a previous browser session: - // one completed upload item and a pending two-step batch with a dataset ID. - const itemId = uploadState.addUploadItem(makeUrlItem({ name: "recovered.txt" })); - uploadState.setStatus(itemId, "uploading"); - uploadState.updateProgress(itemId, 100); // auto-marks as completed - - const batchId = uploadState.addBatch( - { name: "Recovery Collection", type: "list", hideSourceItems: false, historyId: "hist_1" }, - [itemId], - false, // not directCreation — qualifies for recovery - ); - uploadState.addBatchDatasetId(batchId, "ds_recovered"); - - server.use(http.post("/api/dataset_collections", () => HttpResponse.json({ id: "col_recovered" }))); - - // Creating a new queue instance triggers recoverIncompleteBatches. - const recoveryQueue = useUploadQueue(); - await flushPromises(); - - expect(recoveryQueue.state.getBatch(batchId)?.collectionId).toBe("col_recovered"); - expect(recoveryQueue.state.getBatch(batchId)?.status).toBe("completed"); - }); - }); -}); diff --git a/client/src/composables/uploadQueue.ts b/client/src/composables/uploadQueue.ts deleted file mode 100644 index 5bf317cab8c..00000000000 --- a/client/src/composables/uploadQueue.ts +++ /dev/null @@ -1,825 +0,0 @@ -/** - * Upload queue composable for managing dataset uploads to Galaxy. - * - * This composable provides a queue-based upload system that: - * - Converts UI upload items to API-ready format - * - Processes uploads sequentially with progress tracking - * - Persists upload state for UI monitoring - * - Creates dataset collections after successful batch uploads - */ -import type { CollectionElementIdentifiers } from "@/api"; -import { createHistoryDatasetCollectionInstanceFull } from "@/api/datasetCollections"; -import { copyDataset } from "@/api/datasets"; -import type { FetchDataResponse } from "@/api/tools"; -import { - COMMON_FILTERS, - DEFAULT_FILTER, - guessInitialFilterType, - guessNameForPair, -} from "@/components/Collections/pairing"; -import { useUploadState } from "@/components/Panels/Upload/uploadState"; -import type { CollectionCreationInput, SupportedCollectionType } from "@/composables/upload/collectionTypes"; -import type { CompositeFileUploadItem, NewUploadItem } from "@/composables/upload/uploadItemTypes"; -import { useHistoryStore } from "@/stores/historyStore"; -import { getHistoryUploadActionErrorMessage, getHistoryUploadBlockReason } from "@/utils/historyUpload"; -import { errorMessageAsString } from "@/utils/simple-error"; -import { - createFileUploadItem, - createPastedUploadItem, - createUrlUploadItem, - toApiUploadItem, - uploadCollectionDatasets, - uploadDatasets, -} from "@/utils/upload"; - -/** - * Collection configuration for batch uploads that will be combined - * into a collection after uploads complete. - */ -export interface CollectionConfig extends CollectionCreationInput { - /** Whether to hide source datasets after collection creation */ - hideSourceItems: boolean; - /** Target history ID where the collection will be created */ - historyId: string; -} - -/** - * Represents a batch of uploads that will be combined into a dataset collection - * after all uploads complete successfully. - */ -interface CollectionBatch { - /** Batch ID used in the persisted upload state */ - batchId: string; - /** Upload item IDs belonging to this batch */ - ids: string[]; - /** Original upload items (needed for building collection elements) */ - items: NewUploadItem[]; - /** Dataset IDs collected from successful upload responses */ - datasetIds: string[]; - /** Collection configuration specifying name, type, target history, and options */ - collectionConfig: CollectionConfig; -} - -/** - * Builds collection element identifiers based on collection type. - * Uses the shared pairing abstractions from @/components/Collections/pairing for - * pair name extraction (synchronized with the backend via auto_pairing_spec.yml). - * - * @param items - Upload items with dataset IDs - * @param datasetIds - Array of created dataset IDs (in upload order) - * @param collectionType - Type of collection to create - * @returns Collection element identifiers ready for API - */ -function buildCollectionElements( - items: NewUploadItem[], - datasetIds: string[], - collectionType: SupportedCollectionType, -): CollectionElementIdentifiers { - if (collectionType === "list") { - // Simple list: one element per dataset - return items.map((item, index) => ({ - name: item.name || `element_${index + 1}`, - src: "hda" as const, - id: datasetIds[index], - })); - } else { - // List of pairs: group consecutive files into pairs - const pairs: CollectionElementIdentifiers = []; - const usedNames = new Set(); - - // Use the shared filter detection to determine forward/reverse naming convention - const filterType = guessInitialFilterType(items) ?? DEFAULT_FILTER; - const [forwardFilter, reverseFilter] = COMMON_FILTERS[filterType]; - - for (let i = 0; i < items.length; i += 2) { - if (i + 1 >= items.length) { - // Odd number of files - skip last one or handle as error - console.warn(`Skipping unpaired file at index ${i}: ${items[i]?.name}`); - break; - } - - const item1 = items[i]; - const item2 = items[i + 1]; - - if (!item1 || !item2) { - continue; - } - - const basePairName = - guessNameForPair(item1, item2, forwardFilter, reverseFilter, true) || `pair_${Math.floor(i / 2) + 1}`; - let pairName = basePairName; - - // Ensure unique pair names by adding suffix if needed - let counter = 1; - while (usedNames.has(pairName)) { - pairName = `${basePairName}_${counter}`; - counter++; - } - usedNames.add(pairName); - - pairs.push({ - collection_type: "paired", - src: "new_collection" as const, - name: pairName, - element_identifiers: [ - { - name: "forward", - src: "hda" as const, - id: datasetIds[i], - }, - { - name: "reverse", - src: "hda" as const, - id: datasetIds[i + 1], - }, - ], - }); - } - - return pairs; - } -} - -/** - * Validates a UI upload item before conversion. - * Returns an error message if invalid, undefined if valid. - * - * @param item - The upload item to validate - * @returns Error message if invalid, undefined if valid - */ -export function validateUploadItem(item: NewUploadItem): string | undefined { - switch (item.uploadMode) { - case "local-file": - if (!item.fileData) { - return `No file selected for "${item.name}"`; - } - if (item.fileData.size === 0) { - return `File "${item.name}" is empty`; - } - break; - - case "paste-content": - if (!item.content || item.content.trim().length === 0) { - return `No content provided for "${item.name}"`; - } - break; - - case "paste-links": - case "remote-files": - if (!item.url || item.url.trim().length === 0) { - return `No URL provided for "${item.name}"`; - } - break; - - case "data-library": - if (!item.lddaId) { - return `No library dataset ID provided for "${item.name}"`; - } - break; - - case "composite-file": { - if (!item.extension || item.extension === "auto") { - return `No composite type selected for "${item.name}"`; - } - for (const slot of item.slots) { - if (slot.optional) { - continue; - } - const isEmpty = - (slot.src === "files" && !slot.file) || - (slot.src === "url" && !slot.url?.trim()) || - (slot.src === "paste" && !slot.content?.trim()); - if (isEmpty) { - return `Required slot "${slot.slotName}" has no content for "${item.name}"`; - } - } - break; - } - - default: - return `Unknown upload mode: ${(item as NewUploadItem).uploadMode}`; - } - return undefined; -} - -/** - * Composable for managing upload queue and processing. - * - * Provides methods to enqueue different types of uploads and track their progress. - * Uses the shared upload state for persistence and UI updates. - * Supports automatic collection creation after batch uploads. - * - * @example - * ```typescript - * const uploadQueue = useUploadQueue(); - * - * // Enqueue files for upload - * const ids = uploadQueue.enqueue([ - * { uploadMode: "local-file", fileData: file, ... }, - * { uploadMode: "paste-links", url: "http://...", ... }, - * ]); - * - * // Clear completed uploads - * uploadQueue.clearCompleted(); - * ``` - */ -export function useUploadQueue() { - const uploadState = useUploadState(); - const historyStore = useHistoryStore(); - const queue: string[] = []; - const batches: CollectionBatch[] = []; - let processing = false; - - /** - * Helper to find an upload item by ID from the active items. - */ - function findUploadItem(id: string) { - return uploadState.activeItems.value.find((i) => i.id === id); - } - - /** - * Creates a dataset collection from uploaded datasets. - * - * @param batchId - Batch ID in upload state - */ - async function createCollection(batchId: string): Promise { - const batchState = uploadState.getBatch(batchId); - if (!batchState) { - console.error(`Batch not found: ${batchId}`); - return; - } - - // Check if collection already created (avoid duplicates on retry) - if (batchState.collectionId) { - console.log(`Collection already created for batch ${batchId}: ${batchState.collectionId}`); - return; - } - - const { name, type, hideSourceItems, historyId, datasetIds } = batchState; - - // Check if we have dataset IDs (either from persisted state or internal batch) - if (!datasetIds || datasetIds.length === 0) { - const errorMsg = "No dataset IDs available for collection creation"; - uploadState.setBatchError(batchId, errorMsg); - return; - } - - // Get upload items to build collection elements - // Try internal batch first (has full item data), fall back to state - const batch = batches.find((b) => b.batchId === batchId); - const items = - batch?.items || - batchState.uploadIds - .map((id) => findUploadItem(id)) - .filter((item): item is NonNullable => item !== undefined); - - if (items.length === 0) { - const errorMsg = "No upload items available for collection creation"; - uploadState.setBatchError(batchId, errorMsg); - return; - } - - // Validate items have required data - if (items.length !== batchState.uploadIds.length) { - const errorMsg = `Cannot create collection: only ${items.length} of ${batchState.uploadIds.length} upload items found. This can happen after a page refresh. Please re-upload the files or manually create the collection.`; - uploadState.setBatchError(batchId, errorMsg); - return; - } - - uploadState.updateBatchStatus(batchId, "creating-collection"); - - try { - // Build element identifiers based on collection type - const elementIdentifiers = buildCollectionElements(items, datasetIds, type); - - if (elementIdentifiers.length === 0) { - throw new Error("No valid collection elements to create"); - } - - // Create collection using the API - const response = await createHistoryDatasetCollectionInstanceFull({ - name, - collection_type: type, - element_identifiers: elementIdentifiers, - history_id: historyId, - hide_source_items: hideSourceItems, - instance_type: "history", - copy_elements: true, - fields: "auto", - }); - - uploadState.setBatchCollectionId(batchId, response.id); - uploadState.updateBatchStatus(batchId, "completed"); - - console.log(`Successfully created ${type} collection: ${name} (${response.id})`); - } catch (err) { - // Collection creation failed - datasets are still uploaded - const errorMsg = errorMessageAsString(err); - console.error("Collection creation failed:", errorMsg); - - uploadState.setBatchError(batchId, errorMsg); - - // Mark all batch items with error message (non-fatal) - const batchForError = uploadState.getBatch(batchId); - batchForError?.uploadIds.forEach((id) => { - const item = findUploadItem(id); - if (item && !item.error) { - item.error = `Uploaded successfully, but collection creation failed`; - } - }); - } - } - - /** - * Retries collection creation for a failed batch. - * @param batchId - Batch ID to retry - */ - async function retryCollectionCreation(batchId: string): Promise { - const batch = uploadState.getBatch(batchId); - if (!batch) { - console.error(`Batch not found: ${batchId}`); - return; - } - - // Reset error state - batch.error = undefined; - uploadState.updateBatchStatus(batchId, "uploading"); - - // Clear error messages from individual upload items - batch.uploadIds.forEach((id) => { - const item = findUploadItem(id); - if (item?.error?.includes("collection creation failed")) { - item.error = undefined; - } - }); - - try { - await createCollection(batchId); - } catch (err) { - uploadState.setBatchError(batchId, `Retry failed: ${errorMessageAsString(err)}`); - } - } - - /** - * Stores a dataset ID in both the internal batch and persisted state. - * - * @param batch - Internal batch tracking object - * @param datasetId - Dataset ID to store - */ - function collectDatasetId(batch: CollectionBatch | undefined, datasetId: string) { - if (batch) { - batch.datasetIds.push(datasetId); - uploadState.addBatchDatasetId(batch.batchId, datasetId); - } - } - - /** - * Checks if all uploads in a batch have completed (success or error). - * - * @param batch - Internal batch tracking object - * @returns True if all uploads are complete - */ - function isBatchComplete(batch: CollectionBatch): boolean { - return batch.ids.every((uploadId) => { - const item = findUploadItem(uploadId); - return item?.status === "completed" || item?.status === "error"; - }); - } - - /** - * Checks batch completion status and triggers collection creation if ready. - * - * @param batch - Internal batch tracking object - */ - async function checkAndCompleteBatch(batch?: CollectionBatch): Promise { - if (!batch) { - return; - } - - if (isBatchComplete(batch)) { - await createCollection(batch.batchId).catch((err) => { - uploadState.setBatchError(batch.batchId, errorMessageAsString(err)); - }); - } - } - - /** - * Determines if a collection batch should use direct HDCA creation. - * Returns false for batches containing data-library items (which need the two-step approach - * since they use copyDataset instead of /api/tools/fetch). - */ - function canUseDirectCollection(batch: CollectionBatch): boolean { - return batch.items.every((item) => item.uploadMode !== "data-library"); - } - - async function validateTargetHistory(targetHistoryId: string): Promise { - let history = historyStore.getHistoryById(targetHistoryId, false) ?? null; - if (!history) { - await historyStore.loadHistoryById(targetHistoryId); - history = historyStore.getHistoryById(targetHistoryId, false) ?? null; - } - - // If history still cannot be resolved, treat as no validation error here - // (downstream upload logic will surface appropriate API errors if needed) - if (!history) { - return null; - } - - const blockReason = getHistoryUploadBlockReason(history); - return blockReason ? getHistoryUploadActionErrorMessage(blockReason) : null; - } - - /** - * Processes an entire collection batch as a single HDCA upload. - * All items are uploaded together in one /api/tools/fetch request with - * destination { type: "hdca" }, creating the collection atomically. - */ - async function processCollectionBatch(batch: CollectionBatch): Promise { - const { batchId, items, collectionConfig } = batch; - - // Mark all items as uploading - batch.ids.forEach((id) => uploadState.setStatus(id, "uploading")); - uploadState.updateBatchStatus(batchId, "uploading"); - - try { - const historyError = await validateTargetHistory(collectionConfig.historyId); - if (historyError) { - throw new Error(historyError); - } - - // Validate all items first - for (const item of items) { - const validationError = validateUploadItem(item); - if (validationError) { - throw new Error(validationError); - } - } - - // Convert all UI items to API items - const apiItems = items.map((item) => toApiUploadItem(item)); - - await uploadCollectionDatasets( - apiItems, - { - collectionName: collectionConfig.name, - collectionType: collectionConfig.type, - }, - { - progress: (percentage) => { - batch.ids.forEach((id) => uploadState.updateProgress(id, percentage)); - }, - success: (_response: FetchDataResponse) => { - batch.ids.forEach((id) => uploadState.updateProgress(id, 100)); - uploadState.updateBatchStatus(batchId, "completed"); - }, - error: (err) => { - const errorMsg = errorMessageAsString(err); - batch.ids.forEach((id) => uploadState.setError(id, errorMsg)); - uploadState.setBatchError(batchId, errorMsg); - }, - }, - ); - } catch (err) { - const errorMsg = errorMessageAsString(err); - batch.ids.forEach((id) => uploadState.setError(id, errorMsg)); - uploadState.setBatchError(batchId, errorMsg); - } - } - - /** - * Processes a library dataset upload by importing it to the target history. - * - * @param id - Upload item ID - * @param item - Upload item with library dataset details - * @param batch - Internal batch tracking object (if part of a batch) - */ - async function processLibraryDatasetUpload( - id: string, - item: NewUploadItem, - batch?: CollectionBatch, - ): Promise { - if (item.uploadMode !== "data-library") { - throw new Error("Invalid upload mode for library dataset upload"); - } - - uploadState.updateProgress(id, 50); - - // Import library dataset to history - const response = await copyDataset(item.lddaId, item.targetHistoryId, "dataset", "library"); - - uploadState.updateProgress(id, 100); - - // Collect dataset ID for collection creation - // Response is an HDA (HistoryDatasetAssociation) which has an id field - if (response && "id" in response && response.id) { - collectDatasetId(batch, response.id); - } - - // Check if batch is ready for collection creation - await checkAndCompleteBatch(batch); - } - - /** - * Processes a composite upload by converting each slot into an ApiUploadItem - * and submitting them together as a single composite dataset. - * - * @param id - Upload item ID - * @param item - CompositeFileUploadItem with all slot data - */ - async function processCompositeFileUpload(id: string, item: CompositeFileUploadItem): Promise { - const baseOptions = { - dbkey: item.dbkey, - ext: item.extension, - space_to_tab: item.spaceToTab, - to_posix_lines: item.toPosixLines, - deferred: false, - }; - // Build one ApiUploadItem per slot, all sharing the dataset-level ext/dbkey/name - const slotApiItems = item.slots - .filter((slot) => slot.src !== "files" || !!slot.file) // skip empty optional local-file slots - .map((slot) => { - const slotOptions = { - name: slot.slotName, - ...baseOptions, - }; - - if (slot.src === "files") { - return createFileUploadItem(slot.file!, item.targetHistoryId, { - ...slotOptions, - size: slot.file!.size, - }); - } else if (slot.src === "url") { - return createUrlUploadItem(slot.url!, item.targetHistoryId, { - ...slotOptions, - size: 0, - }); - } else { - return createPastedUploadItem(slot.content ?? "", item.targetHistoryId, { - ...slotOptions, - size: (slot.content ?? "").length, - }); - } - }); - - await uploadDatasets(slotApiItems, { - composite: true, - compositeName: item.name, - progress: (percentage) => uploadState.updateProgress(id, percentage), - success: () => { - uploadState.updateProgress(id, 100); - }, - error: (err) => { - uploadState.setError(id, errorMessageAsString(err)); - }, - }); - } - - /** - * Processes a regular upload (file, URL, or pasted content) via the upload API. - * - * @param id - Upload item ID - * @param item - Upload item to process - * @param batch - Internal batch tracking object (if part of a batch) - */ - async function processRegularUpload(id: string, item: NewUploadItem, batch?: CollectionBatch): Promise { - const uploadItem = toApiUploadItem(item); - - await uploadDatasets([uploadItem], { - progress: (percentage) => uploadState.updateProgress(id, percentage), - success: (response: FetchDataResponse) => { - uploadState.updateProgress(id, 100); - - // Collect dataset IDs for collection creation - if (batch && response.outputs) { - // The outputs field is Record but is actually an array of dataset objects - const outputs = response.outputs as unknown as Array<{ - id: string; - hid?: number; - name?: string; - }>; - const datasetId = outputs[0]?.id; - if (datasetId) { - collectDatasetId(batch, datasetId); - } - - // Check if batch is ready for collection creation - checkAndCompleteBatch(batch); - } - }, - error: (err) => { - uploadState.setError(id, errorMessageAsString(err)); - }, - }); - } - - /** - * Processes the next upload in the queue. - * Orchestrates upload execution by delegating to specialized handlers. - * - * When the next item belongs to a collection batch that can use direct - * HDCA creation, all items in the batch are processed together in a - * single /api/tools/fetch request instead of one-by-one. - */ - async function processNext(): Promise { - if (processing || queue.length === 0) { - return; - } - - const id = queue[0]!; // Peek, don't remove yet - const item = findUploadItem(id); - - if (!item) { - queue.shift(); - // Item was removed from state (e.g., user cleared it), skip to next - return processNext(); - } - - // Check if this item belongs to a collection batch that can use direct creation - const batch = batches.find((b) => b.ids.includes(id)); - if (batch && canUseDirectCollection(batch)) { - // Remove all batch items from queue at once - for (const batchItemId of batch.ids) { - const idx = queue.indexOf(batchItemId); - if (idx !== -1) { - queue.splice(idx, 1); - } - } - - processing = true; - try { - await processCollectionBatch(batch); - } finally { - processing = false; - processNext(); - } - return; - } - - // Single-item processing (non-collection or data-library fallback) - queue.shift(); - processing = true; - uploadState.setStatus(id, "uploading"); - - try { - const historyError = await validateTargetHistory(item.targetHistoryId); - if (historyError) { - throw new Error(historyError); - } - - // Pre-validate before attempting upload - const validationError = validateUploadItem(item); - if (validationError) { - throw new Error(validationError); - } - - // Delegate to appropriate upload handler based on upload mode - if (item.uploadMode === "data-library") { - await processLibraryDatasetUpload(id, item, batch); - } else if (item.uploadMode === "composite-file") { - await processCompositeFileUpload(id, item); - } else { - await processRegularUpload(id, item, batch); - } - } catch (err) { - // This catches validation errors and any unexpected errors - uploadState.setError(id, errorMessageAsString(err)); - } finally { - processing = false; - processNext(); - } - } - - /** - * Enqueues items and starts processing. - * - * @param items - Array of upload items to enqueue - * @param collectionConfig - Optional collection configuration for creating a collection from uploaded datasets - * @returns Array of upload IDs for tracking - */ - function enqueue(items: NewUploadItem[], collectionConfig?: CollectionConfig): string[] { - // Determine if this batch can use direct HDCA creation - const isDirectCreation = - collectionConfig !== undefined && items.every((item) => item.uploadMode !== "data-library"); - - // Create batch first if collection config provided - const batchId = collectionConfig ? uploadState.addBatch(collectionConfig, [], isDirectCreation) : undefined; - - // Add upload items with batch ID - const ids = items.map((item) => uploadState.addUploadItem(item, batchId)); - - // Update batch with upload IDs and create internal batch for tracking - if (batchId) { - const batch = uploadState.getBatch(batchId); - if (batch) { - batch.uploadIds = ids; - } - - batches.push({ - batchId, - ids, - items, - datasetIds: [], - collectionConfig: collectionConfig!, - }); - } - - queue.push(...ids); - processNext(); - return ids; - } - - /** - * Removes batches that no longer have any active upload items. - * Called when uploads are cleared from the state. - */ - function cleanupOrphanedBatches(): void { - for (let i = batches.length - 1; i >= 0; i--) { - const batch = batches[i]!; - const hasActiveUploads = batch.ids.some((id) => findUploadItem(id) !== undefined); - - if (!hasActiveUploads) { - batches.splice(i, 1); - } - } - } - - /** - * Recovers incomplete collection creation on initialization. - * Checks for batches where uploads completed but collection wasn't created. - * - * For direct-creation batches, there's no separate collection creation step - * to recover — if the upload was interrupted, the user must re-upload. - */ - function recoverIncompleteBatches(): void { - uploadState.activeBatches.value.forEach((batch) => { - // Skip if collection already created or batch has errors - if (batch.collectionId || batch.status === "error") { - return; - } - - // Direct-creation batches don't have a separate collection creation step - if (batch.directCreation) { - return; - } - - // Two-step batch recovery (data-library fallback path) - // Check if all uploads in batch are completed - const allCompleted = batch.uploadIds.every((uploadId) => { - const upload = findUploadItem(uploadId); - return upload?.status === "completed"; - }); - - // If uploads are complete and we have dataset IDs, try to create the collection - if (allCompleted && batch.uploadIds.length > 0 && batch.datasetIds.length > 0) { - console.log(`Recovering incomplete batch: ${batch.name}`); - - // Check if we still have the upload items (they might be lost after refresh) - const availableItems = batch.uploadIds.filter((uploadId) => findUploadItem(uploadId) !== undefined); - - if (availableItems.length !== batch.uploadIds.length) { - uploadState.setBatchError( - batch.id, - `Collection creation failed: upload data lost after page refresh. Please re-upload the files to create the collection or manually create the collection.`, - ); - return; - } - - // Attempt to create the collection - createCollection(batch.id).catch((err) => { - console.error("Recovery failed:", err); - uploadState.setBatchError(batch.id, "Collection creation interrupted. Please retry manually."); - }); - } else if (allCompleted && batch.uploadIds.length > 0) { - // Uploads complete but no dataset IDs (shouldn't happen, but handle gracefully) - uploadState.setBatchError( - batch.id, - "Collection creation interrupted and cannot be recovered. Dataset IDs not available. Please create the collection manually.", - ); - } - }); - } - - // Initialize: recover any incomplete batches from previous session - recoverIncompleteBatches(); - - /** Removes all completed uploads from the state */ - function clearCompleted(): void { - uploadState.clearCompleted(); - cleanupOrphanedBatches(); - } - - /** Clears all uploads from the state */ - function clearAll(): void { - uploadState.clearAll(); - batches.length = 0; // Clear all batches when clearing all uploads - } - - return { - enqueue, - clearCompleted, - clearAll, - retryCollectionCreation, - /** Access to upload state for UI components */ - state: uploadState, - }; -} diff --git a/client/src/utils/upload.ts b/client/src/utils/upload.ts index df52f47e8d3..aa1a6fe28d1 100644 --- a/client/src/utils/upload.ts +++ b/client/src/utils/upload.ts @@ -52,8 +52,9 @@ import { guessInitialFilterType, guessNameForPair, } from "@/components/Collections/pairing"; +import type { PreparedUpload } from "@/components/Panels/Upload/types"; import type { UploadRowModel } from "@/components/Upload/model"; -import type { SupportedCollectionType } from "@/composables/upload/collectionTypes"; +import type { SupportedCollectionType, UploadCollectionConfig } from "@/composables/upload/collectionTypes"; import type { NewUploadItem } from "@/composables/upload/uploadItemTypes"; import { getAppRoot } from "@/onload/loadConfig"; import { errorMessageAsString } from "@/utils/simple-error"; @@ -1006,6 +1007,39 @@ export async function uploadDatasets(items: ApiUploadItem[], config: UploadDatas } } +/** + * Determines if an upload item is compatible with the Fetch API (i.e., can be included in the payload). + * Items with uploadMode "data-library" are not compatible because they are copied from the data library and require a different handling approach. + */ +export function isFetchApiCompatible(item: NewUploadItem): boolean { + return item.uploadMode !== "data-library"; +} + +/** + * Builds a PreparedUpload object from UI upload items. + */ +export function buildPreparedUpload(items: NewUploadItem[], collectionConfig?: UploadCollectionConfig): PreparedUpload { + return buildPreparedUploadWithOptions(items, collectionConfig); +} + +interface PreparedUploadBuildOptions { + apiItems?: ApiUploadItem[]; + uploadOptions?: PreparedUpload["uploadOptions"]; +} + +export function buildPreparedUploadWithOptions( + items: NewUploadItem[], + collectionConfig?: UploadCollectionConfig, + options?: PreparedUploadBuildOptions, +): PreparedUpload { + return { + apiItems: options?.apiItems ?? items.filter(isFetchApiCompatible).map((item) => toApiUploadItem(item)), + collectionConfig, + uploadItems: items, + uploadOptions: options?.uploadOptions, + }; +} + /** * Uploads datasets as a collection directly via a single /api/tools/fetch request. * Uses HdcaDataItemsTarget to create the collection atomically during the fetch.
+ No {{ urlTracker.isAtRoot.value ? "file sources" : "files or folders" }} match your search "{{ + searchQuery + }}" +
+ No file sources available. Connect a new remote source to get started. +
This directory is empty.