From ae17260a725942f1a3b568bf01ec4772dba05dfc Mon Sep 17 00:00:00 2001 From: overtrue Date: Mon, 7 Jul 2025 22:58:40 +0800 Subject: [PATCH] fix: rustfs/rustfs#80 --- components.d.ts | 2 +- components/object/list.vue | 6 +- components/object/upload/picker.vue | 77 +++++--- config/navs.ts | 2 +- lib/delete-task-manager.ts | 257 ++++++++++++++++++++---- lib/upload-task-manager.ts | 293 +++++++++++++++++++++------- store/delete-tasks.ts | 48 ++++- store/upload-tasks.ts | 47 ++++- 8 files changed, 587 insertions(+), 145 deletions(-) diff --git a/components.d.ts b/components.d.ts index 5d113e6..de0fa14 100644 --- a/components.d.ts +++ b/components.d.ts @@ -40,7 +40,6 @@ declare module 'vue' { NGrid: typeof import('naive-ui')['NGrid'] NH3: typeof import('naive-ui')['NH3'] NH4: typeof import('naive-ui')['NH4'] - NIcon: typeof import('naive-ui')['NIcon'] NInput: typeof import('naive-ui')['NInput'] NInputGroup: typeof import('naive-ui')['NInputGroup'] NInputGroupLabel: typeof import('naive-ui')['NInputGroupLabel'] @@ -72,6 +71,7 @@ declare module 'vue' { NText: typeof import('naive-ui')['NText'] NThing: typeof import('naive-ui')['NThing'] NTooltip: typeof import('naive-ui')['NTooltip'] + NVirtualList: typeof import('naive-ui')['NVirtualList'] RouterLink: typeof import('vue-router')['RouterLink'] RouterView: typeof import('vue-router')['RouterView'] } diff --git a/components/object/list.vue b/components/object/list.vue index 4516e22..7027dc2 100644 --- a/components/object/list.vue +++ b/components/object/list.vue @@ -388,9 +388,9 @@ function handleBatchDelete() { return; } - // 执行删除 - await Promise.all(deletableKeys.map((item) => deleteTaskStore.addKeys([String(item)], bucketName.value))); - message.success("删除成功"); + // 执行删除 - 一次性添加所有文件 + deleteTaskStore.addKeys(deletableKeys.map(String), bucketName.value); + message.success("删除任务已创建"); salt.value = randomString(); refresh(); } catch (error) { diff --git a/components/object/upload/picker.vue b/components/object/upload/picker.vue index 21863e8..b45cfee 100644 --- a/components/object/upload/picker.vue +++ b/components/object/upload/picker.vue @@ -31,17 +31,10 @@ - - - - -
+ + + +
+ + + +
- {{ t('Configure') }} - {{ t('Start Upload') }} + {{ t('Configure') }} + {{ t('Start Upload') }}
@@ -94,6 +93,9 @@ const folderInput = ref(null) // 待上传的文件/文件夹列表 const selectedItems = ref([]) +const isAdding = ref(false) +const addProgress = ref(0) + const closeModal = () => emit('update:show', false) const selectFile = () => fileInput.value?.click() const selectFolder = () => folderInput.value?.click() @@ -156,13 +158,42 @@ function removeItem(index: number) { selectedItems.value.splice(index, 1) } -function handleUpload() { - console.log('uploading', allFiles.value); - allFiles.value.forEach(fileItem => { - uploadTaskManagerStore.addFiles([fileItem.file], props.bucketName, fileItem.prefix) - }) - selectedItems.value = [] - emit('submit') - closeModal() +async function handleUpload() { + const filesToUpload = allFiles.value; + if (!filesToUpload.length) return; + + isAdding.value = true; + addProgress.value = 0; + + // 分组 + const uploadGroups = new Map(); + filesToUpload.forEach(fileItem => { + const key = `${props.bucketName}:${fileItem.prefix}`; + if (!uploadGroups.has(key)) uploadGroups.set(key, []); + uploadGroups.get(key)!.push(fileItem.file); + }); + + // 统计总数 + const total = Array.from(uploadGroups.values()).reduce((sum, arr) => sum + arr.length, 0); + let done = 0; + + // 分批异步添加，避免阻塞主线程 + for (const [key, files] of uploadGroups.entries()) { + const [bucketName, prefix] = key.split(':'); + for (let i = 0; i < files.length; i += 50) { + const batch = files.slice(i, i + 50); + // 让出主线程，保证 UI 可响应 + await new Promise(resolve => setTimeout(resolve, 0)); + await uploadTaskManagerStore.addFiles(batch, bucketName, prefix); + done += batch.length; + addProgress.value = Math.round((done / total) * 100); + } + } + + isAdding.value = false; + addProgress.value = 100; + selectedItems.value = []; + emit('submit'); + closeModal(); } diff --git a/config/navs.ts b/config/navs.ts index bda6f81..11d75e5 100644 --- a/config/navs.ts +++ b/config/navs.ts @@ -86,7 +86,7 @@ export default [ }, { label: "Documentation", - to: "https://rustfs.com/docs/", + to: "https://docs.rustfs.com/", target: "_blank", icon: "ri:file-list-3-line", }, diff --git a/lib/delete-task-manager.ts b/lib/delete-task-manager.ts index 45bf9d7..52220fe 100644 --- a/lib/delete-task-manager.ts +++ b/lib/delete-task-manager.ts @@ -9,11 +9,14 @@ export interface DeleteTask { progress: number; error?: string; abortController?: AbortController; + retryCount?: number; } export interface TaskManagerConfig { chunkSize?: number; // in MB maxConcurrentUploads?: number; + maxRetries?: number; + retryDelay?: number; } class DeleteTaskManager { @@ -21,48 +24,98 @@ class DeleteTaskManager { private maxConcurrentUploads: number; private chunkSize: number; private s3Client: S3Client; - private activeUploads: number = 0; + private activeDeletes: number = 0; + private maxRetries: number; + private retryDelay: number; + private isStarted: boolean = false; + private isProcessing: boolean = false; - constructor(s3Client: S3Client, config: TaskManagerConfig) { + constructor(s3Client: S3Client, config: TaskManagerConfig = {}) { this.s3Client = s3Client; this.chunkSize = (config.chunkSize ?? 5) * 1024 * 1024; // Default 5MB this.maxConcurrentUploads = config.maxConcurrentUploads ?? 6; + this.maxRetries = config.maxRetries ?? 3; + this.retryDelay = config.retryDelay ?? 1000; } addKeys(keys: string[], bucketName: string, prefix?: string) { - keys.forEach((key) => { - const task: DeleteTask = { - id: `${Date.now()}-${key}`, - key, - status: "pending", - progress: 0, - bucketName, - prefix: prefix ?? "", - error: undefined, - abortController: undefined, - }; - this.tasks.push(task); + const newTasks = keys.map((key) => ({ + id: `${Date.now()}-${Math.random().toString(36).substr(2, 9)}-${key}`, + key, + status: "pending" as const, + progress: 0, + bucketName, + prefix: prefix ?? "", + error: undefined, + abortController: undefined, + retryCount: 0, + })); + + // 批量添加任务 + this.tasks.push(...newTasks); + console.log("🗑️ Added keys to delete queue:", keys.length, "keys"); + console.log("🔍 Current state:", { + isStarted: this.isStarted, + isProcessing: this.isProcessing, + activeDeletes: this.activeDeletes, + totalTasks: this.tasks.length, + pendingTasks: this.tasks.filter(t => t.status === "pending").length }); - - console.debug("Added keys to delete queue", keys); - } - - start() { + // 强制重置状态并立即处理队列 + this.isStarted = true; + this.isProcessing = false; this.processQueue(); } + start() { + console.log("🚀 Starting delete task manager, current state:", { + isStarted: this.isStarted, + isProcessing: this.isProcessing + }); + + if (!this.isStarted) { + this.isStarted = true; + this.isProcessing = false; // 确保重置处理状态 + } + + // 立即处理队列 + if (!this.isProcessing) { + this.processQueue(); + } + } + + stop() { + this.isStarted = false; + this.isProcessing = false; + console.log("⏹️ Stopping delete task manager"); + } + cancel() { + console.log("❌ Canceling all delete tasks"); + + // 取消所有待处理的任务 this.tasks .filter((task) => task.status === "pending") .forEach((task) => { task.status = "canceled"; }); + + // 取消所有正在进行的删除 + this.tasks + .filter((task) => task.status === "deleting" && task.abortController) + .forEach((task) => { + task.abortController?.abort(); + task.status = "canceled"; + }); } cancelTask(taskId: string) { const task = this.tasks.find((task) => task.id === taskId); - if (task && task.abortController) { - task.abortController.abort(); + if (task) { + console.log("❌ Canceling task:", taskId); + if (task.abortController) { + task.abortController.abort(); + } task.status = "canceled"; } } @@ -70,7 +123,13 @@ class DeleteTaskManager { removeTask(taskId: string) { const taskIndex = this.tasks.findIndex((task) => task.id === taskId); if (taskIndex !== -1) { + // 如果任务正在进行，先取消它 + const task = this.tasks[taskIndex]; + if (task.status === "deleting" && task.abortController) { + task.abortController.abort(); + } this.tasks.splice(taskIndex, 1); + console.log("🗑️ Removed task:", taskId); } } @@ -79,40 +138,156 @@ class DeleteTaskManager { } clearTasks() { - this.tasks = []; + console.log("🧹 Clearing all delete tasks"); + // 先取消所有正在进行的任务 + this.cancel(); + // 清空任务列表 + this.tasks.splice(0, this.tasks.length); + // 重置所有状态 + this.activeDeletes = 0; + this.isStarted = false; + this.isProcessing = false; } + // 重写队列处理逻辑，使用更简单的方式 private async processQueue() { - while (this.activeUploads < this.maxConcurrentUploads) { - const nextTask = this.tasks.find((task) => task.status === "pending"); - if (!nextTask) { - break; + if (!this.isStarted) { + console.log("⏸️ Task manager not started, skipping queue processing"); + return; + } + + if (this.isProcessing) { + console.log("⏸️ Already processing queue, skipping"); + return; + } + + this.isProcessing = true; + console.log("🔄 Starting queue processing"); + + try { + const pendingTasks = this.tasks.filter(task => task.status === "pending"); + console.log(`🔄 Processing queue: ${pendingTasks.length} pending, ${this.activeDeletes} active`); + + // 启动新的删除任务 + while (this.activeDeletes < this.maxConcurrentUploads && pendingTasks.length > 0) { + const nextTask = pendingTasks.shift(); + if (!nextTask) break; + + this.activeDeletes++; + console.log(`▶️ Starting delete task: ${nextTask.id} (${nextTask.key})`); + + // 立即启动删除任务，不等待 + this.executeDelete(nextTask); } - this.activeUploads++; - nextTask.status = "deleting"; - - try { - if (nextTask.key) { - await this.deleteObject(nextTask); - nextTask.status = "completed"; - } else { - throw new Error("Task key is undefined"); + // 如果还有待处理的任务，延迟继续处理 + const remainingPending = this.tasks.filter(task => task.status === "pending").length; + if (remainingPending > 0) { + console.log(`⏳ ${remainingPending} tasks remaining, will retry in 1 second`); + setTimeout(() => { + this.isProcessing = false; + this.processQueue(); + }, 1000); + } else { + console.log("✅ No more pending tasks, queue processing complete"); + this.isProcessing = false; + // 所有任务完成时自动重置 isStarted + if (this.activeDeletes === 0) { + this.isStarted = false; } - } catch (error: any) { - nextTask.status = "failed"; - nextTask.error = error.message; - } finally { - this.activeUploads--; - this.processQueue(); // Continue with the next task + } + } catch (error) { + console.error("❌ Error in processQueue:", error); + this.isProcessing = false; + } + } + + private async executeDelete(task: DeleteTask) { + task.status = "deleting"; + task.progress = 0; + + try { + console.log(`🗑️ Deleting: ${task.key}`); + await this.deleteObject(task); + task.status = "completed"; + task.progress = 100; + console.log(`✅ Deleted successfully: ${task.key}`); + } catch (error: any) { + console.error(`❌ Delete failed for ${task.key}:`, error.message); + + // 检查是否是取消操作 + if (error.name === 'AbortError' || error.message?.includes('canceled')) { + task.status = "canceled"; + console.log(`❌ Delete canceled: ${task.key}`); + return; + } + + // 检查是否应该重试 + if (this.shouldRetry(task, error)) { + console.log(`🔄 Will retry delete: ${task.key} (attempt ${(task.retryCount || 0) + 1})`); + await this.retryTask(task); + } else { + task.status = "failed"; + task.error = error.message; + console.log(`💥 Delete permanently failed: ${task.key}`); + } + } finally { + this.activeDeletes--; + console.log(`📊 Delete completed, ${this.activeDeletes} active deletes remaining`); + + // 继续处理队列 - 这是关键的修复 + if (this.isStarted) { + setTimeout(() => { + if (!this.isProcessing) { + this.processQueue(); + } + }, 100); } } } + private shouldRetry(task: DeleteTask, error: any): boolean { + // 如果任务被取消，不重试 + if (task.status === "canceled") { + return false; + } + + // 如果达到最大重试次数，不重试 + if ((task.retryCount || 0) >= this.maxRetries) { + return false; + } + + // 检查错误类型，某些错误不应该重试 + const errorMessage = error.message?.toLowerCase() || ''; + const nonRetryableErrors = [ + 'access denied', + 'forbidden', + 'invalid credentials', + 'bucket not found', + 'no such key' + ]; + + return !nonRetryableErrors.some(msg => errorMessage.includes(msg)); + } + + private async retryTask(task: DeleteTask) { + task.retryCount = (task.retryCount || 0) + 1; + task.status = "pending"; + task.progress = 0; + task.error = undefined; + + // 等待一段时间后重试 + const retryCount = task.retryCount || 1; + await new Promise(resolve => setTimeout(resolve, this.retryDelay * retryCount)); + + console.log(`🔄 Retrying delete task ${task.id}, attempt ${retryCount}`); + } + private async deleteObject(task: DeleteTask) { const { key, bucketName, prefix } = task; const abortController = new AbortController(); task.abortController = abortController; + const command = new DeleteObjectCommand({ Bucket: bucketName, Key: prefix + key, diff --git a/lib/upload-task-manager.ts b/lib/upload-task-manager.ts index 7ebf8fc..7901c28 100644 --- a/lib/upload-task-manager.ts +++ b/lib/upload-task-manager.ts @@ -9,11 +9,14 @@ export interface UploadTask { progress: number; error?: string; abortController?: AbortController; + retryCount?: number; } export interface TaskManagerConfig { chunkSize?: number; // in MB maxConcurrentUploads?: number; + maxRetries?: number; + retryDelay?: number; } class UploadTaskManager { @@ -22,45 +25,71 @@ class UploadTaskManager { private chunkSize: number; private s3Client: S3Client; private activeUploads: number = 0; + private maxRetries: number; + private retryDelay: number; + private isStarted: boolean = false; - constructor(s3Client: S3Client, config: TaskManagerConfig) { + constructor(s3Client: S3Client, config: TaskManagerConfig = {}) { this.s3Client = s3Client; this.chunkSize = (config.chunkSize ?? 5) * 1024 * 1024; // Default 5MB this.maxConcurrentUploads = config.maxConcurrentUploads ?? 6; + this.maxRetries = config.maxRetries ?? 3; + this.retryDelay = config.retryDelay ?? 1000; } addFiles(files: File[], bucketName: string, prefix?: string) { - files.forEach((file) => { - const task: UploadTask = { - id: `${Date.now()}-${file.name}`, - file, - status: "pending", - progress: 0, - bucketName, - prefix: prefix ?? "", - }; - this.tasks.push(task); - }); + const newTasks = files.map((file) => ({ + id: `${Date.now()}-${Math.random().toString(36).substr(2, 9)}-${file.name}`, + file, + status: "pending" as const, + progress: 0, + bucketName, + prefix: prefix ?? "", + retryCount: 0, + })); - console.debug("Added files to upload queue", files); + // 批量添加任务 + this.tasks.push(...newTasks); + console.debug("Added files to upload queue", files.length, "files"); + + // 启动处理（如果尚未启动） + this.start(); } start() { - this.processQueue(); + if (!this.isStarted) { + this.isStarted = true; + this.processQueue(); + } + } + + stop() { + this.isStarted = false; } cancel() { + // 取消所有待处理的任务 this.tasks .filter((task) => task.status === "pending") .forEach((task) => { task.status = "canceled"; }); + + // 取消所有正在进行的上传 + this.tasks + .filter((task) => task.status === "uploading" && task.abortController) + .forEach((task) => { + task.abortController?.abort(); + task.status = "canceled"; + }); } cancelTask(taskId: string) { const task = this.tasks.find((task) => task.id === taskId); - if (task && task.abortController) { - task.abortController.abort(); + if (task) { + if (task.abortController) { + task.abortController.abort(); + } task.status = "canceled"; } } @@ -68,6 +97,11 @@ class UploadTaskManager { removeTask(taskId: string) { const taskIndex = this.tasks.findIndex((task) => task.id === taskId); if (taskIndex !== -1) { + // 如果任务正在进行，先取消它 + const task = this.tasks[taskIndex]; + if (task.status === "uploading" && task.abortController) { + task.abortController.abort(); + } this.tasks.splice(taskIndex, 1); } } @@ -77,46 +111,126 @@ class UploadTaskManager { } clearTasks() { - this.tasks = []; + // 先取消所有正在进行的任务 + this.cancel(); + // 清空任务列表 + this.tasks.splice(0, this.tasks.length); + // 重置状态 + this.activeUploads = 0; + this.isStarted = false; } + // 简化的队列处理逻辑 private async processQueue() { + if (!this.isStarted) { + return; + } + + // 启动新的上传任务 while (this.activeUploads < this.maxConcurrentUploads) { const nextTask = this.tasks.find((task) => task.status === "pending"); - if (!nextTask) { break; } + if (!nextTask) { + break; + } this.activeUploads++; - nextTask.status = "uploading"; + // 不等待，直接启动异步上传 + this.startUpload(nextTask).catch(console.error); + } - try { - if (nextTask.file.size > this.chunkSize) { - await this.multipartUpload(nextTask); - } else { - await this.putObject(nextTask); - } - nextTask.status = "completed"; - } catch (error: any) { - nextTask.status = "failed"; - nextTask.error = error.message; - } finally { - this.activeUploads--; - this.processQueue(); // Continue with the next task + // 如果还有待处理的任务，继续处理 + if (this.tasks.some(task => task.status === "pending")) { + // 使用 setTimeout 避免阻塞主线程 + setTimeout(() => { + this.processQueue(); + }, 100); + } + } + + private async startUpload(task: UploadTask) { + task.status = "uploading"; + task.progress = 0; + + try { + if (task.file.size > this.chunkSize) { + await this.multipartUpload(task); + } else { + await this.putObject(task); + } + task.status = "completed"; + task.progress = 100; + } catch (error: any) { + // 检查是否是取消操作 + if (error.name === 'AbortError' || error.message?.includes('canceled')) { + task.status = "canceled"; + return; + } + + console.error("Upload failed:", error.message, "Task:", task.id); + + // 检查是否应该重试 + if (this.shouldRetry(task, error)) { + await this.retryTask(task); + } else { + task.status = "failed"; + task.error = error.message; + } + } finally { + this.activeUploads--; + // 继续处理队列 + if (this.isStarted) { + setTimeout(() => this.processQueue(), 50); } } } + private shouldRetry(task: UploadTask, error: any): boolean { + // 如果任务被取消，不重试 + if (task.status === "canceled") { + return false; + } + + // 如果达到最大重试次数，不重试 + if ((task.retryCount || 0) >= this.maxRetries) { + return false; + } + + // 检查错误类型，某些错误不应该重试 + const errorMessage = error.message?.toLowerCase() || ''; + const nonRetryableErrors = [ + 'access denied', + 'forbidden', + 'invalid credentials', + 'bucket not found', + 'file not found' + ]; + + return !nonRetryableErrors.some(msg => errorMessage.includes(msg)); + } + + private async retryTask(task: UploadTask) { + task.retryCount = (task.retryCount || 0) + 1; + task.status = "pending"; + task.progress = 0; + task.error = undefined; + + // 等待一段时间后重试 + const retryCount = task.retryCount || 1; + await new Promise(resolve => setTimeout(resolve, this.retryDelay * retryCount)); + + console.debug(`Retrying task ${task.id}, attempt ${retryCount}`); + } + private async putObject(task: UploadTask) { const { file, bucketName, prefix } = task; const abortController = new AbortController(); task.abortController = abortController; - console.log('putObject', task); - const fileBuffer = await file.arrayBuffer(); const command = new PutObjectCommand({ Bucket: bucketName, - Key: prefix + task.file.name, + Key: prefix + file.name, Body: file, - ContentType: file.type, + ContentType: file.type || 'application/octet-stream', }); await this.s3Client.send(command, { abortSignal: abortController.signal }); @@ -128,43 +242,90 @@ class UploadTaskManager { const abortController = new AbortController(); task.abortController = abortController; - const Key = prefix + task.file.name; - const createCommand = new CreateMultipartUploadCommand({ - Bucket: bucketName, - Key: Key - }); - const { UploadId } = await this.s3Client.send(createCommand); + const Key = prefix + file.name; + let uploadId: string | undefined; - const totalChunks = Math.ceil(file.size / this.chunkSize); - let completedParts = []; - - for (let partNumber = 1; partNumber <= totalChunks; partNumber++) { - const start = (partNumber - 1) * this.chunkSize; - const end = Math.min(start + this.chunkSize, file.size); - - const chunk = file.slice(start, end); - const uploadPartCommand = new UploadPartCommand({ + try { + // 创建分片上传 + const createCommand = new CreateMultipartUploadCommand({ Bucket: bucketName, Key: Key, - UploadId, - PartNumber: partNumber, - Body: chunk, + ContentType: file.type || 'application/octet-stream', }); - const { ETag } = await this.s3Client.send(uploadPartCommand, { abortSignal: abortController.signal }); - completedParts.push({ ETag, PartNumber: partNumber }); + const createResponse = await this.s3Client.send(createCommand, { + abortSignal: abortController.signal + }); + uploadId = createResponse.UploadId; - task.progress = Math.round((partNumber / totalChunks) * 100); + if (!uploadId) { + throw new Error('Failed to create multipart upload'); + } + + const totalChunks = Math.ceil(file.size / this.chunkSize); + const completedParts = []; + + // 上传分片 + for (let partNumber = 1; partNumber <= totalChunks; partNumber++) { + // 检查是否被取消 + if (abortController.signal.aborted) { + throw new Error('Upload canceled'); + } + + const start = (partNumber - 1) * this.chunkSize; + const end = Math.min(start + this.chunkSize, file.size); + const chunk = file.slice(start, end); + + const uploadPartCommand = new UploadPartCommand({ + Bucket: bucketName, + Key: Key, + UploadId: uploadId, + PartNumber: partNumber, + Body: chunk, + }); + + const { ETag } = await this.s3Client.send(uploadPartCommand, { + abortSignal: abortController.signal + }); + + if (!ETag) { + throw new Error(`Failed to upload part ${partNumber}`); + } + + completedParts.push({ ETag, PartNumber: partNumber }); + task.progress = Math.round((partNumber / totalChunks) * 100); + } + + // 完成分片上传 + const completeCommand = new CompleteMultipartUploadCommand({ + Bucket: bucketName, + Key: Key, + UploadId: uploadId, + MultipartUpload: { Parts: completedParts }, + }); + + await this.s3Client.send(completeCommand, { + abortSignal: abortController.signal + }); + + } catch (error: any) { + // 如果上传失败，尝试清理分片上传 + if (uploadId) { + try { + const { AbortMultipartUploadCommand } = await import("@aws-sdk/client-s3"); + const abortCommand = new AbortMultipartUploadCommand({ + Bucket: bucketName, + Key: Key, + UploadId: uploadId, + }); + await this.s3Client.send(abortCommand); + } catch (cleanupError) { + console.warn('Failed to cleanup multipart upload:', cleanupError); + } + } + + throw error; } - - const completeCommand = new CompleteMultipartUploadCommand({ - Bucket: bucketName, - Key: Key, - UploadId, - MultipartUpload: { Parts: completedParts }, - }); - - await this.s3Client.send(completeCommand); } } diff --git a/store/delete-tasks.ts b/store/delete-tasks.ts index 8e3f58e..7af4325 100644 --- a/store/delete-tasks.ts +++ b/store/delete-tasks.ts @@ -4,23 +4,48 @@ import DeleteTaskManager from "~/lib/delete-task-manager"; export const useDeleteTaskManagerStore = defineStore("DeleteTaskManager", { state: () => ({ taskManager: new DeleteTaskManager(useNuxtApp().$s3Client, { - chunkSize: 16, - maxConcurrentUploads: 6, + chunkSize: 16, // Not used for delete, kept for consistency + maxConcurrentUploads: 6, // Limit concurrent deletes + maxRetries: 3, // Retry failed deletes + retryDelay: 1000, // Wait 1 second before retrying }), }), getters: { tasks: (state) => state.taskManager.getTasks(), + pendingTasks: (state) => state.taskManager.getTasks().filter(task => task.status === "pending"), + deletingTasks: (state) => state.taskManager.getTasks().filter(task => task.status === "deleting"), + completedTasks: (state) => state.taskManager.getTasks().filter(task => task.status === "completed"), + failedTasks: (state) => state.taskManager.getTasks().filter(task => task.status === "failed"), }, actions: { addKeys(keys: string[], bucketName: string, prefix?: string) { - console.log("addKeys", keys, bucketName, prefix); - this.taskManager.addKeys(keys, bucketName, prefix); - this.taskManager.start(); + console.log("addKeys", keys.length, "keys to bucket", bucketName, "prefix:", prefix); + + // 对于大量文件，分批处理 + const batchSize = 50; + if (keys.length > batchSize) { + console.log(`Processing ${keys.length} keys in batches of ${batchSize}`); + for (let i = 0; i < keys.length; i += batchSize) { + const batch = keys.slice(i, i + batchSize); + this.taskManager.addKeys(batch, bucketName, prefix); + } + } else { + this.taskManager.addKeys(keys, bucketName, prefix); + } }, + getTasks() { return this.taskManager.getTasks(); }, + cancelTask(taskId: string) { + this.taskManager.cancelTask(taskId); + }, + + cancelAllTasks() { + this.taskManager.cancel(); + }, + removeTask(taskId: string) { this.taskManager.removeTask(taskId); }, @@ -28,5 +53,18 @@ export const useDeleteTaskManagerStore = defineStore("DeleteTaskManager", { clearTasks() { this.taskManager.clearTasks(); }, + + // 获取统计信息 + getStats() { + const tasks = this.tasks; + return { + total: tasks.length, + pending: tasks.filter(task => task.status === "pending").length, + deleting: tasks.filter(task => task.status === "deleting").length, + completed: tasks.filter(task => task.status === "completed").length, + failed: tasks.filter(task => task.status === "failed").length, + canceled: tasks.filter(task => task.status === "canceled").length, + }; + }, }, }); diff --git a/store/upload-tasks.ts b/store/upload-tasks.ts index f5dac3a..22662b4 100644 --- a/store/upload-tasks.ts +++ b/store/upload-tasks.ts @@ -4,24 +4,48 @@ import UploadTaskManager from "~/lib/upload-task-manager"; export const useUploadTaskManagerStore = defineStore("uploadTaskManager", { state: () => ({ taskManager: new UploadTaskManager(useNuxtApp().$s3Client, { - chunkSize: 16, - maxConcurrentUploads: 6, + chunkSize: 16, // 16MB chunks for better performance + maxConcurrentUploads: 6, // Limit concurrent uploads + maxRetries: 3, // Retry failed uploads + retryDelay: 1000, // Wait 1 second before retrying }), }), getters: { tasks: (state) => state.taskManager.getTasks(), + pendingTasks: (state) => state.taskManager.getTasks().filter(task => task.status === "pending"), + uploadingTasks: (state) => state.taskManager.getTasks().filter(task => task.status === "uploading"), + completedTasks: (state) => state.taskManager.getTasks().filter(task => task.status === "completed"), + failedTasks: (state) => state.taskManager.getTasks().filter(task => task.status === "failed"), }, actions: { addFiles(files: File[], bucketName: string, prefix?: string) { - console.log("addFiles", files, bucketName, prefix); + console.log("addFiles", files.length, "files to bucket", bucketName, "prefix:", prefix); - this.taskManager.addFiles(files, bucketName, prefix); - this.taskManager.start(); + // 对于大量文件，分批处理 + const batchSize = 50; + if (files.length > batchSize) { + console.log(`Processing ${files.length} files in batches of ${batchSize}`); + for (let i = 0; i < files.length; i += batchSize) { + const batch = files.slice(i, i + batchSize); + this.taskManager.addFiles(batch, bucketName, prefix); + } + } else { + this.taskManager.addFiles(files, bucketName, prefix); + } }, + getTasks() { return this.taskManager.getTasks(); }, + cancelTask(taskId: string) { + this.taskManager.cancelTask(taskId); + }, + + cancelAllTasks() { + this.taskManager.cancel(); + }, + removeTask(taskId: string) { this.taskManager.removeTask(taskId); }, @@ -29,5 +53,18 @@ export const useUploadTaskManagerStore = defineStore("uploadTaskManager", { clearTasks() { this.taskManager.clearTasks(); }, + + // 获取统计信息 + getStats() { + const tasks = this.tasks; + return { + total: tasks.length, + pending: tasks.filter(task => task.status === "pending").length, + uploading: tasks.filter(task => task.status === "uploading").length, + completed: tasks.filter(task => task.status === "completed").length, + failed: tasks.filter(task => task.status === "failed").length, + canceled: tasks.filter(task => task.status === "canceled").length, + }; + }, }, });