This commit is contained in:
overtrue
2025-07-07 22:58:40 +08:00
parent f3bb968cd8
commit ae17260a72
8 changed files with 587 additions and 145 deletions
+1 -1
View File
@@ -40,7 +40,6 @@ declare module 'vue' {
NGrid: typeof import('naive-ui')['NGrid']
NH3: typeof import('naive-ui')['NH3']
NH4: typeof import('naive-ui')['NH4']
NIcon: typeof import('naive-ui')['NIcon']
NInput: typeof import('naive-ui')['NInput']
NInputGroup: typeof import('naive-ui')['NInputGroup']
NInputGroupLabel: typeof import('naive-ui')['NInputGroupLabel']
@@ -72,6 +71,7 @@ declare module 'vue' {
NText: typeof import('naive-ui')['NText']
NThing: typeof import('naive-ui')['NThing']
NTooltip: typeof import('naive-ui')['NTooltip']
NVirtualList: typeof import('naive-ui')['NVirtualList']
RouterLink: typeof import('vue-router')['RouterLink']
RouterView: typeof import('vue-router')['RouterView']
}
+3 -3
View File
@@ -388,9 +388,9 @@ function handleBatchDelete() {
return;
}
// 执行删除
await Promise.all(deletableKeys.map((item) => deleteTaskStore.addKeys([String(item)], bucketName.value)));
message.success("删除成功");
// 执行删除 - 一次性添加所有文件
deleteTaskStore.addKeys(deletableKeys.map(String), bucketName.value);
message.success("删除任务已创建");
salt.value = randomString();
refresh();
} catch (error) {
+54 -23
View File
@@ -31,17 +31,10 @@
</div>
</div>
<!-- 展示文件与文件夹 -->
<n-list v-if="selectedItems.length" bordered class="h-[40vh] overflow-y-auto sticky-header relative">
<template #header>
<div class="flex items-center gap-2 font-bold">
<div class="flex-1">{{ t('Selected Items') }}</div>
<div class="w-32 text-center">{{ t('Size') }}</div>
<div class="w-32 text-center">{{ t('Operation') }}</div>
</div>
</template>
<n-list-item v-for="(item, index) in selectedItems" :key="index">
<div class="flex items-center gap-2 w-full">
<!-- 虚拟滚动文件与文件夹列表 -->
<n-virtual-list v-if="selectedItems.length" :items="selectedItems" :item-size="40" class="h-[40vh] overflow-y-auto border rounded">
<template #default="{ item, index }">
<div class="flex items-center gap-2 w-full border-b py-1">
<div class="flex-1">
<span v-if="item.type === 'folder'">{{ item.name }}/</span>
<span v-else>{{ item.name }}</span>
@@ -51,13 +44,19 @@
<n-button text type="info" @click="removeItem(index)">{{ t('Delete') }}</n-button>
</div>
</div>
</n-list-item>
</n-list>
</template>
</n-virtual-list>
</div>
<n-progress v-if="isAdding" :percentage="addProgress" type="line" :show-indicator="true" style="margin-bottom: 16px;">
<template #default>
正在添加到上传队列 ({{ addProgress }}%)
</template>
</n-progress>
<div class="flex justify-center gap-4">
<n-button type="default" :disabled="!hasFiles">{{ t('Configure') }}</n-button>
<n-button type="primary" :disabled="!hasFiles" @click="handleUpload">{{ t('Start Upload') }}</n-button>
<n-button type="default" :disabled="!hasFiles || isAdding">{{ t('Configure') }}</n-button>
<n-button type="primary" :disabled="!hasFiles || isAdding" :loading="isAdding" @click="handleUpload">{{ t('Start Upload') }}</n-button>
</div>
</div>
</n-card>
@@ -94,6 +93,9 @@ const folderInput = ref<HTMLInputElement | null>(null)
// 待上传的文件/文件夹列表
const selectedItems = ref<SelectedItem[]>([])
const isAdding = ref(false)
const addProgress = ref(0)
const closeModal = () => emit('update:show', false)
const selectFile = () => fileInput.value?.click()
const selectFolder = () => folderInput.value?.click()
@@ -156,13 +158,42 @@ function removeItem(index: number) {
selectedItems.value.splice(index, 1)
}
function handleUpload() {
console.log('uploading', allFiles.value);
allFiles.value.forEach(fileItem => {
uploadTaskManagerStore.addFiles([fileItem.file], props.bucketName, fileItem.prefix)
})
selectedItems.value = []
emit('submit')
closeModal()
async function handleUpload() {
const filesToUpload = allFiles.value;
if (!filesToUpload.length) return;
isAdding.value = true;
addProgress.value = 0;
// 分组
const uploadGroups = new Map<string, File[]>();
filesToUpload.forEach(fileItem => {
const key = `${props.bucketName}:${fileItem.prefix}`;
if (!uploadGroups.has(key)) uploadGroups.set(key, []);
uploadGroups.get(key)!.push(fileItem.file);
});
// 统计总数
const total = Array.from(uploadGroups.values()).reduce((sum, arr) => sum + arr.length, 0);
let done = 0;
// 分批异步添加,避免阻塞主线程
for (const [key, files] of uploadGroups.entries()) {
const [bucketName, prefix] = key.split(':');
for (let i = 0; i < files.length; i += 50) {
const batch = files.slice(i, i + 50);
// 让出主线程,保证 UI 可响应
await new Promise(resolve => setTimeout(resolve, 0));
await uploadTaskManagerStore.addFiles(batch, bucketName, prefix);
done += batch.length;
addProgress.value = Math.round((done / total) * 100);
}
}
isAdding.value = false;
addProgress.value = 100;
selectedItems.value = [];
emit('submit');
closeModal();
}
</script>
+1 -1
View File
@@ -86,7 +86,7 @@ export default [
},
{
label: "Documentation",
to: "https://rustfs.com/docs/",
to: "https://docs.rustfs.com/",
target: "_blank",
icon: "ri:file-list-3-line",
},
+216 -41
View File
@@ -9,11 +9,14 @@ export interface DeleteTask {
progress: number;
error?: string;
abortController?: AbortController;
retryCount?: number;
}
export interface TaskManagerConfig {
chunkSize?: number; // in MB
maxConcurrentUploads?: number;
maxRetries?: number;
retryDelay?: number;
}
class DeleteTaskManager {
@@ -21,48 +24,98 @@ class DeleteTaskManager {
private maxConcurrentUploads: number;
private chunkSize: number;
private s3Client: S3Client;
private activeUploads: number = 0;
private activeDeletes: number = 0;
private maxRetries: number;
private retryDelay: number;
private isStarted: boolean = false;
private isProcessing: boolean = false;
constructor(s3Client: S3Client, config: TaskManagerConfig) {
constructor(s3Client: S3Client, config: TaskManagerConfig = {}) {
this.s3Client = s3Client;
this.chunkSize = (config.chunkSize ?? 5) * 1024 * 1024; // Default 5MB
this.maxConcurrentUploads = config.maxConcurrentUploads ?? 6;
this.maxRetries = config.maxRetries ?? 3;
this.retryDelay = config.retryDelay ?? 1000;
}
addKeys(keys: string[], bucketName: string, prefix?: string) {
keys.forEach((key) => {
const task: DeleteTask = {
id: `${Date.now()}-${key}`,
key,
status: "pending",
progress: 0,
bucketName,
prefix: prefix ?? "",
error: undefined,
abortController: undefined,
};
this.tasks.push(task);
const newTasks = keys.map((key) => ({
id: `${Date.now()}-${Math.random().toString(36).substr(2, 9)}-${key}`,
key,
status: "pending" as const,
progress: 0,
bucketName,
prefix: prefix ?? "",
error: undefined,
abortController: undefined,
retryCount: 0,
}));
// 批量添加任务
this.tasks.push(...newTasks);
console.log("🗑️ Added keys to delete queue:", keys.length, "keys");
console.log("🔍 Current state:", {
isStarted: this.isStarted,
isProcessing: this.isProcessing,
activeDeletes: this.activeDeletes,
totalTasks: this.tasks.length,
pendingTasks: this.tasks.filter(t => t.status === "pending").length
});
console.debug("Added keys to delete queue", keys);
}
start() {
// 强制重置状态并立即处理队列
this.isStarted = true;
this.isProcessing = false;
this.processQueue();
}
start() {
console.log("🚀 Starting delete task manager, current state:", {
isStarted: this.isStarted,
isProcessing: this.isProcessing
});
if (!this.isStarted) {
this.isStarted = true;
this.isProcessing = false; // 确保重置处理状态
}
// 立即处理队列
if (!this.isProcessing) {
this.processQueue();
}
}
stop() {
this.isStarted = false;
this.isProcessing = false;
console.log("⏹️ Stopping delete task manager");
}
cancel() {
console.log("❌ Canceling all delete tasks");
// 取消所有待处理的任务
this.tasks
.filter((task) => task.status === "pending")
.forEach((task) => {
task.status = "canceled";
});
// 取消所有正在进行的删除
this.tasks
.filter((task) => task.status === "deleting" && task.abortController)
.forEach((task) => {
task.abortController?.abort();
task.status = "canceled";
});
}
cancelTask(taskId: string) {
const task = this.tasks.find((task) => task.id === taskId);
if (task && task.abortController) {
task.abortController.abort();
if (task) {
console.log("❌ Canceling task:", taskId);
if (task.abortController) {
task.abortController.abort();
}
task.status = "canceled";
}
}
@@ -70,7 +123,13 @@ class DeleteTaskManager {
removeTask(taskId: string) {
const taskIndex = this.tasks.findIndex((task) => task.id === taskId);
if (taskIndex !== -1) {
// 如果任务正在进行,先取消它
const task = this.tasks[taskIndex];
if (task.status === "deleting" && task.abortController) {
task.abortController.abort();
}
this.tasks.splice(taskIndex, 1);
console.log("🗑️ Removed task:", taskId);
}
}
@@ -79,40 +138,156 @@ class DeleteTaskManager {
}
clearTasks() {
this.tasks = [];
console.log("🧹 Clearing all delete tasks");
// 先取消所有正在进行的任务
this.cancel();
// 清空任务列表
this.tasks.splice(0, this.tasks.length);
// 重置所有状态
this.activeDeletes = 0;
this.isStarted = false;
this.isProcessing = false;
}
// 重写队列处理逻辑,使用更简单的方式
private async processQueue() {
while (this.activeUploads < this.maxConcurrentUploads) {
const nextTask = this.tasks.find((task) => task.status === "pending");
if (!nextTask) {
break;
if (!this.isStarted) {
console.log("⏸️ Task manager not started, skipping queue processing");
return;
}
if (this.isProcessing) {
console.log("⏸️ Already processing queue, skipping");
return;
}
this.isProcessing = true;
console.log("🔄 Starting queue processing");
try {
const pendingTasks = this.tasks.filter(task => task.status === "pending");
console.log(`🔄 Processing queue: ${pendingTasks.length} pending, ${this.activeDeletes} active`);
// 启动新的删除任务
while (this.activeDeletes < this.maxConcurrentUploads && pendingTasks.length > 0) {
const nextTask = pendingTasks.shift();
if (!nextTask) break;
this.activeDeletes++;
console.log(`▶️ Starting delete task: ${nextTask.id} (${nextTask.key})`);
// 立即启动删除任务,不等待
this.executeDelete(nextTask);
}
this.activeUploads++;
nextTask.status = "deleting";
try {
if (nextTask.key) {
await this.deleteObject(nextTask);
nextTask.status = "completed";
} else {
throw new Error("Task key is undefined");
// 如果还有待处理的任务,延迟继续处理
const remainingPending = this.tasks.filter(task => task.status === "pending").length;
if (remainingPending > 0) {
console.log(`⏳ ${remainingPending} tasks remaining, will retry in 1 second`);
setTimeout(() => {
this.isProcessing = false;
this.processQueue();
}, 1000);
} else {
console.log("✅ No more pending tasks, queue processing complete");
this.isProcessing = false;
// 所有任务完成时自动重置 isStarted
if (this.activeDeletes === 0) {
this.isStarted = false;
}
} catch (error: any) {
nextTask.status = "failed";
nextTask.error = error.message;
} finally {
this.activeUploads--;
this.processQueue(); // Continue with the next task
}
} catch (error) {
console.error("❌ Error in processQueue:", error);
this.isProcessing = false;
}
}
private async executeDelete(task: DeleteTask) {
task.status = "deleting";
task.progress = 0;
try {
console.log(`🗑️ Deleting: ${task.key}`);
await this.deleteObject(task);
task.status = "completed";
task.progress = 100;
console.log(`✅ Deleted successfully: ${task.key}`);
} catch (error: any) {
console.error(`❌ Delete failed for ${task.key}:`, error.message);
// 检查是否是取消操作
if (error.name === 'AbortError' || error.message?.includes('canceled')) {
task.status = "canceled";
console.log(`❌ Delete canceled: ${task.key}`);
return;
}
// 检查是否应该重试
if (this.shouldRetry(task, error)) {
console.log(`🔄 Will retry delete: ${task.key} (attempt ${(task.retryCount || 0) + 1})`);
await this.retryTask(task);
} else {
task.status = "failed";
task.error = error.message;
console.log(`💥 Delete permanently failed: ${task.key}`);
}
} finally {
this.activeDeletes--;
console.log(`📊 Delete completed, ${this.activeDeletes} active deletes remaining`);
// 继续处理队列 - 这是关键的修复
if (this.isStarted) {
setTimeout(() => {
if (!this.isProcessing) {
this.processQueue();
}
}, 100);
}
}
}
private shouldRetry(task: DeleteTask, error: any): boolean {
// 如果任务被取消,不重试
if (task.status === "canceled") {
return false;
}
// 如果达到最大重试次数,不重试
if ((task.retryCount || 0) >= this.maxRetries) {
return false;
}
// 检查错误类型,某些错误不应该重试
const errorMessage = error.message?.toLowerCase() || '';
const nonRetryableErrors = [
'access denied',
'forbidden',
'invalid credentials',
'bucket not found',
'no such key'
];
return !nonRetryableErrors.some(msg => errorMessage.includes(msg));
}
private async retryTask(task: DeleteTask) {
task.retryCount = (task.retryCount || 0) + 1;
task.status = "pending";
task.progress = 0;
task.error = undefined;
// 等待一段时间后重试
const retryCount = task.retryCount || 1;
await new Promise(resolve => setTimeout(resolve, this.retryDelay * retryCount));
console.log(`🔄 Retrying delete task ${task.id}, attempt ${retryCount}`);
}
private async deleteObject(task: DeleteTask) {
const { key, bucketName, prefix } = task;
const abortController = new AbortController();
task.abortController = abortController;
const command = new DeleteObjectCommand({
Bucket: bucketName,
Key: prefix + key,
+227 -66
View File
@@ -9,11 +9,14 @@ export interface UploadTask {
progress: number;
error?: string;
abortController?: AbortController;
retryCount?: number;
}
export interface TaskManagerConfig {
chunkSize?: number; // in MB
maxConcurrentUploads?: number;
maxRetries?: number;
retryDelay?: number;
}
class UploadTaskManager {
@@ -22,45 +25,71 @@ class UploadTaskManager {
private chunkSize: number;
private s3Client: S3Client;
private activeUploads: number = 0;
private maxRetries: number;
private retryDelay: number;
private isStarted: boolean = false;
constructor(s3Client: S3Client, config: TaskManagerConfig) {
constructor(s3Client: S3Client, config: TaskManagerConfig = {}) {
this.s3Client = s3Client;
this.chunkSize = (config.chunkSize ?? 5) * 1024 * 1024; // Default 5MB
this.maxConcurrentUploads = config.maxConcurrentUploads ?? 6;
this.maxRetries = config.maxRetries ?? 3;
this.retryDelay = config.retryDelay ?? 1000;
}
addFiles(files: File[], bucketName: string, prefix?: string) {
files.forEach((file) => {
const task: UploadTask = {
id: `${Date.now()}-${file.name}`,
file,
status: "pending",
progress: 0,
bucketName,
prefix: prefix ?? "",
};
this.tasks.push(task);
});
const newTasks = files.map((file) => ({
id: `${Date.now()}-${Math.random().toString(36).substr(2, 9)}-${file.name}`,
file,
status: "pending" as const,
progress: 0,
bucketName,
prefix: prefix ?? "",
retryCount: 0,
}));
console.debug("Added files to upload queue", files);
// 批量添加任务
this.tasks.push(...newTasks);
console.debug("Added files to upload queue", files.length, "files");
// 启动处理(如果尚未启动)
this.start();
}
start() {
this.processQueue();
if (!this.isStarted) {
this.isStarted = true;
this.processQueue();
}
}
stop() {
this.isStarted = false;
}
cancel() {
// 取消所有待处理的任务
this.tasks
.filter((task) => task.status === "pending")
.forEach((task) => {
task.status = "canceled";
});
// 取消所有正在进行的上传
this.tasks
.filter((task) => task.status === "uploading" && task.abortController)
.forEach((task) => {
task.abortController?.abort();
task.status = "canceled";
});
}
cancelTask(taskId: string) {
const task = this.tasks.find((task) => task.id === taskId);
if (task && task.abortController) {
task.abortController.abort();
if (task) {
if (task.abortController) {
task.abortController.abort();
}
task.status = "canceled";
}
}
@@ -68,6 +97,11 @@ class UploadTaskManager {
removeTask(taskId: string) {
const taskIndex = this.tasks.findIndex((task) => task.id === taskId);
if (taskIndex !== -1) {
// 如果任务正在进行,先取消它
const task = this.tasks[taskIndex];
if (task.status === "uploading" && task.abortController) {
task.abortController.abort();
}
this.tasks.splice(taskIndex, 1);
}
}
@@ -77,46 +111,126 @@ class UploadTaskManager {
}
clearTasks() {
this.tasks = [];
// 先取消所有正在进行的任务
this.cancel();
// 清空任务列表
this.tasks.splice(0, this.tasks.length);
// 重置状态
this.activeUploads = 0;
this.isStarted = false;
}
// 简化的队列处理逻辑
private async processQueue() {
if (!this.isStarted) {
return;
}
// 启动新的上传任务
while (this.activeUploads < this.maxConcurrentUploads) {
const nextTask = this.tasks.find((task) => task.status === "pending");
if (!nextTask) { break; }
if (!nextTask) {
break;
}
this.activeUploads++;
nextTask.status = "uploading";
// 不等待,直接启动异步上传
this.startUpload(nextTask).catch(console.error);
}
try {
if (nextTask.file.size > this.chunkSize) {
await this.multipartUpload(nextTask);
} else {
await this.putObject(nextTask);
}
nextTask.status = "completed";
} catch (error: any) {
nextTask.status = "failed";
nextTask.error = error.message;
} finally {
this.activeUploads--;
this.processQueue(); // Continue with the next task
// 如果还有待处理的任务,继续处理
if (this.tasks.some(task => task.status === "pending")) {
// 使用 setTimeout 避免阻塞主线程
setTimeout(() => {
this.processQueue();
}, 100);
}
}
private async startUpload(task: UploadTask) {
task.status = "uploading";
task.progress = 0;
try {
if (task.file.size > this.chunkSize) {
await this.multipartUpload(task);
} else {
await this.putObject(task);
}
task.status = "completed";
task.progress = 100;
} catch (error: any) {
// 检查是否是取消操作
if (error.name === 'AbortError' || error.message?.includes('canceled')) {
task.status = "canceled";
return;
}
console.error("Upload failed:", error.message, "Task:", task.id);
// 检查是否应该重试
if (this.shouldRetry(task, error)) {
await this.retryTask(task);
} else {
task.status = "failed";
task.error = error.message;
}
} finally {
this.activeUploads--;
// 继续处理队列
if (this.isStarted) {
setTimeout(() => this.processQueue(), 50);
}
}
}
private shouldRetry(task: UploadTask, error: any): boolean {
// 如果任务被取消,不重试
if (task.status === "canceled") {
return false;
}
// 如果达到最大重试次数,不重试
if ((task.retryCount || 0) >= this.maxRetries) {
return false;
}
// 检查错误类型,某些错误不应该重试
const errorMessage = error.message?.toLowerCase() || '';
const nonRetryableErrors = [
'access denied',
'forbidden',
'invalid credentials',
'bucket not found',
'file not found'
];
return !nonRetryableErrors.some(msg => errorMessage.includes(msg));
}
private async retryTask(task: UploadTask) {
task.retryCount = (task.retryCount || 0) + 1;
task.status = "pending";
task.progress = 0;
task.error = undefined;
// 等待一段时间后重试
const retryCount = task.retryCount || 1;
await new Promise(resolve => setTimeout(resolve, this.retryDelay * retryCount));
console.debug(`Retrying task ${task.id}, attempt ${retryCount}`);
}
private async putObject(task: UploadTask) {
const { file, bucketName, prefix } = task;
const abortController = new AbortController();
task.abortController = abortController;
console.log('putObject', task);
const fileBuffer = await file.arrayBuffer();
const command = new PutObjectCommand({
Bucket: bucketName,
Key: prefix + task.file.name,
Key: prefix + file.name,
Body: file,
ContentType: file.type,
ContentType: file.type || 'application/octet-stream',
});
await this.s3Client.send(command, { abortSignal: abortController.signal });
@@ -128,43 +242,90 @@ class UploadTaskManager {
const abortController = new AbortController();
task.abortController = abortController;
const Key = prefix + task.file.name;
const createCommand = new CreateMultipartUploadCommand({
Bucket: bucketName,
Key: Key
});
const { UploadId } = await this.s3Client.send(createCommand);
const Key = prefix + file.name;
let uploadId: string | undefined;
const totalChunks = Math.ceil(file.size / this.chunkSize);
let completedParts = [];
for (let partNumber = 1; partNumber <= totalChunks; partNumber++) {
const start = (partNumber - 1) * this.chunkSize;
const end = Math.min(start + this.chunkSize, file.size);
const chunk = file.slice(start, end);
const uploadPartCommand = new UploadPartCommand({
try {
// 创建分片上传
const createCommand = new CreateMultipartUploadCommand({
Bucket: bucketName,
Key: Key,
UploadId,
PartNumber: partNumber,
Body: chunk,
ContentType: file.type || 'application/octet-stream',
});
const { ETag } = await this.s3Client.send(uploadPartCommand, { abortSignal: abortController.signal });
completedParts.push({ ETag, PartNumber: partNumber });
const createResponse = await this.s3Client.send(createCommand, {
abortSignal: abortController.signal
});
uploadId = createResponse.UploadId;
task.progress = Math.round((partNumber / totalChunks) * 100);
if (!uploadId) {
throw new Error('Failed to create multipart upload');
}
const totalChunks = Math.ceil(file.size / this.chunkSize);
const completedParts = [];
// 上传分片
for (let partNumber = 1; partNumber <= totalChunks; partNumber++) {
// 检查是否被取消
if (abortController.signal.aborted) {
throw new Error('Upload canceled');
}
const start = (partNumber - 1) * this.chunkSize;
const end = Math.min(start + this.chunkSize, file.size);
const chunk = file.slice(start, end);
const uploadPartCommand = new UploadPartCommand({
Bucket: bucketName,
Key: Key,
UploadId: uploadId,
PartNumber: partNumber,
Body: chunk,
});
const { ETag } = await this.s3Client.send(uploadPartCommand, {
abortSignal: abortController.signal
});
if (!ETag) {
throw new Error(`Failed to upload part ${partNumber}`);
}
completedParts.push({ ETag, PartNumber: partNumber });
task.progress = Math.round((partNumber / totalChunks) * 100);
}
// 完成分片上传
const completeCommand = new CompleteMultipartUploadCommand({
Bucket: bucketName,
Key: Key,
UploadId: uploadId,
MultipartUpload: { Parts: completedParts },
});
await this.s3Client.send(completeCommand, {
abortSignal: abortController.signal
});
} catch (error: any) {
// 如果上传失败,尝试清理分片上传
if (uploadId) {
try {
const { AbortMultipartUploadCommand } = await import("@aws-sdk/client-s3");
const abortCommand = new AbortMultipartUploadCommand({
Bucket: bucketName,
Key: Key,
UploadId: uploadId,
});
await this.s3Client.send(abortCommand);
} catch (cleanupError) {
console.warn('Failed to cleanup multipart upload:', cleanupError);
}
}
throw error;
}
const completeCommand = new CompleteMultipartUploadCommand({
Bucket: bucketName,
Key: Key,
UploadId,
MultipartUpload: { Parts: completedParts },
});
await this.s3Client.send(completeCommand);
}
}
+43 -5
View File
@@ -4,23 +4,48 @@ import DeleteTaskManager from "~/lib/delete-task-manager";
export const useDeleteTaskManagerStore = defineStore("DeleteTaskManager", {
state: () => ({
taskManager: new DeleteTaskManager(useNuxtApp().$s3Client, {
chunkSize: 16,
maxConcurrentUploads: 6,
chunkSize: 16, // Not used for delete, kept for consistency
maxConcurrentUploads: 6, // Limit concurrent deletes
maxRetries: 3, // Retry failed deletes
retryDelay: 1000, // Wait 1 second before retrying
}),
}),
getters: {
tasks: (state) => state.taskManager.getTasks(),
pendingTasks: (state) => state.taskManager.getTasks().filter(task => task.status === "pending"),
deletingTasks: (state) => state.taskManager.getTasks().filter(task => task.status === "deleting"),
completedTasks: (state) => state.taskManager.getTasks().filter(task => task.status === "completed"),
failedTasks: (state) => state.taskManager.getTasks().filter(task => task.status === "failed"),
},
actions: {
addKeys(keys: string[], bucketName: string, prefix?: string) {
console.log("addKeys", keys, bucketName, prefix);
this.taskManager.addKeys(keys, bucketName, prefix);
this.taskManager.start();
console.log("addKeys", keys.length, "keys to bucket", bucketName, "prefix:", prefix);
// 对于大量文件,分批处理
const batchSize = 50;
if (keys.length > batchSize) {
console.log(`Processing ${keys.length} keys in batches of ${batchSize}`);
for (let i = 0; i < keys.length; i += batchSize) {
const batch = keys.slice(i, i + batchSize);
this.taskManager.addKeys(batch, bucketName, prefix);
}
} else {
this.taskManager.addKeys(keys, bucketName, prefix);
}
},
getTasks() {
return this.taskManager.getTasks();
},
cancelTask(taskId: string) {
this.taskManager.cancelTask(taskId);
},
cancelAllTasks() {
this.taskManager.cancel();
},
removeTask(taskId: string) {
this.taskManager.removeTask(taskId);
},
@@ -28,5 +53,18 @@ export const useDeleteTaskManagerStore = defineStore("DeleteTaskManager", {
clearTasks() {
this.taskManager.clearTasks();
},
// 获取统计信息
getStats() {
const tasks = this.tasks;
return {
total: tasks.length,
pending: tasks.filter(task => task.status === "pending").length,
deleting: tasks.filter(task => task.status === "deleting").length,
completed: tasks.filter(task => task.status === "completed").length,
failed: tasks.filter(task => task.status === "failed").length,
canceled: tasks.filter(task => task.status === "canceled").length,
};
},
},
});
+42 -5
View File
@@ -4,24 +4,48 @@ import UploadTaskManager from "~/lib/upload-task-manager";
export const useUploadTaskManagerStore = defineStore("uploadTaskManager", {
state: () => ({
taskManager: new UploadTaskManager(useNuxtApp().$s3Client, {
chunkSize: 16,
maxConcurrentUploads: 6,
chunkSize: 16, // 16MB chunks for better performance
maxConcurrentUploads: 6, // Limit concurrent uploads
maxRetries: 3, // Retry failed uploads
retryDelay: 1000, // Wait 1 second before retrying
}),
}),
getters: {
tasks: (state) => state.taskManager.getTasks(),
pendingTasks: (state) => state.taskManager.getTasks().filter(task => task.status === "pending"),
uploadingTasks: (state) => state.taskManager.getTasks().filter(task => task.status === "uploading"),
completedTasks: (state) => state.taskManager.getTasks().filter(task => task.status === "completed"),
failedTasks: (state) => state.taskManager.getTasks().filter(task => task.status === "failed"),
},
actions: {
addFiles(files: File[], bucketName: string, prefix?: string) {
console.log("addFiles", files, bucketName, prefix);
console.log("addFiles", files.length, "files to bucket", bucketName, "prefix:", prefix);
this.taskManager.addFiles(files, bucketName, prefix);
this.taskManager.start();
// 对于大量文件,分批处理
const batchSize = 50;
if (files.length > batchSize) {
console.log(`Processing ${files.length} files in batches of ${batchSize}`);
for (let i = 0; i < files.length; i += batchSize) {
const batch = files.slice(i, i + batchSize);
this.taskManager.addFiles(batch, bucketName, prefix);
}
} else {
this.taskManager.addFiles(files, bucketName, prefix);
}
},
getTasks() {
return this.taskManager.getTasks();
},
cancelTask(taskId: string) {
this.taskManager.cancelTask(taskId);
},
cancelAllTasks() {
this.taskManager.cancel();
},
removeTask(taskId: string) {
this.taskManager.removeTask(taskId);
},
@@ -29,5 +53,18 @@ export const useUploadTaskManagerStore = defineStore("uploadTaskManager", {
clearTasks() {
this.taskManager.clearTasks();
},
// 获取统计信息
getStats() {
const tasks = this.tasks;
return {
total: tasks.length,
pending: tasks.filter(task => task.status === "pending").length,
uploading: tasks.filter(task => task.status === "uploading").length,
completed: tasks.filter(task => task.status === "completed").length,
failed: tasks.filter(task => task.status === "failed").length,
canceled: tasks.filter(task => task.status === "canceled").length,
};
},
},
});