mirror of
https://github.com/rustfs/console.git
synced 2026-10-30 19:33:40 +08:00
fix: rustfs/rustfs#80
This commit is contained in:
Vendored
+1
-1
@@ -40,7 +40,6 @@ declare module 'vue' {
|
||||
NGrid: typeof import('naive-ui')['NGrid']
|
||||
NH3: typeof import('naive-ui')['NH3']
|
||||
NH4: typeof import('naive-ui')['NH4']
|
||||
NIcon: typeof import('naive-ui')['NIcon']
|
||||
NInput: typeof import('naive-ui')['NInput']
|
||||
NInputGroup: typeof import('naive-ui')['NInputGroup']
|
||||
NInputGroupLabel: typeof import('naive-ui')['NInputGroupLabel']
|
||||
@@ -72,6 +71,7 @@ declare module 'vue' {
|
||||
NText: typeof import('naive-ui')['NText']
|
||||
NThing: typeof import('naive-ui')['NThing']
|
||||
NTooltip: typeof import('naive-ui')['NTooltip']
|
||||
NVirtualList: typeof import('naive-ui')['NVirtualList']
|
||||
RouterLink: typeof import('vue-router')['RouterLink']
|
||||
RouterView: typeof import('vue-router')['RouterView']
|
||||
}
|
||||
|
||||
@@ -388,9 +388,9 @@ function handleBatchDelete() {
|
||||
return;
|
||||
}
|
||||
|
||||
// 执行删除
|
||||
await Promise.all(deletableKeys.map((item) => deleteTaskStore.addKeys([String(item)], bucketName.value)));
|
||||
message.success("删除成功");
|
||||
// 执行删除 - 一次性添加所有文件
|
||||
deleteTaskStore.addKeys(deletableKeys.map(String), bucketName.value);
|
||||
message.success("删除任务已创建");
|
||||
salt.value = randomString();
|
||||
refresh();
|
||||
} catch (error) {
|
||||
|
||||
@@ -31,17 +31,10 @@
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- 展示文件与文件夹 -->
|
||||
<n-list v-if="selectedItems.length" bordered class="h-[40vh] overflow-y-auto sticky-header relative">
|
||||
<template #header>
|
||||
<div class="flex items-center gap-2 font-bold">
|
||||
<div class="flex-1">{{ t('Selected Items') }}</div>
|
||||
<div class="w-32 text-center">{{ t('Size') }}</div>
|
||||
<div class="w-32 text-center">{{ t('Operation') }}</div>
|
||||
</div>
|
||||
</template>
|
||||
<n-list-item v-for="(item, index) in selectedItems" :key="index">
|
||||
<div class="flex items-center gap-2 w-full">
|
||||
<!-- 虚拟滚动文件与文件夹列表 -->
|
||||
<n-virtual-list v-if="selectedItems.length" :items="selectedItems" :item-size="40" class="h-[40vh] overflow-y-auto border rounded">
|
||||
<template #default="{ item, index }">
|
||||
<div class="flex items-center gap-2 w-full border-b py-1">
|
||||
<div class="flex-1">
|
||||
<span v-if="item.type === 'folder'">{{ item.name }}/</span>
|
||||
<span v-else>{{ item.name }}</span>
|
||||
@@ -51,13 +44,19 @@
|
||||
<n-button text type="info" @click="removeItem(index)">{{ t('Delete') }}</n-button>
|
||||
</div>
|
||||
</div>
|
||||
</n-list-item>
|
||||
</n-list>
|
||||
</template>
|
||||
</n-virtual-list>
|
||||
</div>
|
||||
|
||||
<n-progress v-if="isAdding" :percentage="addProgress" type="line" :show-indicator="true" style="margin-bottom: 16px;">
|
||||
<template #default>
|
||||
正在添加到上传队列 ({{ addProgress }}%)
|
||||
</template>
|
||||
</n-progress>
|
||||
|
||||
<div class="flex justify-center gap-4">
|
||||
<n-button type="default" :disabled="!hasFiles">{{ t('Configure') }}</n-button>
|
||||
<n-button type="primary" :disabled="!hasFiles" @click="handleUpload">{{ t('Start Upload') }}</n-button>
|
||||
<n-button type="default" :disabled="!hasFiles || isAdding">{{ t('Configure') }}</n-button>
|
||||
<n-button type="primary" :disabled="!hasFiles || isAdding" :loading="isAdding" @click="handleUpload">{{ t('Start Upload') }}</n-button>
|
||||
</div>
|
||||
</div>
|
||||
</n-card>
|
||||
@@ -94,6 +93,9 @@ const folderInput = ref<HTMLInputElement | null>(null)
|
||||
// 待上传的文件/文件夹列表
|
||||
const selectedItems = ref<SelectedItem[]>([])
|
||||
|
||||
const isAdding = ref(false)
|
||||
const addProgress = ref(0)
|
||||
|
||||
const closeModal = () => emit('update:show', false)
|
||||
const selectFile = () => fileInput.value?.click()
|
||||
const selectFolder = () => folderInput.value?.click()
|
||||
@@ -156,13 +158,42 @@ function removeItem(index: number) {
|
||||
selectedItems.value.splice(index, 1)
|
||||
}
|
||||
|
||||
function handleUpload() {
|
||||
console.log('uploading', allFiles.value);
|
||||
allFiles.value.forEach(fileItem => {
|
||||
uploadTaskManagerStore.addFiles([fileItem.file], props.bucketName, fileItem.prefix)
|
||||
})
|
||||
selectedItems.value = []
|
||||
emit('submit')
|
||||
closeModal()
|
||||
async function handleUpload() {
|
||||
const filesToUpload = allFiles.value;
|
||||
if (!filesToUpload.length) return;
|
||||
|
||||
isAdding.value = true;
|
||||
addProgress.value = 0;
|
||||
|
||||
// 分组
|
||||
const uploadGroups = new Map<string, File[]>();
|
||||
filesToUpload.forEach(fileItem => {
|
||||
const key = `${props.bucketName}:${fileItem.prefix}`;
|
||||
if (!uploadGroups.has(key)) uploadGroups.set(key, []);
|
||||
uploadGroups.get(key)!.push(fileItem.file);
|
||||
});
|
||||
|
||||
// 统计总数
|
||||
const total = Array.from(uploadGroups.values()).reduce((sum, arr) => sum + arr.length, 0);
|
||||
let done = 0;
|
||||
|
||||
// 分批异步添加,避免阻塞主线程
|
||||
for (const [key, files] of uploadGroups.entries()) {
|
||||
const [bucketName, prefix] = key.split(':');
|
||||
for (let i = 0; i < files.length; i += 50) {
|
||||
const batch = files.slice(i, i + 50);
|
||||
// 让出主线程,保证 UI 可响应
|
||||
await new Promise(resolve => setTimeout(resolve, 0));
|
||||
await uploadTaskManagerStore.addFiles(batch, bucketName, prefix);
|
||||
done += batch.length;
|
||||
addProgress.value = Math.round((done / total) * 100);
|
||||
}
|
||||
}
|
||||
|
||||
isAdding.value = false;
|
||||
addProgress.value = 100;
|
||||
selectedItems.value = [];
|
||||
emit('submit');
|
||||
closeModal();
|
||||
}
|
||||
</script>
|
||||
|
||||
+1
-1
@@ -86,7 +86,7 @@ export default [
|
||||
},
|
||||
{
|
||||
label: "Documentation",
|
||||
to: "https://rustfs.com/docs/",
|
||||
to: "https://docs.rustfs.com/",
|
||||
target: "_blank",
|
||||
icon: "ri:file-list-3-line",
|
||||
},
|
||||
|
||||
+216
-41
@@ -9,11 +9,14 @@ export interface DeleteTask {
|
||||
progress: number;
|
||||
error?: string;
|
||||
abortController?: AbortController;
|
||||
retryCount?: number;
|
||||
}
|
||||
|
||||
export interface TaskManagerConfig {
|
||||
chunkSize?: number; // in MB
|
||||
maxConcurrentUploads?: number;
|
||||
maxRetries?: number;
|
||||
retryDelay?: number;
|
||||
}
|
||||
|
||||
class DeleteTaskManager {
|
||||
@@ -21,48 +24,98 @@ class DeleteTaskManager {
|
||||
private maxConcurrentUploads: number;
|
||||
private chunkSize: number;
|
||||
private s3Client: S3Client;
|
||||
private activeUploads: number = 0;
|
||||
private activeDeletes: number = 0;
|
||||
private maxRetries: number;
|
||||
private retryDelay: number;
|
||||
private isStarted: boolean = false;
|
||||
private isProcessing: boolean = false;
|
||||
|
||||
constructor(s3Client: S3Client, config: TaskManagerConfig) {
|
||||
constructor(s3Client: S3Client, config: TaskManagerConfig = {}) {
|
||||
this.s3Client = s3Client;
|
||||
this.chunkSize = (config.chunkSize ?? 5) * 1024 * 1024; // Default 5MB
|
||||
this.maxConcurrentUploads = config.maxConcurrentUploads ?? 6;
|
||||
this.maxRetries = config.maxRetries ?? 3;
|
||||
this.retryDelay = config.retryDelay ?? 1000;
|
||||
}
|
||||
|
||||
addKeys(keys: string[], bucketName: string, prefix?: string) {
|
||||
keys.forEach((key) => {
|
||||
const task: DeleteTask = {
|
||||
id: `${Date.now()}-${key}`,
|
||||
key,
|
||||
status: "pending",
|
||||
progress: 0,
|
||||
bucketName,
|
||||
prefix: prefix ?? "",
|
||||
error: undefined,
|
||||
abortController: undefined,
|
||||
};
|
||||
this.tasks.push(task);
|
||||
const newTasks = keys.map((key) => ({
|
||||
id: `${Date.now()}-${Math.random().toString(36).substr(2, 9)}-${key}`,
|
||||
key,
|
||||
status: "pending" as const,
|
||||
progress: 0,
|
||||
bucketName,
|
||||
prefix: prefix ?? "",
|
||||
error: undefined,
|
||||
abortController: undefined,
|
||||
retryCount: 0,
|
||||
}));
|
||||
|
||||
// 批量添加任务
|
||||
this.tasks.push(...newTasks);
|
||||
console.log("🗑️ Added keys to delete queue:", keys.length, "keys");
|
||||
console.log("🔍 Current state:", {
|
||||
isStarted: this.isStarted,
|
||||
isProcessing: this.isProcessing,
|
||||
activeDeletes: this.activeDeletes,
|
||||
totalTasks: this.tasks.length,
|
||||
pendingTasks: this.tasks.filter(t => t.status === "pending").length
|
||||
});
|
||||
|
||||
console.debug("Added keys to delete queue", keys);
|
||||
}
|
||||
|
||||
start() {
|
||||
// 强制重置状态并立即处理队列
|
||||
this.isStarted = true;
|
||||
this.isProcessing = false;
|
||||
this.processQueue();
|
||||
}
|
||||
|
||||
start() {
|
||||
console.log("🚀 Starting delete task manager, current state:", {
|
||||
isStarted: this.isStarted,
|
||||
isProcessing: this.isProcessing
|
||||
});
|
||||
|
||||
if (!this.isStarted) {
|
||||
this.isStarted = true;
|
||||
this.isProcessing = false; // 确保重置处理状态
|
||||
}
|
||||
|
||||
// 立即处理队列
|
||||
if (!this.isProcessing) {
|
||||
this.processQueue();
|
||||
}
|
||||
}
|
||||
|
||||
stop() {
|
||||
this.isStarted = false;
|
||||
this.isProcessing = false;
|
||||
console.log("⏹️ Stopping delete task manager");
|
||||
}
|
||||
|
||||
cancel() {
|
||||
console.log("❌ Canceling all delete tasks");
|
||||
|
||||
// 取消所有待处理的任务
|
||||
this.tasks
|
||||
.filter((task) => task.status === "pending")
|
||||
.forEach((task) => {
|
||||
task.status = "canceled";
|
||||
});
|
||||
|
||||
// 取消所有正在进行的删除
|
||||
this.tasks
|
||||
.filter((task) => task.status === "deleting" && task.abortController)
|
||||
.forEach((task) => {
|
||||
task.abortController?.abort();
|
||||
task.status = "canceled";
|
||||
});
|
||||
}
|
||||
|
||||
cancelTask(taskId: string) {
|
||||
const task = this.tasks.find((task) => task.id === taskId);
|
||||
if (task && task.abortController) {
|
||||
task.abortController.abort();
|
||||
if (task) {
|
||||
console.log("❌ Canceling task:", taskId);
|
||||
if (task.abortController) {
|
||||
task.abortController.abort();
|
||||
}
|
||||
task.status = "canceled";
|
||||
}
|
||||
}
|
||||
@@ -70,7 +123,13 @@ class DeleteTaskManager {
|
||||
removeTask(taskId: string) {
|
||||
const taskIndex = this.tasks.findIndex((task) => task.id === taskId);
|
||||
if (taskIndex !== -1) {
|
||||
// 如果任务正在进行,先取消它
|
||||
const task = this.tasks[taskIndex];
|
||||
if (task.status === "deleting" && task.abortController) {
|
||||
task.abortController.abort();
|
||||
}
|
||||
this.tasks.splice(taskIndex, 1);
|
||||
console.log("🗑️ Removed task:", taskId);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -79,40 +138,156 @@ class DeleteTaskManager {
|
||||
}
|
||||
|
||||
clearTasks() {
|
||||
this.tasks = [];
|
||||
console.log("🧹 Clearing all delete tasks");
|
||||
// 先取消所有正在进行的任务
|
||||
this.cancel();
|
||||
// 清空任务列表
|
||||
this.tasks.splice(0, this.tasks.length);
|
||||
// 重置所有状态
|
||||
this.activeDeletes = 0;
|
||||
this.isStarted = false;
|
||||
this.isProcessing = false;
|
||||
}
|
||||
|
||||
// 重写队列处理逻辑,使用更简单的方式
|
||||
private async processQueue() {
|
||||
while (this.activeUploads < this.maxConcurrentUploads) {
|
||||
const nextTask = this.tasks.find((task) => task.status === "pending");
|
||||
if (!nextTask) {
|
||||
break;
|
||||
if (!this.isStarted) {
|
||||
console.log("⏸️ Task manager not started, skipping queue processing");
|
||||
return;
|
||||
}
|
||||
|
||||
if (this.isProcessing) {
|
||||
console.log("⏸️ Already processing queue, skipping");
|
||||
return;
|
||||
}
|
||||
|
||||
this.isProcessing = true;
|
||||
console.log("🔄 Starting queue processing");
|
||||
|
||||
try {
|
||||
const pendingTasks = this.tasks.filter(task => task.status === "pending");
|
||||
console.log(`🔄 Processing queue: ${pendingTasks.length} pending, ${this.activeDeletes} active`);
|
||||
|
||||
// 启动新的删除任务
|
||||
while (this.activeDeletes < this.maxConcurrentUploads && pendingTasks.length > 0) {
|
||||
const nextTask = pendingTasks.shift();
|
||||
if (!nextTask) break;
|
||||
|
||||
this.activeDeletes++;
|
||||
console.log(`▶️ Starting delete task: ${nextTask.id} (${nextTask.key})`);
|
||||
|
||||
// 立即启动删除任务,不等待
|
||||
this.executeDelete(nextTask);
|
||||
}
|
||||
|
||||
this.activeUploads++;
|
||||
nextTask.status = "deleting";
|
||||
|
||||
try {
|
||||
if (nextTask.key) {
|
||||
await this.deleteObject(nextTask);
|
||||
nextTask.status = "completed";
|
||||
} else {
|
||||
throw new Error("Task key is undefined");
|
||||
// 如果还有待处理的任务,延迟继续处理
|
||||
const remainingPending = this.tasks.filter(task => task.status === "pending").length;
|
||||
if (remainingPending > 0) {
|
||||
console.log(`⏳ ${remainingPending} tasks remaining, will retry in 1 second`);
|
||||
setTimeout(() => {
|
||||
this.isProcessing = false;
|
||||
this.processQueue();
|
||||
}, 1000);
|
||||
} else {
|
||||
console.log("✅ No more pending tasks, queue processing complete");
|
||||
this.isProcessing = false;
|
||||
// 所有任务完成时自动重置 isStarted
|
||||
if (this.activeDeletes === 0) {
|
||||
this.isStarted = false;
|
||||
}
|
||||
} catch (error: any) {
|
||||
nextTask.status = "failed";
|
||||
nextTask.error = error.message;
|
||||
} finally {
|
||||
this.activeUploads--;
|
||||
this.processQueue(); // Continue with the next task
|
||||
}
|
||||
} catch (error) {
|
||||
console.error("❌ Error in processQueue:", error);
|
||||
this.isProcessing = false;
|
||||
}
|
||||
}
|
||||
|
||||
private async executeDelete(task: DeleteTask) {
|
||||
task.status = "deleting";
|
||||
task.progress = 0;
|
||||
|
||||
try {
|
||||
console.log(`🗑️ Deleting: ${task.key}`);
|
||||
await this.deleteObject(task);
|
||||
task.status = "completed";
|
||||
task.progress = 100;
|
||||
console.log(`✅ Deleted successfully: ${task.key}`);
|
||||
} catch (error: any) {
|
||||
console.error(`❌ Delete failed for ${task.key}:`, error.message);
|
||||
|
||||
// 检查是否是取消操作
|
||||
if (error.name === 'AbortError' || error.message?.includes('canceled')) {
|
||||
task.status = "canceled";
|
||||
console.log(`❌ Delete canceled: ${task.key}`);
|
||||
return;
|
||||
}
|
||||
|
||||
// 检查是否应该重试
|
||||
if (this.shouldRetry(task, error)) {
|
||||
console.log(`🔄 Will retry delete: ${task.key} (attempt ${(task.retryCount || 0) + 1})`);
|
||||
await this.retryTask(task);
|
||||
} else {
|
||||
task.status = "failed";
|
||||
task.error = error.message;
|
||||
console.log(`💥 Delete permanently failed: ${task.key}`);
|
||||
}
|
||||
} finally {
|
||||
this.activeDeletes--;
|
||||
console.log(`📊 Delete completed, ${this.activeDeletes} active deletes remaining`);
|
||||
|
||||
// 继续处理队列 - 这是关键的修复
|
||||
if (this.isStarted) {
|
||||
setTimeout(() => {
|
||||
if (!this.isProcessing) {
|
||||
this.processQueue();
|
||||
}
|
||||
}, 100);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private shouldRetry(task: DeleteTask, error: any): boolean {
|
||||
// 如果任务被取消,不重试
|
||||
if (task.status === "canceled") {
|
||||
return false;
|
||||
}
|
||||
|
||||
// 如果达到最大重试次数,不重试
|
||||
if ((task.retryCount || 0) >= this.maxRetries) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// 检查错误类型,某些错误不应该重试
|
||||
const errorMessage = error.message?.toLowerCase() || '';
|
||||
const nonRetryableErrors = [
|
||||
'access denied',
|
||||
'forbidden',
|
||||
'invalid credentials',
|
||||
'bucket not found',
|
||||
'no such key'
|
||||
];
|
||||
|
||||
return !nonRetryableErrors.some(msg => errorMessage.includes(msg));
|
||||
}
|
||||
|
||||
private async retryTask(task: DeleteTask) {
|
||||
task.retryCount = (task.retryCount || 0) + 1;
|
||||
task.status = "pending";
|
||||
task.progress = 0;
|
||||
task.error = undefined;
|
||||
|
||||
// 等待一段时间后重试
|
||||
const retryCount = task.retryCount || 1;
|
||||
await new Promise(resolve => setTimeout(resolve, this.retryDelay * retryCount));
|
||||
|
||||
console.log(`🔄 Retrying delete task ${task.id}, attempt ${retryCount}`);
|
||||
}
|
||||
|
||||
private async deleteObject(task: DeleteTask) {
|
||||
const { key, bucketName, prefix } = task;
|
||||
const abortController = new AbortController();
|
||||
task.abortController = abortController;
|
||||
|
||||
const command = new DeleteObjectCommand({
|
||||
Bucket: bucketName,
|
||||
Key: prefix + key,
|
||||
|
||||
+227
-66
@@ -9,11 +9,14 @@ export interface UploadTask {
|
||||
progress: number;
|
||||
error?: string;
|
||||
abortController?: AbortController;
|
||||
retryCount?: number;
|
||||
}
|
||||
|
||||
export interface TaskManagerConfig {
|
||||
chunkSize?: number; // in MB
|
||||
maxConcurrentUploads?: number;
|
||||
maxRetries?: number;
|
||||
retryDelay?: number;
|
||||
}
|
||||
|
||||
class UploadTaskManager {
|
||||
@@ -22,45 +25,71 @@ class UploadTaskManager {
|
||||
private chunkSize: number;
|
||||
private s3Client: S3Client;
|
||||
private activeUploads: number = 0;
|
||||
private maxRetries: number;
|
||||
private retryDelay: number;
|
||||
private isStarted: boolean = false;
|
||||
|
||||
constructor(s3Client: S3Client, config: TaskManagerConfig) {
|
||||
constructor(s3Client: S3Client, config: TaskManagerConfig = {}) {
|
||||
this.s3Client = s3Client;
|
||||
this.chunkSize = (config.chunkSize ?? 5) * 1024 * 1024; // Default 5MB
|
||||
this.maxConcurrentUploads = config.maxConcurrentUploads ?? 6;
|
||||
this.maxRetries = config.maxRetries ?? 3;
|
||||
this.retryDelay = config.retryDelay ?? 1000;
|
||||
}
|
||||
|
||||
addFiles(files: File[], bucketName: string, prefix?: string) {
|
||||
files.forEach((file) => {
|
||||
const task: UploadTask = {
|
||||
id: `${Date.now()}-${file.name}`,
|
||||
file,
|
||||
status: "pending",
|
||||
progress: 0,
|
||||
bucketName,
|
||||
prefix: prefix ?? "",
|
||||
};
|
||||
this.tasks.push(task);
|
||||
});
|
||||
const newTasks = files.map((file) => ({
|
||||
id: `${Date.now()}-${Math.random().toString(36).substr(2, 9)}-${file.name}`,
|
||||
file,
|
||||
status: "pending" as const,
|
||||
progress: 0,
|
||||
bucketName,
|
||||
prefix: prefix ?? "",
|
||||
retryCount: 0,
|
||||
}));
|
||||
|
||||
console.debug("Added files to upload queue", files);
|
||||
// 批量添加任务
|
||||
this.tasks.push(...newTasks);
|
||||
console.debug("Added files to upload queue", files.length, "files");
|
||||
|
||||
// 启动处理(如果尚未启动)
|
||||
this.start();
|
||||
}
|
||||
|
||||
start() {
|
||||
this.processQueue();
|
||||
if (!this.isStarted) {
|
||||
this.isStarted = true;
|
||||
this.processQueue();
|
||||
}
|
||||
}
|
||||
|
||||
stop() {
|
||||
this.isStarted = false;
|
||||
}
|
||||
|
||||
cancel() {
|
||||
// 取消所有待处理的任务
|
||||
this.tasks
|
||||
.filter((task) => task.status === "pending")
|
||||
.forEach((task) => {
|
||||
task.status = "canceled";
|
||||
});
|
||||
|
||||
// 取消所有正在进行的上传
|
||||
this.tasks
|
||||
.filter((task) => task.status === "uploading" && task.abortController)
|
||||
.forEach((task) => {
|
||||
task.abortController?.abort();
|
||||
task.status = "canceled";
|
||||
});
|
||||
}
|
||||
|
||||
cancelTask(taskId: string) {
|
||||
const task = this.tasks.find((task) => task.id === taskId);
|
||||
if (task && task.abortController) {
|
||||
task.abortController.abort();
|
||||
if (task) {
|
||||
if (task.abortController) {
|
||||
task.abortController.abort();
|
||||
}
|
||||
task.status = "canceled";
|
||||
}
|
||||
}
|
||||
@@ -68,6 +97,11 @@ class UploadTaskManager {
|
||||
removeTask(taskId: string) {
|
||||
const taskIndex = this.tasks.findIndex((task) => task.id === taskId);
|
||||
if (taskIndex !== -1) {
|
||||
// 如果任务正在进行,先取消它
|
||||
const task = this.tasks[taskIndex];
|
||||
if (task.status === "uploading" && task.abortController) {
|
||||
task.abortController.abort();
|
||||
}
|
||||
this.tasks.splice(taskIndex, 1);
|
||||
}
|
||||
}
|
||||
@@ -77,46 +111,126 @@ class UploadTaskManager {
|
||||
}
|
||||
|
||||
clearTasks() {
|
||||
this.tasks = [];
|
||||
// 先取消所有正在进行的任务
|
||||
this.cancel();
|
||||
// 清空任务列表
|
||||
this.tasks.splice(0, this.tasks.length);
|
||||
// 重置状态
|
||||
this.activeUploads = 0;
|
||||
this.isStarted = false;
|
||||
}
|
||||
|
||||
// 简化的队列处理逻辑
|
||||
private async processQueue() {
|
||||
if (!this.isStarted) {
|
||||
return;
|
||||
}
|
||||
|
||||
// 启动新的上传任务
|
||||
while (this.activeUploads < this.maxConcurrentUploads) {
|
||||
const nextTask = this.tasks.find((task) => task.status === "pending");
|
||||
if (!nextTask) { break; }
|
||||
if (!nextTask) {
|
||||
break;
|
||||
}
|
||||
|
||||
this.activeUploads++;
|
||||
nextTask.status = "uploading";
|
||||
// 不等待,直接启动异步上传
|
||||
this.startUpload(nextTask).catch(console.error);
|
||||
}
|
||||
|
||||
try {
|
||||
if (nextTask.file.size > this.chunkSize) {
|
||||
await this.multipartUpload(nextTask);
|
||||
} else {
|
||||
await this.putObject(nextTask);
|
||||
}
|
||||
nextTask.status = "completed";
|
||||
} catch (error: any) {
|
||||
nextTask.status = "failed";
|
||||
nextTask.error = error.message;
|
||||
} finally {
|
||||
this.activeUploads--;
|
||||
this.processQueue(); // Continue with the next task
|
||||
// 如果还有待处理的任务,继续处理
|
||||
if (this.tasks.some(task => task.status === "pending")) {
|
||||
// 使用 setTimeout 避免阻塞主线程
|
||||
setTimeout(() => {
|
||||
this.processQueue();
|
||||
}, 100);
|
||||
}
|
||||
}
|
||||
|
||||
private async startUpload(task: UploadTask) {
|
||||
task.status = "uploading";
|
||||
task.progress = 0;
|
||||
|
||||
try {
|
||||
if (task.file.size > this.chunkSize) {
|
||||
await this.multipartUpload(task);
|
||||
} else {
|
||||
await this.putObject(task);
|
||||
}
|
||||
task.status = "completed";
|
||||
task.progress = 100;
|
||||
} catch (error: any) {
|
||||
// 检查是否是取消操作
|
||||
if (error.name === 'AbortError' || error.message?.includes('canceled')) {
|
||||
task.status = "canceled";
|
||||
return;
|
||||
}
|
||||
|
||||
console.error("Upload failed:", error.message, "Task:", task.id);
|
||||
|
||||
// 检查是否应该重试
|
||||
if (this.shouldRetry(task, error)) {
|
||||
await this.retryTask(task);
|
||||
} else {
|
||||
task.status = "failed";
|
||||
task.error = error.message;
|
||||
}
|
||||
} finally {
|
||||
this.activeUploads--;
|
||||
// 继续处理队列
|
||||
if (this.isStarted) {
|
||||
setTimeout(() => this.processQueue(), 50);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private shouldRetry(task: UploadTask, error: any): boolean {
|
||||
// 如果任务被取消,不重试
|
||||
if (task.status === "canceled") {
|
||||
return false;
|
||||
}
|
||||
|
||||
// 如果达到最大重试次数,不重试
|
||||
if ((task.retryCount || 0) >= this.maxRetries) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// 检查错误类型,某些错误不应该重试
|
||||
const errorMessage = error.message?.toLowerCase() || '';
|
||||
const nonRetryableErrors = [
|
||||
'access denied',
|
||||
'forbidden',
|
||||
'invalid credentials',
|
||||
'bucket not found',
|
||||
'file not found'
|
||||
];
|
||||
|
||||
return !nonRetryableErrors.some(msg => errorMessage.includes(msg));
|
||||
}
|
||||
|
||||
private async retryTask(task: UploadTask) {
|
||||
task.retryCount = (task.retryCount || 0) + 1;
|
||||
task.status = "pending";
|
||||
task.progress = 0;
|
||||
task.error = undefined;
|
||||
|
||||
// 等待一段时间后重试
|
||||
const retryCount = task.retryCount || 1;
|
||||
await new Promise(resolve => setTimeout(resolve, this.retryDelay * retryCount));
|
||||
|
||||
console.debug(`Retrying task ${task.id}, attempt ${retryCount}`);
|
||||
}
|
||||
|
||||
private async putObject(task: UploadTask) {
|
||||
const { file, bucketName, prefix } = task;
|
||||
const abortController = new AbortController();
|
||||
task.abortController = abortController;
|
||||
console.log('putObject', task);
|
||||
|
||||
const fileBuffer = await file.arrayBuffer();
|
||||
const command = new PutObjectCommand({
|
||||
Bucket: bucketName,
|
||||
Key: prefix + task.file.name,
|
||||
Key: prefix + file.name,
|
||||
Body: file,
|
||||
ContentType: file.type,
|
||||
ContentType: file.type || 'application/octet-stream',
|
||||
});
|
||||
|
||||
await this.s3Client.send(command, { abortSignal: abortController.signal });
|
||||
@@ -128,43 +242,90 @@ class UploadTaskManager {
|
||||
const abortController = new AbortController();
|
||||
task.abortController = abortController;
|
||||
|
||||
const Key = prefix + task.file.name;
|
||||
const createCommand = new CreateMultipartUploadCommand({
|
||||
Bucket: bucketName,
|
||||
Key: Key
|
||||
});
|
||||
const { UploadId } = await this.s3Client.send(createCommand);
|
||||
const Key = prefix + file.name;
|
||||
let uploadId: string | undefined;
|
||||
|
||||
const totalChunks = Math.ceil(file.size / this.chunkSize);
|
||||
let completedParts = [];
|
||||
|
||||
for (let partNumber = 1; partNumber <= totalChunks; partNumber++) {
|
||||
const start = (partNumber - 1) * this.chunkSize;
|
||||
const end = Math.min(start + this.chunkSize, file.size);
|
||||
|
||||
const chunk = file.slice(start, end);
|
||||
const uploadPartCommand = new UploadPartCommand({
|
||||
try {
|
||||
// 创建分片上传
|
||||
const createCommand = new CreateMultipartUploadCommand({
|
||||
Bucket: bucketName,
|
||||
Key: Key,
|
||||
UploadId,
|
||||
PartNumber: partNumber,
|
||||
Body: chunk,
|
||||
ContentType: file.type || 'application/octet-stream',
|
||||
});
|
||||
|
||||
const { ETag } = await this.s3Client.send(uploadPartCommand, { abortSignal: abortController.signal });
|
||||
completedParts.push({ ETag, PartNumber: partNumber });
|
||||
const createResponse = await this.s3Client.send(createCommand, {
|
||||
abortSignal: abortController.signal
|
||||
});
|
||||
uploadId = createResponse.UploadId;
|
||||
|
||||
task.progress = Math.round((partNumber / totalChunks) * 100);
|
||||
if (!uploadId) {
|
||||
throw new Error('Failed to create multipart upload');
|
||||
}
|
||||
|
||||
const totalChunks = Math.ceil(file.size / this.chunkSize);
|
||||
const completedParts = [];
|
||||
|
||||
// 上传分片
|
||||
for (let partNumber = 1; partNumber <= totalChunks; partNumber++) {
|
||||
// 检查是否被取消
|
||||
if (abortController.signal.aborted) {
|
||||
throw new Error('Upload canceled');
|
||||
}
|
||||
|
||||
const start = (partNumber - 1) * this.chunkSize;
|
||||
const end = Math.min(start + this.chunkSize, file.size);
|
||||
const chunk = file.slice(start, end);
|
||||
|
||||
const uploadPartCommand = new UploadPartCommand({
|
||||
Bucket: bucketName,
|
||||
Key: Key,
|
||||
UploadId: uploadId,
|
||||
PartNumber: partNumber,
|
||||
Body: chunk,
|
||||
});
|
||||
|
||||
const { ETag } = await this.s3Client.send(uploadPartCommand, {
|
||||
abortSignal: abortController.signal
|
||||
});
|
||||
|
||||
if (!ETag) {
|
||||
throw new Error(`Failed to upload part ${partNumber}`);
|
||||
}
|
||||
|
||||
completedParts.push({ ETag, PartNumber: partNumber });
|
||||
task.progress = Math.round((partNumber / totalChunks) * 100);
|
||||
}
|
||||
|
||||
// 完成分片上传
|
||||
const completeCommand = new CompleteMultipartUploadCommand({
|
||||
Bucket: bucketName,
|
||||
Key: Key,
|
||||
UploadId: uploadId,
|
||||
MultipartUpload: { Parts: completedParts },
|
||||
});
|
||||
|
||||
await this.s3Client.send(completeCommand, {
|
||||
abortSignal: abortController.signal
|
||||
});
|
||||
|
||||
} catch (error: any) {
|
||||
// 如果上传失败,尝试清理分片上传
|
||||
if (uploadId) {
|
||||
try {
|
||||
const { AbortMultipartUploadCommand } = await import("@aws-sdk/client-s3");
|
||||
const abortCommand = new AbortMultipartUploadCommand({
|
||||
Bucket: bucketName,
|
||||
Key: Key,
|
||||
UploadId: uploadId,
|
||||
});
|
||||
await this.s3Client.send(abortCommand);
|
||||
} catch (cleanupError) {
|
||||
console.warn('Failed to cleanup multipart upload:', cleanupError);
|
||||
}
|
||||
}
|
||||
|
||||
throw error;
|
||||
}
|
||||
|
||||
const completeCommand = new CompleteMultipartUploadCommand({
|
||||
Bucket: bucketName,
|
||||
Key: Key,
|
||||
UploadId,
|
||||
MultipartUpload: { Parts: completedParts },
|
||||
});
|
||||
|
||||
await this.s3Client.send(completeCommand);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+43
-5
@@ -4,23 +4,48 @@ import DeleteTaskManager from "~/lib/delete-task-manager";
|
||||
export const useDeleteTaskManagerStore = defineStore("DeleteTaskManager", {
|
||||
state: () => ({
|
||||
taskManager: new DeleteTaskManager(useNuxtApp().$s3Client, {
|
||||
chunkSize: 16,
|
||||
maxConcurrentUploads: 6,
|
||||
chunkSize: 16, // Not used for delete, kept for consistency
|
||||
maxConcurrentUploads: 6, // Limit concurrent deletes
|
||||
maxRetries: 3, // Retry failed deletes
|
||||
retryDelay: 1000, // Wait 1 second before retrying
|
||||
}),
|
||||
}),
|
||||
getters: {
|
||||
tasks: (state) => state.taskManager.getTasks(),
|
||||
pendingTasks: (state) => state.taskManager.getTasks().filter(task => task.status === "pending"),
|
||||
deletingTasks: (state) => state.taskManager.getTasks().filter(task => task.status === "deleting"),
|
||||
completedTasks: (state) => state.taskManager.getTasks().filter(task => task.status === "completed"),
|
||||
failedTasks: (state) => state.taskManager.getTasks().filter(task => task.status === "failed"),
|
||||
},
|
||||
actions: {
|
||||
addKeys(keys: string[], bucketName: string, prefix?: string) {
|
||||
console.log("addKeys", keys, bucketName, prefix);
|
||||
this.taskManager.addKeys(keys, bucketName, prefix);
|
||||
this.taskManager.start();
|
||||
console.log("addKeys", keys.length, "keys to bucket", bucketName, "prefix:", prefix);
|
||||
|
||||
// 对于大量文件,分批处理
|
||||
const batchSize = 50;
|
||||
if (keys.length > batchSize) {
|
||||
console.log(`Processing ${keys.length} keys in batches of ${batchSize}`);
|
||||
for (let i = 0; i < keys.length; i += batchSize) {
|
||||
const batch = keys.slice(i, i + batchSize);
|
||||
this.taskManager.addKeys(batch, bucketName, prefix);
|
||||
}
|
||||
} else {
|
||||
this.taskManager.addKeys(keys, bucketName, prefix);
|
||||
}
|
||||
},
|
||||
|
||||
getTasks() {
|
||||
return this.taskManager.getTasks();
|
||||
},
|
||||
|
||||
cancelTask(taskId: string) {
|
||||
this.taskManager.cancelTask(taskId);
|
||||
},
|
||||
|
||||
cancelAllTasks() {
|
||||
this.taskManager.cancel();
|
||||
},
|
||||
|
||||
removeTask(taskId: string) {
|
||||
this.taskManager.removeTask(taskId);
|
||||
},
|
||||
@@ -28,5 +53,18 @@ export const useDeleteTaskManagerStore = defineStore("DeleteTaskManager", {
|
||||
clearTasks() {
|
||||
this.taskManager.clearTasks();
|
||||
},
|
||||
|
||||
// 获取统计信息
|
||||
getStats() {
|
||||
const tasks = this.tasks;
|
||||
return {
|
||||
total: tasks.length,
|
||||
pending: tasks.filter(task => task.status === "pending").length,
|
||||
deleting: tasks.filter(task => task.status === "deleting").length,
|
||||
completed: tasks.filter(task => task.status === "completed").length,
|
||||
failed: tasks.filter(task => task.status === "failed").length,
|
||||
canceled: tasks.filter(task => task.status === "canceled").length,
|
||||
};
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
+42
-5
@@ -4,24 +4,48 @@ import UploadTaskManager from "~/lib/upload-task-manager";
|
||||
export const useUploadTaskManagerStore = defineStore("uploadTaskManager", {
|
||||
state: () => ({
|
||||
taskManager: new UploadTaskManager(useNuxtApp().$s3Client, {
|
||||
chunkSize: 16,
|
||||
maxConcurrentUploads: 6,
|
||||
chunkSize: 16, // 16MB chunks for better performance
|
||||
maxConcurrentUploads: 6, // Limit concurrent uploads
|
||||
maxRetries: 3, // Retry failed uploads
|
||||
retryDelay: 1000, // Wait 1 second before retrying
|
||||
}),
|
||||
}),
|
||||
getters: {
|
||||
tasks: (state) => state.taskManager.getTasks(),
|
||||
pendingTasks: (state) => state.taskManager.getTasks().filter(task => task.status === "pending"),
|
||||
uploadingTasks: (state) => state.taskManager.getTasks().filter(task => task.status === "uploading"),
|
||||
completedTasks: (state) => state.taskManager.getTasks().filter(task => task.status === "completed"),
|
||||
failedTasks: (state) => state.taskManager.getTasks().filter(task => task.status === "failed"),
|
||||
},
|
||||
actions: {
|
||||
addFiles(files: File[], bucketName: string, prefix?: string) {
|
||||
console.log("addFiles", files, bucketName, prefix);
|
||||
console.log("addFiles", files.length, "files to bucket", bucketName, "prefix:", prefix);
|
||||
|
||||
this.taskManager.addFiles(files, bucketName, prefix);
|
||||
this.taskManager.start();
|
||||
// 对于大量文件,分批处理
|
||||
const batchSize = 50;
|
||||
if (files.length > batchSize) {
|
||||
console.log(`Processing ${files.length} files in batches of ${batchSize}`);
|
||||
for (let i = 0; i < files.length; i += batchSize) {
|
||||
const batch = files.slice(i, i + batchSize);
|
||||
this.taskManager.addFiles(batch, bucketName, prefix);
|
||||
}
|
||||
} else {
|
||||
this.taskManager.addFiles(files, bucketName, prefix);
|
||||
}
|
||||
},
|
||||
|
||||
getTasks() {
|
||||
return this.taskManager.getTasks();
|
||||
},
|
||||
|
||||
cancelTask(taskId: string) {
|
||||
this.taskManager.cancelTask(taskId);
|
||||
},
|
||||
|
||||
cancelAllTasks() {
|
||||
this.taskManager.cancel();
|
||||
},
|
||||
|
||||
removeTask(taskId: string) {
|
||||
this.taskManager.removeTask(taskId);
|
||||
},
|
||||
@@ -29,5 +53,18 @@ export const useUploadTaskManagerStore = defineStore("uploadTaskManager", {
|
||||
clearTasks() {
|
||||
this.taskManager.clearTasks();
|
||||
},
|
||||
|
||||
// 获取统计信息
|
||||
getStats() {
|
||||
const tasks = this.tasks;
|
||||
return {
|
||||
total: tasks.length,
|
||||
pending: tasks.filter(task => task.status === "pending").length,
|
||||
uploading: tasks.filter(task => task.status === "uploading").length,
|
||||
completed: tasks.filter(task => task.status === "completed").length,
|
||||
failed: tasks.filter(task => task.status === "failed").length,
|
||||
canceled: tasks.filter(task => task.status === "canceled").length,
|
||||
};
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user