opensource/zpan
3
0
Fork 0
mirror of https://github.com/saltbo/zpan.git synced 2026-08-28 15:51:29 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(downloads): preserve directory uploads

Browse Source
This commit is contained in:
saltbo
2026-06-03 12:05:55 -04:00
parent cd3a3a7148
commit cf3324b4a2
23 changed files with 4406 additions and 127 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docs/openapi/downloader.json
+28 -2
View File
@@ -116,12 +116,14 @@
},
"uploadUrl": {
"type": "string"
},
"contentDisposition": {
"type": "string"
}
},
"required": [
"id",
"name",
"uploadUrl"
"name"
]
},
"ConfirmObjectRequest": {
@@ -308,6 +310,10 @@
"type": "integer",
"format": "int64"
},
"uploadedBytes": {
"type": "integer",
"format": "int64"
},
"totalBytes": {
"type": "integer",
"format": "int64",
@@ -497,6 +503,7 @@
"targetFolder",
"status",
"downloadedBytes",
"uploadedBytes",
"totalBytes",
"downloadBps",
"uploadBps"
@@ -635,6 +642,10 @@
"type": "integer",
"format": "int64"
},
"uploadedBytes": {
"type": "integer",
"format": "int64"
},
"totalBytes": {
"type": "integer",
"format": "int64",
@@ -824,6 +835,7 @@
"targetFolder",
"status",
"downloadedBytes",
"uploadedBytes",
"totalBytes",
"downloadBps",
"uploadBps"
@@ -946,6 +958,10 @@
"type": "integer",
"format": "int64"
},
"uploadedBytes": {
"type": "integer",
"format": "int64"
},
"totalBytes": {
"type": "integer",
"format": "int64",
@@ -1135,6 +1151,7 @@
"targetFolder",
"status",
"downloadedBytes",
"uploadedBytes",
"totalBytes",
"downloadBps",
"uploadBps"
@@ -1198,6 +1215,10 @@
"type": "integer",
"minimum": 0
},
"uploadedBytes": {
"type": "integer",
"minimum": 0
},
"totalBytes": {
"type": "integer",
"nullable": true,
@@ -1423,6 +1444,10 @@
"type": "integer",
"format": "int64"
},
"uploadedBytes": {
"type": "integer",
"format": "int64"
},
"totalBytes": {
"type": "integer",
"format": "int64",
@@ -1612,6 +1637,7 @@
"targetFolder",
"status",
"downloadedBytes",
"uploadedBytes",
"totalBytes",
"downloadBps",
"uploadBps"
downloader/internal/client/client.go
+25 -2
View File
@@ -33,6 +33,7 @@ type DownloadTask struct {
TargetFolder string `json:"targetFolder"`
Status string `json:"status"`
DownloadedBytes int64 `json:"downloadedBytes"`
UploadedBytes int64 `json:"uploadedBytes"`
TotalBytes *int64 `json:"totalBytes"`
DownloadBps int64 `json:"downloadBps"`
UploadBps int64 `json:"uploadBps"`
@@ -100,6 +101,7 @@ type Heartbeat struct {
type TaskPatch struct {
Status string `json:"status,omitempty"`
DownloadedBytes *int64 `json:"downloadedBytes,omitempty"`
UploadedBytes *int64 `json:"uploadedBytes,omitempty"`
TotalBytes *int64 `json:"totalBytes,omitempty"`
DownloadBps *int64 `json:"downloadBps,omitempty"`
UploadBps *int64 `json:"uploadBps,omitempty"`
@@ -115,6 +117,11 @@ type ObjectDraft struct {
ContentDisposition string `json:"contentDisposition,omitempty"`
}
const (
dirTypeFile = 0
dirTypeUserFolder = 1
)
type DeviceCode struct {
DeviceCode string `json:"device_code"`
UserCode string `json:"user_code"`
@@ -285,6 +292,22 @@ func (c *Client) UpdateTask(ctx context.Context, id string, patch TaskPatch) (Do
}
func (c *Client) CreateObject(ctx context.Context, token string, name string, size int64, parent string) (ObjectDraft, error) {
return c.createMatter(ctx, token, name, "application/octet-stream", size, parent, dirTypeFile)
}
func (c *Client) CreateFolder(ctx context.Context, token string, name string, parent string) (ObjectDraft, error) {
return c.createMatter(ctx, token, name, "folder", 0, parent, dirTypeUserFolder)
}
func (c *Client) createMatter(
ctx context.Context,
token string,
name string,
contentType string,
size int64,
parent string,
dirtype int,
) (ObjectDraft, error) {
body, err := jsonBody(struct {
Name string `json:"name"`
Type string `json:"type"`
@@ -293,10 +316,10 @@ func (c *Client) CreateObject(ctx context.Context, token string, name string, si
Dirtype int `json:"dirtype"`
}{
Name: name,
Type: "application/octet-stream",
Type: contentType,
Size: size,
Parent: parent,
Dirtype: 0,
Dirtype: dirtype,
})
if err != nil {
return ObjectDraft{}, err
downloader/internal/engine/engine.go
+27 -9
View File
@@ -19,9 +19,10 @@ import (
)
type Result struct {
Path string
Name string
Size int64
Path string
Name string
Size int64
IsDir bool
}
type Progress func(downloaded int64, total *int64, bps int64, detail *client.DownloadTaskDetail) error
@@ -661,16 +662,14 @@ func resultFromPath(task client.DownloadTask, path string, fallbackName string)
if len(visible) == 1 && !visible[0].IsDir() {
return resultFromFile(task, filepath.Join(path, visible[0].Name()))
}
zipName := outputName(task, fallbackName)
if !strings.HasSuffix(strings.ToLower(zipName), ".zip") {
zipName += ".zip"
if len(visible) == 1 && visible[0].IsDir() && strings.TrimSpace(task.Name) == "" {
return resultFromPath(task, filepath.Join(path, visible[0].Name()), visible[0].Name())
}
zipPath := filepath.Join(filepath.Dir(path), zipName)
size, err := ZipDirectory(path, zipPath)
size, err := directorySize(path)
if err != nil {
return Result{}, err
}
return Result{Path: zipPath, Name: zipName, Size: size}, nil
return Result{Path: path, Name: outputName(task, fallbackName), Size: size, IsDir: true}, nil
}
func isAria2MetadataPath(path string) bool {
@@ -685,6 +684,25 @@ func resultFromFile(task client.DownloadTask, path string) (Result, error) {
return Result{Path: path, Name: outputName(task, filepath.Base(path)), Size: info.Size()}, nil
}
func directorySize(path string) (int64, error) {
var total int64
err := filepath.WalkDir(path, func(entryPath string, entry os.DirEntry, err error) error {
if err != nil {
return err
}
if entry.IsDir() {
return nil
}
info, err := entry.Info()
if err != nil {
return err
}
total += info.Size()
return nil
})
return total, err
}
func cleanDownloadedPath(baseDir string, path string) string {
if filepath.IsAbs(path) {
return filepath.Clean(path)
downloader/internal/engine/engine_test.go
+12 -6
View File
@@ -158,7 +158,7 @@ func TestAria2DetailIncludesPeerSamples(t *testing.T) {
}
}
func TestResultFromPathZipsDirectory(t *testing.T) {
func TestResultFromPathReturnsDirectory(t *testing.T) {
dir := t.TempDir()
taskDir := filepath.Join(dir, "task-1")
if err := os.MkdirAll(filepath.Join(taskDir, "folder"), 0o755); err != nil {
@@ -175,13 +175,19 @@ func TestResultFromPathZipsDirectory(t *testing.T) {
if err != nil {
t.Fatal(err)
}
if result.Name != "bundle.zip" {
t.Fatalf("expected bundle.zip, got %s", result.Name)
if !result.IsDir {
t.Fatal("expected directory result")
}
if result.Size <= 0 {
t.Fatalf("expected zip size > 0, got %d", result.Size)
if result.Name != "bundle" {
t.Fatalf("expected bundle, got %s", result.Name)
}
if _, err := os.Stat(result.Path); err != nil {
if result.Size != 2 {
t.Fatalf("expected directory size 2, got %d", result.Size)
}
if result.Path != taskDir {
t.Fatalf("expected task dir path, got %s", result.Path)
}
if _, err := os.Stat(filepath.Join(result.Path, "folder", "a.txt")); err != nil {
t.Fatal(err)
}
}
downloader/internal/engine/zip.go
-65
View File
@@ -1,65 +0,0 @@
package engine
import (
"archive/zip"
"io"
"os"
"path/filepath"
"strings"
)
func ZipDirectory(sourceDir string, destination string) (int64, error) {
out, err := os.Create(destination)
if err != nil {
return 0, err
}
defer out.Close()
archive := zip.NewWriter(out)
defer archive.Close()
if err := filepath.WalkDir(sourceDir, func(path string, entry os.DirEntry, err error) error {
if err != nil {
return err
}
if entry.IsDir() {
return nil
}
if filepath.Clean(path) == filepath.Clean(destination) {
return nil
}
if strings.HasPrefix(entry.Name(), ".") {
return nil
}
rel, err := filepath.Rel(sourceDir, path)
if err != nil {
return err
}
file, err := os.Open(path)
if err != nil {
return err
}
defer file.Close()
writer, err := archive.Create(filepath.ToSlash(rel))
if err != nil {
return err
}
_, err = io.Copy(writer, file)
return err
}); err != nil {
return 0, err
}
if err := archive.Close(); err != nil {
return 0, err
}
if err := out.Close(); err != nil {
return 0, err
}
info, err := os.Stat(destination)
if err != nil {
return 0, err
}
return info.Size(), nil
}
downloader/internal/openapi/client.gen.go
+13 -3
View File
@@ -749,9 +749,10 @@ type ErrorResponse struct {
// ObjectDraft defines model for ObjectDraft.
type ObjectDraft struct {
Id string `json:"id"`
Name string `json:"name"`
UploadUrl string `json:"uploadUrl"`
ContentDisposition *string `json:"contentDisposition,omitempty"`
Id string `json:"id"`
Name string `json:"name"`
UploadUrl *string `json:"uploadUrl,omitempty"`
}
// GetApiAdminDownloaders200JSONResponseBodyItemsHeartbeatEngine defines parameters for GetApiAdminDownloaders.
@@ -899,6 +900,7 @@ type PatchApiDownloadTasksIdJSONBody struct {
Status *PatchApiDownloadTasksIdJSONBodyStatus `json:"status,omitempty"`
TotalBytes *int `json:"totalBytes,omitempty"`
UploadBps *int `json:"uploadBps,omitempty"`
UploadedBytes *int `json:"uploadedBytes,omitempty"`
}
// PatchApiDownloadTasksIdJSONBodyDetailEngine defines parameters for PatchApiDownloadTasksId.
@@ -2351,6 +2353,7 @@ type GetApiDownloadTasksResponse struct {
TotalBytes *int64 `json:"totalBytes"`
UploadBps int64 `json:"uploadBps"`
UploadToken *string `json:"uploadToken,omitempty"`
UploadedBytes int64 `json:"uploadedBytes"`
} `json:"items"`
Page int `json:"page"`
PageSize int `json:"pageSize"`
@@ -2437,6 +2440,7 @@ type PostApiDownloadTasksResponse struct {
TotalBytes *int64 `json:"totalBytes"`
UploadBps int64 `json:"uploadBps"`
UploadToken *string `json:"uploadToken,omitempty"`
UploadedBytes int64 `json:"uploadedBytes"`
}
JSON401 *struct {
Error string `json:"error"`
@@ -2525,6 +2529,7 @@ type GetApiDownloadTasksIdResponse struct {
TotalBytes *int64 `json:"totalBytes"`
UploadBps int64 `json:"uploadBps"`
UploadToken *string `json:"uploadToken,omitempty"`
UploadedBytes int64 `json:"uploadedBytes"`
}
JSON404 *struct {
Error string `json:"error"`
@@ -2607,6 +2612,7 @@ type PatchApiDownloadTasksIdResponse struct {
TotalBytes *int64 `json:"totalBytes"`
UploadBps int64 `json:"uploadBps"`
UploadToken *string `json:"uploadToken,omitempty"`
UploadedBytes int64 `json:"uploadedBytes"`
}
JSON401 *struct {
Error string `json:"error"`
@@ -3278,6 +3284,7 @@ func ParseGetApiDownloadTasksResponse(rsp *http.Response) (*GetApiDownloadTasksR
TotalBytes *int64 `json:"totalBytes"`
UploadBps int64 `json:"uploadBps"`
UploadToken *string `json:"uploadToken,omitempty"`
UploadedBytes int64 `json:"uploadedBytes"`
} `json:"items"`
Page int `json:"page"`
PageSize int `json:"pageSize"`
@@ -3366,6 +3373,7 @@ func ParsePostApiDownloadTasksResponse(rsp *http.Response) (*PostApiDownloadTask
TotalBytes *int64 `json:"totalBytes"`
UploadBps int64 `json:"uploadBps"`
UploadToken *string `json:"uploadToken,omitempty"`
UploadedBytes int64 `json:"uploadedBytes"`
}
if err := json.Unmarshal(bodyBytes, &dest); err != nil {
return nil, err
@@ -3468,6 +3476,7 @@ func ParseGetApiDownloadTasksIdResponse(rsp *http.Response) (*GetApiDownloadTask
TotalBytes *int64 `json:"totalBytes"`
UploadBps int64 `json:"uploadBps"`
UploadToken *string `json:"uploadToken,omitempty"`
UploadedBytes int64 `json:"uploadedBytes"`
}
if err := json.Unmarshal(bodyBytes, &dest); err != nil {
return nil, err
@@ -3552,6 +3561,7 @@ func ParsePatchApiDownloadTasksIdResponse(rsp *http.Response) (*PatchApiDownload
TotalBytes *int64 `json:"totalBytes"`
UploadBps int64 `json:"uploadBps"`
UploadToken *string `json:"uploadToken,omitempty"`
UploadedBytes int64 `json:"uploadedBytes"`
}
if err := json.Unmarshal(bodyBytes, &dest); err != nil {
return nil, err
downloader/internal/worker/worker.go
+222 -24
View File
@@ -10,7 +10,10 @@ import (
"net/url"
"os"
"os/exec"
"path"
"path/filepath"
"runtime"
"sort"
"strconv"
"strings"
"sync"
@@ -122,7 +125,11 @@ func (w *Worker) process(ctx context.Context, task client.DownloadTask) {
}
var lastProgressLog time.Time
currentDetail := task.Detail
result, err := w.engine.Download(ctx, task, func(downloaded int64, total *int64, bps int64, detail *client.DownloadTaskDetail) error {
if detail != nil {
currentDetail = detail
}
_, updateErr := w.api.UpdateTask(ctx, task.ID, client.TaskPatch{
DownloadedBytes: &downloaded,
TotalBytes: total,
@@ -160,42 +167,214 @@ func (w *Worker) process(ctx context.Context, task client.DownloadTask) {
if _, err := w.api.UpdateTask(ctx, task.ID, client.TaskPatch{Status: "uploading"}); err != nil {
log.Error("failed to mark task uploading", "error", err)
}
log.Info("creating remote object", "name", result.Name, "size", result.Size, "target_folder", task.TargetFolder)
draft, err := w.api.CreateObject(ctx, task.UploadToken, result.Name, result.Size, task.TargetFolder)
task.Detail = currentDetail
resultObjectID, err := w.uploadResult(ctx, log, task, result)
if err != nil {
msg := taskErrorMessage(err)
log.Error("failed to create remote object", "error", err)
log.Error("failed to upload result", "error", err)
if _, updateErr := w.api.UpdateTask(ctx, task.ID, client.TaskPatch{Status: "failed", ErrorMessage: &msg}); updateErr != nil {
log.Error("failed to mark task failed", "error", updateErr)
}
return
}
log.Info("uploading file to object storage", "object_id", draft.ID, "path", result.Path)
if err := uploadFile(ctx, draft.UploadURL, result.Path, draft.ContentDisposition); err != nil {
msg := taskErrorMessage(err)
log.Error("failed to upload file to object storage", "object_id", draft.ID, "error", err)
if _, updateErr := w.api.UpdateTask(ctx, task.ID, client.TaskPatch{Status: "failed", ErrorMessage: &msg}); updateErr != nil {
log.Error("failed to mark task failed", "error", updateErr)
}
zero := int64(0)
uploadedBytes := result.Size
completedDetail := task.Detail
if completedDetail == nil {
completedDetail = &client.DownloadTaskDetail{}
}
completedDetail.Phase = "completed"
completedDetail.UploadedBytes = &uploadedBytes
if _, err := w.api.UpdateTask(ctx, task.ID, client.TaskPatch{
Status: "completed",
ResultObjectID: &resultObjectID,
UploadedBytes: &uploadedBytes,
UploadBps: &zero,
Detail: completedDetail,
}); err != nil {
log.Error("failed to mark task completed", "object_id", resultObjectID, "error", err)
return
}
log.Debug("task completed", "object_id", resultObjectID)
if err := os.RemoveAll(result.Path); err != nil {
log.Warn("failed to remove local downloaded result", "path", result.Path, "error", err)
}
}
type uploadProgress struct {
totalBytes int64
uploaded int64
lastAt time.Time
lastBytes int64
}
func (w *Worker) uploadResult(
ctx context.Context,
log *slog.Logger,
task client.DownloadTask,
result engine.Result,
) (string, error) {
if !result.IsDir {
progress := &uploadProgress{totalBytes: result.Size, lastAt: time.Now()}
return w.uploadSingleFile(ctx, log, task, result.Path, result.Name, result.Size, task.TargetFolder, progress)
}
progress := &uploadProgress{totalBytes: result.Size, lastAt: time.Now()}
log.Info("creating remote folder", "name", result.Name, "size", result.Size, "target_folder", task.TargetFolder)
root, err := w.api.CreateFolder(ctx, task.UploadToken, result.Name, task.TargetFolder)
if err != nil {
return "", fmt.Errorf("create remote folder: %w", err)
}
rootPath := joinObjectPath(task.TargetFolder, root.Name)
entries, err := collectDirectoryEntries(result.Path)
if err != nil {
return "", err
}
for _, entry := range entries {
parent := joinObjectPath(rootPath, path.Dir(entry.relativePath))
if entry.isDir {
log.Debug("creating remote subfolder", "name", entry.name, "parent", parent)
if _, err := w.api.CreateFolder(ctx, task.UploadToken, entry.name, parent); err != nil {
return "", fmt.Errorf("create remote subfolder %s: %w", entry.relativePath, err)
}
continue
}
if _, err := w.uploadSingleFile(ctx, log, task, entry.path, entry.name, entry.size, parent, progress); err != nil {
return "", err
}
}
return root.ID, nil
}
func (w *Worker) uploadSingleFile(
ctx context.Context,
log *slog.Logger,
task client.DownloadTask,
path string,
name string,
size int64,
parent string,
progress *uploadProgress,
) (string, error) {
log.Info("creating remote object", "name", name, "size", size, "target_folder", parent)
draft, err := w.api.CreateObject(ctx, task.UploadToken, name, size, parent)
if err != nil {
return "", fmt.Errorf("create remote object: %w", err)
}
log.Info("uploading file to object storage", "object_id", draft.ID, "path", path)
if err := uploadFile(ctx, draft.UploadURL, path, draft.ContentDisposition, func(written int64) error {
return w.reportUploadProgress(ctx, log, task, progress, written)
}); err != nil {
return "", fmt.Errorf("upload object %s: %w", draft.ID, err)
}
log.Info("confirming uploaded object", "object_id", draft.ID)
if err := w.api.ConfirmObject(ctx, task.UploadToken, draft.ID); err != nil {
msg := taskErrorMessage(err)
log.Error("failed to confirm uploaded object", "object_id", draft.ID, "error", err)
if _, updateErr := w.api.UpdateTask(ctx, task.ID, client.TaskPatch{Status: "failed", ErrorMessage: &msg}); updateErr != nil {
log.Error("failed to mark task failed", "error", updateErr)
return "", fmt.Errorf("confirm object %s: %w", draft.ID, err)
}
return draft.ID, nil
}
func (w *Worker) reportUploadProgress(
ctx context.Context,
log *slog.Logger,
task client.DownloadTask,
progress *uploadProgress,
written int64,
) error {
progress.uploaded += written
now := time.Now()
if progress.uploaded < progress.totalBytes && now.Sub(progress.lastAt) < time.Second {
return nil
}
elapsed := now.Sub(progress.lastAt).Seconds()
var bps int64
if elapsed > 0 {
bps = int64(float64(progress.uploaded-progress.lastBytes) / elapsed)
}
detail := task.Detail
if detail == nil {
detail = &client.DownloadTaskDetail{}
}
detail.Phase = "uploading"
detail.UploadedBytes = &progress.uploaded
_, err := w.api.UpdateTask(ctx, task.ID, client.TaskPatch{
Status: "uploading",
UploadedBytes: &progress.uploaded,
UploadBps: &bps,
Detail: detail,
})
if err != nil {
log.Error("failed to report upload progress", "uploaded_bytes", progress.uploaded, "total_bytes", progress.totalBytes, "bps", bps, "error", err)
return err
}
log.Debug("task upload progress", "uploaded_bytes", progress.uploaded, "total_bytes", progress.totalBytes, "bps", bps)
progress.lastAt = now
progress.lastBytes = progress.uploaded
return nil
}
type directoryEntry struct {
path string
relativePath string
name string
size int64
isDir bool
}
func collectDirectoryEntries(root string) ([]directoryEntry, error) {
var entries []directoryEntry
err := filepath.WalkDir(root, func(path string, entry os.DirEntry, err error) error {
if err != nil {
return err
}
return
if path == root {
return nil
}
if strings.HasPrefix(entry.Name(), ".") {
if entry.IsDir() {
return filepath.SkipDir
}
return nil
}
relativePath, err := filepath.Rel(root, path)
if err != nil {
return err
}
item := directoryEntry{
path: path,
relativePath: filepath.ToSlash(relativePath),
name: entry.Name(),
isDir: entry.IsDir(),
}
if !entry.IsDir() {
info, err := entry.Info()
if err != nil {
return err
}
item.size = info.Size()
}
entries = append(entries, item)
return nil
})
sort.Slice(entries, func(i, j int) bool {
if entries[i].isDir != entries[j].isDir {
return entries[i].isDir
}
return entries[i].relativePath < entries[j].relativePath
})
return entries, err
}
func joinObjectPath(parent string, name string) string {
name = strings.Trim(filepath.ToSlash(name), "/")
if name == "" || name == "." {
return strings.Trim(parent, "/")
}
if _, err := w.api.UpdateTask(ctx, task.ID, client.TaskPatch{Status: "completed", ResultObjectID: &draft.ID}); err != nil {
log.Error("failed to mark task completed", "object_id", draft.ID, "error", err)
return
}
log.Debug("task completed", "object_id", draft.ID)
if err := os.Remove(result.Path); err != nil {
log.Warn("failed to remove local downloaded file", "path", result.Path, "error", err)
parent = strings.Trim(parent, "/")
if parent == "" {
return name
}
return parent + "/" + name
}
func (w *Worker) canStart(taskID string) bool {
@@ -530,7 +709,7 @@ func capabilities(name string) []string {
}
}
func uploadFile(ctx context.Context, url, path string, contentDisposition string) error {
func uploadFile(ctx context.Context, url, path string, contentDisposition string, progress func(written int64) error) error {
file, err := os.Open(path)
if err != nil {
return err
@@ -540,7 +719,11 @@ func uploadFile(ctx context.Context, url, path string, contentDisposition string
if err != nil {
return err
}
req, err := http.NewRequestWithContext(ctx, http.MethodPut, url, file)
reader := io.Reader(file)
if progress != nil {
reader = &uploadProgressReader{reader: file, progress: progress}
}
req, err := http.NewRequestWithContext(ctx, http.MethodPut, url, reader)
if err != nil {
return err
}
@@ -564,6 +747,21 @@ func uploadFile(ctx context.Context, url, path string, contentDisposition string
return nil
}
type uploadProgressReader struct {
reader io.Reader
progress func(written int64) error
}
func (r *uploadProgressReader) Read(p []byte) (int, error) {
n, err := r.reader.Read(p)
if n > 0 {
if progressErr := r.progress(int64(n)); progressErr != nil {
return n, progressErr
}
}
return n, err
}
func taskErrorMessage(err error) string {
msg := err.Error()
if len(msg) <= maxTaskErrorMessageLength {
downloader/internal/worker/worker_test.go
+9 -2
View File
@@ -28,6 +28,7 @@ func TestUploadFileSendsContentLength(t *testing.T) {
path := writeTempFile(t, "hello world")
var contentLength string
var contentDisposition string
var uploaded int64
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
contentLength = r.Header.Get("Content-Length")
@@ -39,7 +40,10 @@ func TestUploadFileSendsContentLength(t *testing.T) {
}))
defer server.Close()
if err := uploadFile(context.Background(), server.URL, path, `attachment; filename="hello.txt"`); err != nil {
if err := uploadFile(context.Background(), server.URL, path, `attachment; filename="hello.txt"`, func(written int64) error {
uploaded += written
return nil
}); err != nil {
t.Fatalf("uploadFile returned error: %v", err)
}
if contentLength != "11" {
@@ -48,6 +52,9 @@ func TestUploadFileSendsContentLength(t *testing.T) {
if contentDisposition != `attachment; filename="hello.txt"` {
t.Fatalf("expected Content-Disposition header, got %q", contentDisposition)
}
if uploaded != 11 {
t.Fatalf("expected uploaded bytes 11, got %d", uploaded)
}
}
func TestUploadFileIncludesErrorBody(t *testing.T) {
@@ -57,7 +64,7 @@ func TestUploadFileIncludesErrorBody(t *testing.T) {
}))
defer server.Close()
err := uploadFile(context.Background(), server.URL, path, "")
err := uploadFile(context.Background(), server.URL, path, "", nil)
if err == nil {
t.Fatal("expected uploadFile error")
}
migrations/0037_add-download-task-upload-progress.sql
+1
View File
@@ -0,0 +1 @@
ALTER TABLE `download_tasks` ADD `uploaded_bytes` integer DEFAULT 0 NOT NULL;
migrations/meta/0037_snapshot.json
+3922
View File
File diff suppressed because it is too large Load Diff
migrations/meta/_journal.json
+7
View File
@@ -253,6 +253,13 @@
"when": 1780499047848,
"tag": "0036_add-download-task-detail",
"breakpoints": true
},
{
"idx": 37,
"version": "6",
"when": 1780502607046,
"tag": "0037_add-download-task-upload-progress",
"breakpoints": true
}
]
}
server/db/schema.ts
+1
View File
@@ -339,6 +339,7 @@ export const downloadTasks = sqliteTable(
assignedDownloaderId: text('assigned_downloader_id'),
status: text('status').notNull(),
downloadedBytes: integer('downloaded_bytes').notNull().default(0),
uploadedBytes: integer('uploaded_bytes').notNull().default(0),
totalBytes: integer('total_bytes'),
authorizedBytes: integer('authorized_bytes').notNull().default(0),
billedBytes: integer('billed_bytes').notNull().default(0),
server/openapi/downloader.ts
+2 -1
View File
@@ -44,7 +44,8 @@ const objectDraftSchema = z
.object({
id: z.string(),
name: z.string(),
uploadUrl: z.string(),
uploadUrl: z.string().optional(),
contentDisposition: z.string().optional(),
})
.openapi('ObjectDraft')
server/routes/download-tasks.integration.test.ts
+58 -1
View File
@@ -224,6 +224,56 @@ describe('Download tasks API integration', () => {
expect(runningTask.detail.infoHash).toBe('abc123')
expect(runningTask.detail.trackers[0].url).toBe('udp://tracker.example/announce')
const createFolderRes = await app.request('/api/objects', {
method: 'POST',
headers: uploadHeaders,
body: JSON.stringify({
name: 'fixture-dir',
type: 'folder',
dirtype: 1,
parent: 'Remote Downloads',
}),
})
expect(createFolderRes.status).toBe(201)
const folder = (await createFolderRes.json()) as { id: string; name: string; status: string; dirtype: number }
expect(folder.status).toBe('active')
expect(folder.name).toBe('fixture-dir')
expect(folder.dirtype).toBe(1)
const createNestedObjectRes = await app.request('/api/objects', {
method: 'POST',
headers: uploadHeaders,
body: JSON.stringify({
name: 'nested.txt',
type: 'text/plain',
size: 0,
parent: 'Remote Downloads/fixture-dir',
}),
})
expect(createNestedObjectRes.status).toBe(201)
const nestedObject = (await createNestedObjectRes.json()) as { id: string; status: string; uploadUrl: string }
expect(nestedObject.status).toBe('draft')
expect(nestedObject.uploadUrl).toBe('https://presigned-upload.example.com')
const nestedConfirmRes = await app.request(`/api/objects/${nestedObject.id}`, {
method: 'PATCH',
headers: uploadHeaders,
body: JSON.stringify({ action: 'confirm', onConflict: 'fail' }),
})
expect(nestedConfirmRes.status).toBe(200)
const outsideFolderRes = await app.request('/api/objects', {
method: 'POST',
headers: uploadHeaders,
body: JSON.stringify({
name: 'outside',
type: 'folder',
dirtype: 1,
parent: 'Other Folder',
}),
})
expect(outsideFolderRes.status).toBe(403)
const createObjectRes = await app.request('/api/objects', {
method: 'POST',
headers: uploadHeaders,
@@ -280,6 +330,7 @@ describe('Download tasks API integration', () => {
body: JSON.stringify({
status: 'completed',
downloadedBytes: 10 * 1024 * 1024,
uploadedBytes: 10 * 1024 * 1024,
totalBytes: 10 * 1024 * 1024,
resultObjectId: object.id,
}),
@@ -288,9 +339,15 @@ describe('Download tasks API integration', () => {
const taskRes = await app.request(`/api/download-tasks/${createdTask.id}`, { headers: user })
expect(taskRes.status).toBe(200)
const task = (await taskRes.json()) as { status: string; resultObjectId: string; downloadedBytes: number }
const task = (await taskRes.json()) as {
status: string
resultObjectId: string
downloadedBytes: number
uploadedBytes: number
}
expect(task.status).toBe('completed')
expect(task.resultObjectId).toBe(object.id)
expect(task.downloadedBytes).toBe(10 * 1024 * 1024)
expect(task.uploadedBytes).toBe(10 * 1024 * 1024)
})
})
server/routes/download-tasks.ts
+1
View File
@@ -40,6 +40,7 @@ const downloadTaskSchema = z.object({
targetFolder: z.string(),
status: z.enum(['queued', 'assigned', 'running', 'billing_paused', 'uploading', 'completed', 'failed', 'canceled']),
downloadedBytes: int64Schema(),
uploadedBytes: int64Schema(),
totalBytes: nullableInt64Schema(),
downloadBps: int64Schema(),
uploadBps: int64Schema(),
server/routes/objects.ts
+20 -4
View File
@@ -65,6 +65,21 @@ function conflictBody(err: NameConflictError) {
}
}
function normalizeMatterPath(path: string): string {
return path
.split('/')
.map((part) => part.trim())
.filter(Boolean)
.join('/')
}
function isWithinDownloadTarget(parent: string, targetFolder: string): boolean {
const normalizedParent = normalizeMatterPath(parent)
const normalizedTarget = normalizeMatterPath(targetFolder)
if (!normalizedTarget) return true
return normalizedParent === normalizedTarget || normalizedParent.startsWith(`${normalizedTarget}/`)
}
const ROLE_LEVELS: Record<string, number> = { owner: 3, editor: 2, viewer: 1, member: 1 }
const requireObjectCreateAccess = createMiddleware<Env>(async (c, next) => {
@@ -127,8 +142,7 @@ const app = new Hono<Env>()
const { name, type, size, parent, dirtype, onConflict } = c.req.valid('json')
const isFolder = dirtype !== DirType.FILE
if (principal?.kind === 'download-task-upload') {
if (isFolder) return c.json({ error: 'Download task upload cannot create folders' }, 403)
if (parent !== principal.targetFolder)
if (!isWithinDownloadTarget(parent, principal.targetFolder))
return c.json({ error: 'Target folder is outside task authorization' }, 403)
await assertTaskUploadAllowed(c.get('platform'), {
taskId: principal.taskId,
@@ -262,7 +276,8 @@ const app = new Hono<Env>()
if (!storage) throw new ObjectUploadSessionError('not_found')
const principal = c.get('principal')
if (principal?.kind === 'download-task-upload') {
if (matter.parent !== principal.targetFolder) throw new ObjectUploadSessionError('invalid_state')
if (!isWithinDownloadTarget(matter.parent, principal.targetFolder))
throw new ObjectUploadSessionError('invalid_state')
await assertTaskUploadAllowed(c.get('platform'), {
taskId: principal.taskId,
downloaderId: principal.downloaderId,
@@ -372,7 +387,8 @@ const app = new Hono<Env>()
if (body.action !== 'confirm')
return c.json({ error: 'Download task upload token can only confirm uploads' }, 403)
const matter = await getMatter(db, c.req.param('id'), orgId)
if (!matter || matter.parent !== principal.targetFolder) return c.json({ error: 'Forbidden' }, 403)
if (!matter || !isWithinDownloadTarget(matter.parent, principal.targetFolder))
return c.json({ error: 'Forbidden' }, 403)
await assertTaskUploadAllowed(c.get('platform'), {
taskId: principal.taskId,
downloaderId: principal.downloaderId,
server/services/downloads.ts
+7
View File
@@ -284,6 +284,7 @@ export async function updateDownloadTask(
const onlyCancel =
input.status === 'canceled' &&
input.downloadedBytes === undefined &&
input.uploadedBytes === undefined &&
input.totalBytes === undefined &&
input.downloadBps === undefined &&
input.uploadBps === undefined &&
@@ -303,6 +304,10 @@ export async function updateDownloadTask(
actor.downloaderId && input.downloadedBytes !== undefined
? Math.max(input.downloadedBytes, task.downloadedBytes)
: (input.downloadedBytes ?? task.downloadedBytes)
const uploadedBytes =
actor.downloaderId && input.uploadedBytes !== undefined
? Math.max(input.uploadedBytes, task.uploadedBytes)
: (input.uploadedBytes ?? task.uploadedBytes)
if (actor.downloaderId && downloadedBytes > task.downloadedBytes) {
const downloader = await loadDownloaderRow(platform, actor.downloaderId)
@@ -342,6 +347,7 @@ export async function updateDownloadTask(
.set({
status,
downloadedBytes,
uploadedBytes,
totalBytes: input.totalBytes === undefined ? task.totalBytes : input.totalBytes,
authorizedBytes,
billedBytes,
@@ -504,6 +510,7 @@ function toDownloadTask(row: DownloadTaskRow): DownloadTask {
assignedDownloaderId: row.assignedDownloaderId,
status: row.status as DownloadTask['status'],
downloadedBytes: row.downloadedBytes,
uploadedBytes: row.uploadedBytes,
totalBytes: row.totalBytes,
authorizedBytes: row.authorizedBytes,
billedBytes: row.billedBytes,
server/test/setup.ts
+1
View File
@@ -428,6 +428,7 @@ const APP_SCHEMA_SQL = `
assigned_downloader_id TEXT,
status TEXT NOT NULL,
downloaded_bytes INTEGER NOT NULL DEFAULT 0,
uploaded_bytes INTEGER NOT NULL DEFAULT 0,
total_bytes INTEGER,
authorized_bytes INTEGER NOT NULL DEFAULT 0,
billed_bytes INTEGER NOT NULL DEFAULT 0,
shared/schemas/downloads.ts
+1
View File
@@ -96,6 +96,7 @@ export const createDownloadTaskSchema = z.object({
export const updateDownloadTaskSchema = z.object({
status: downloadTaskStatusSchema.optional(),
downloadedBytes: z.number().int().min(0).optional(),
uploadedBytes: z.number().int().min(0).optional(),
totalBytes: z.number().int().min(0).nullable().optional(),
downloadBps: z.number().int().min(0).optional(),
uploadBps: z.number().int().min(0).optional(),
shared/types/index.ts
+1
View File
@@ -252,6 +252,7 @@ export interface DownloadTask {
assignedDownloaderId: string | null
status: DownloadTaskStatus
downloadedBytes: number
uploadedBytes: number
totalBytes: number | null
authorizedBytes: number
billedBytes: number
src/i18n/locales/en.json
+1
View File
@@ -176,6 +176,7 @@
"downloads.detail.seeders": "Seeders",
"downloads.detail.leechers": "Leechers",
"downloads.detail.uploaded": "Uploaded",
"downloads.detail.downloaded": "Downloaded",
"downloads.detail.trackers": "Trackers",
"downloads.detail.noTrackers": "No tracker data yet",
"downloads.detail.trackerUrl": "Tracker",
src/i18n/locales/zh.json
+1
View File
@@ -176,6 +176,7 @@
"downloads.detail.seeders": "做种",
"downloads.detail.leechers": "下载者",
"downloads.detail.uploaded": "已上传",
"downloads.detail.downloaded": "已下载",
"downloads.detail.trackers": "Trackers",
"downloads.detail.noTrackers": "暂无 tracker 信息",
"downloads.detail.trackerUrl": "Tracker",
src/routes/_authenticated/downloads/index.tsx
+46 -8
View File
@@ -364,8 +364,7 @@ function TaskRow({
onCancel: (id: string) => void
}) {
const { t } = useTranslation()
const total = task.totalBytes ?? task.downloadedBytes
const progress = total > 0 ? Math.min(100, Math.round((task.downloadedBytes / total) * 100)) : 0
const progress = transferProgress(task)
const active = ACTIVE_STATUSES.has(task.status)
const detail = task.detail
@@ -391,8 +390,8 @@ function TaskRow({
</TableCell>
<TableCell className="min-w-48 py-1">
<div className="flex items-center gap-2">
<Progress value={progress} className="h-1.5" />
<span className="w-8 text-right text-[11px] tabular-nums text-muted-foreground">{progress}%</span>
<TransferProgress task={task} className="h-1.5" />
<span className="w-8 text-right text-[11px] tabular-nums text-muted-foreground">{progress.overall}%</span>
</div>
<div className="mt-0.5 flex items-center gap-2 text-[11px] tabular-nums">
<span>{formatBytes(task.downloadBps)}/s ↓</span>
@@ -427,6 +426,33 @@ function TaskRow({
)
}
function TransferProgress({ task, className }: { task: DownloadTask; className?: string }) {
const progress = transferProgress(task)
return (
<div
className={cn('relative w-full overflow-hidden rounded-full bg-muted', className)}
role="progressbar"
aria-valuemin={0}
aria-valuemax={100}
aria-valuenow={progress.overall}
>
<div className="absolute inset-y-0 left-0 bg-sky-500 transition-all" style={{ width: `${progress.download}%` }} />
<div
className="absolute inset-y-0 left-0 bg-emerald-500 transition-all"
style={{ width: `${progress.upload}%` }}
/>
</div>
)
}
function transferProgress(task: DownloadTask) {
const total = Math.max(task.totalBytes ?? task.downloadedBytes, task.downloadedBytes, task.uploadedBytes, 0)
if (total <= 0) return { download: 0, upload: 0, overall: task.status === 'completed' ? 100 : 0 }
const download = Math.min(100, Math.round((task.downloadedBytes / total) * 100))
const upload = task.status === 'completed' ? 100 : Math.min(100, Math.round((task.uploadedBytes / total) * 100))
return { download, upload, overall: task.status === 'uploading' || upload > 0 ? upload : download }
}
function DownloadInspector({
task,
tab,
@@ -483,8 +509,7 @@ function DownloadInspector({
function OverviewPanel({ task }: { task: DownloadTask }) {
const { t } = useTranslation()
const detail = task.detail
const total = task.totalBytes ?? task.downloadedBytes
const progress = total > 0 ? Math.min(100, Math.round((task.downloadedBytes / total) * 100)) : 0
const progress = transferProgress(task)
return (
<div className="space-y-3">
@@ -511,8 +536,21 @@ function OverviewPanel({ task }: { task: DownloadTask }) {
/>
</div>
<div className="space-y-1.5">
<TransferProgress task={task} className="h-2" />
<div className="flex flex-wrap items-center gap-x-4 gap-y-1 text-[11px] tabular-nums text-muted-foreground">
<span>
{t('downloads.detail.downloaded')}: {formatBytes(task.downloadedBytes)}
</span>
<span>
{t('downloads.detail.uploaded')}: {formatBytes(task.uploadedBytes)}
</span>
<span>{progress.overall}%</span>
</div>
</div>
<div className="grid gap-x-5 gap-y-2 text-xs sm:grid-cols-2 xl:grid-cols-4">
<InspectorField label={t('downloads.detail.progress')} value={`${progress}%`} />
<InspectorField label={t('downloads.detail.progress')} value={`${progress.overall}%`} />
<InspectorField label={t('downloads.detail.engine')} value={detail?.engine || t('downloads.unknown')} />
<InspectorField label={t('downloads.detail.phase')} value={detail?.phase || '-'} />
<InspectorField
@@ -536,7 +574,7 @@ function OverviewPanel({ task }: { task: DownloadTask }) {
<InspectorField label={t('downloads.detail.torrentName')} value={detail?.torrentName || '-'} />
<InspectorField label={t('downloads.detail.seeders')} value={formatNumber(detail?.seeders)} />
<InspectorField label={t('downloads.detail.leechers')} value={formatNumber(detail?.leechers)} />
<InspectorField label={t('downloads.detail.uploaded')} value={formatBytes(detail?.uploadedBytes ?? 0)} />
<InspectorField label={t('downloads.detail.uploaded')} value={formatBytes(task.uploadedBytes)} />
<InspectorField label={t('downloads.detail.createdAt')} value={formatDate(task.createdAt)} />
<InspectorField label={t('downloads.detail.startedAt')} value={formatDate(task.startedAt)} />
<InspectorField label={t('downloads.detail.finishedAt')} value={formatDate(task.finishedAt)} />