From 3fdc7b4ae0a6007481d06ad000229d49b6516b79 Mon Sep 17 00:00:00 2001 From: saltbo Date: Mon, 29 Jun 2026 17:42:24 -0400 Subject: [PATCH] refactor(downloader): reorganize cmd downloader runtime --- .github/workflows/ci.yml | 5 +- cmd/internal/client/client.go | 8 + cmd/internal/client/client_test.go | 447 +- cmd/internal/config/config_test.go | 72 + cmd/internal/downloader/api.go | 66 + .../{worker => downloader}/api_test.go | 10 +- .../{worker => downloader}/attempt_ledger.go | 2 +- cmd/internal/downloader/contract.go | 255 ++ cmd/internal/downloader/coverage_test.go | 972 ++++ cmd/internal/downloader/manager.go | 473 ++ cmd/internal/downloader/registry.go | 39 + cmd/internal/downloader/registry_test.go | 201 + .../{worker => downloader}/seed_ledger.go | 2 +- cmd/internal/{worker => downloader}/seeds.go | 358 +- cmd/internal/downloader/task_mapper.go | 273 ++ .../worker.go => downloader/task_runner.go} | 362 +- .../task_runner_test.go} | 654 +-- .../{worker => downloader}/uploader.go | 176 +- cmd/internal/engine/engine.go | 369 -- cmd/internal/engine/engine_test.go | 1041 ----- cmd/internal/engine/http.go | 196 - cmd/internal/engine/process_windows.go | 7 - cmd/internal/openapi/client.gen.go | 24 +- cmd/internal/worker/api.go | 117 - cmd/internal/worker/disk_unix_test.go | 25 - cmd/internal/worker/engines.go | 227 - cmd/main.go | 27 +- cmd/main_test.go | 181 +- .../engine => pkg/downloaders/aria2}/aria2.go | 297 +- cmd/pkg/downloaders/aria2/aria2_test.go | 570 +++ cmd/pkg/downloaders/core/layout.go | 234 + cmd/pkg/downloaders/core/layout_test.go | 256 ++ cmd/pkg/downloaders/core/peer.go | 13 + cmd/pkg/downloaders/core/trackers.go | 95 + cmd/pkg/downloaders/core/trackers_test.go | 102 + cmd/pkg/downloaders/httpdl/http.go | 253 ++ cmd/pkg/downloaders/httpdl/http_test.go | 337 ++ .../downloaders}/live_download_test.go | 41 +- .../downloaders/qbittorrent}/qbittorrent.go | 248 +- .../qbittorrent/qbittorrent_test.go | 316 ++ cmd/{internal/engine => pkg/geoip}/geoip.go | 23 +- cmd/pkg/geoip/geoip_test.go | 72 + .../worker => pkg/system}/disk_unix.go | 4 +- cmd/pkg/system/disk_unix_test.go | 62 + .../worker => pkg/system}/disk_windows.go | 4 +- cmd/pkg/system/files.go | 51 + cmd/pkg/system/files_test.go | 57 + cmd/{internal/host => pkg/system}/hostname.go | 2 +- .../host => pkg/system}/hostname_test.go | 2 +- .../engine => pkg/system}/process.go | 25 +- cmd/pkg/system/process_test.go | 63 + .../engine => pkg/system}/process_unix.go | 4 +- cmd/pkg/system/process_unix_test.go | 16 + cmd/pkg/system/process_windows.go | 7 + cmd/scripts/test-coverage.sh | 24 + .../0052_rename-downloader-http-engine.sql | 36 + migrations/meta/0052_snapshot.json | 3918 +++++++++++++++++ migrations/meta/_journal.json | 7 + server/db/schema.ts | 2 +- server/middleware/authz.integration.test.ts | 2 +- server/test/setup.ts | 2 +- shared/schemas/downloads.ts | 2 +- shared/types/index.ts | 2 +- 63 files changed, 10722 insertions(+), 3016 deletions(-) create mode 100644 cmd/internal/downloader/api.go rename cmd/internal/{worker => downloader}/api_test.go (68%) rename cmd/internal/{worker => downloader}/attempt_ledger.go (98%) create mode 100644 cmd/internal/downloader/contract.go create mode 100644 cmd/internal/downloader/coverage_test.go create mode 100644 cmd/internal/downloader/manager.go create mode 100644 cmd/internal/downloader/registry.go create mode 100644 cmd/internal/downloader/registry_test.go rename cmd/internal/{worker => downloader}/seed_ledger.go (98%) rename cmd/internal/{worker => downloader}/seeds.go (59%) create mode 100644 cmd/internal/downloader/task_mapper.go rename cmd/internal/{worker/worker.go => downloader/task_runner.go} (74%) rename cmd/internal/{worker/worker_test.go => downloader/task_runner_test.go} (79%) rename cmd/internal/{worker => downloader}/uploader.go (55%) delete mode 100644 cmd/internal/engine/engine.go delete mode 100644 cmd/internal/engine/engine_test.go delete mode 100644 cmd/internal/engine/http.go delete mode 100644 cmd/internal/engine/process_windows.go delete mode 100644 cmd/internal/worker/api.go delete mode 100644 cmd/internal/worker/disk_unix_test.go delete mode 100644 cmd/internal/worker/engines.go rename cmd/{internal/engine => pkg/downloaders/aria2}/aria2.go (78%) create mode 100644 cmd/pkg/downloaders/aria2/aria2_test.go create mode 100644 cmd/pkg/downloaders/core/layout.go create mode 100644 cmd/pkg/downloaders/core/layout_test.go create mode 100644 cmd/pkg/downloaders/core/peer.go create mode 100644 cmd/pkg/downloaders/core/trackers.go create mode 100644 cmd/pkg/downloaders/core/trackers_test.go create mode 100644 cmd/pkg/downloaders/httpdl/http.go create mode 100644 cmd/pkg/downloaders/httpdl/http_test.go rename cmd/{internal/engine => pkg/downloaders}/live_download_test.go (89%) rename cmd/{internal/engine => pkg/downloaders/qbittorrent}/qbittorrent.go (68%) create mode 100644 cmd/pkg/downloaders/qbittorrent/qbittorrent_test.go rename cmd/{internal/engine => pkg/geoip}/geoip.go (70%) create mode 100644 cmd/pkg/geoip/geoip_test.go rename cmd/{internal/worker => pkg/system}/disk_unix.go (91%) create mode 100644 cmd/pkg/system/disk_unix_test.go rename cmd/{internal/worker => pkg/system}/disk_windows.go (92%) create mode 100644 cmd/pkg/system/files.go create mode 100644 cmd/pkg/system/files_test.go rename cmd/{internal/host => pkg/system}/hostname.go (96%) rename cmd/{internal/host => pkg/system}/hostname_test.go (97%) rename cmd/{internal/engine => pkg/system}/process.go (66%) create mode 100644 cmd/pkg/system/process_test.go rename cmd/{internal/engine => pkg/system}/process_unix.go (65%) create mode 100644 cmd/pkg/system/process_unix_test.go create mode 100644 cmd/pkg/system/process_windows.go create mode 100755 cmd/scripts/test-coverage.sh create mode 100644 migrations/0052_rename-downloader-http-engine.sql create mode 100644 migrations/meta/0052_snapshot.json diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7f76955f..6564d452 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -35,12 +35,13 @@ jobs: echo "$files" exit 1 fi - - run: go test ./... + - name: Test with coverage threshold + run: bash scripts/test-coverage.sh - name: Install live downloader engines run: sudo apt-get update && sudo apt-get install -y aria2 qbittorrent-nox working-directory: . - name: Verify live downloads - run: LIVE_DOWNLOAD_VERIFY=1 go test ./internal/engine -run 'TestLive(DownloadThreeSourceTypes|QBittorrentDownloadTorrentURL)' -count=1 -v + run: LIVE_DOWNLOAD_VERIFY=1 go test ./pkg/downloaders -run 'TestLive(DownloadThreeSourceTypes|QBittorrentDownloadTorrentURL)' -count=1 -v check: name: Typecheck & Test diff --git a/cmd/internal/client/client.go b/cmd/internal/client/client.go index a4334e21..c1d27122 100644 --- a/cmd/internal/client/client.go +++ b/cmd/internal/client/client.go @@ -686,6 +686,14 @@ func (c *Client) AbortObjectUploadSession(ctx context.Context, token string, id return expectStatus("DELETE", "/api/objects/"+id+"/uploads/"+sessionID, res.StatusCode(), res.Body, http.StatusNoContent) } +func (c *Client) DeleteObject(ctx context.Context, token string, id string) error { + res, err := c.api.DeleteObjectWithResponse(ctx, id, bearer(token)) + if err != nil { + return err + } + return expectStatus("DELETE", "/api/objects/"+id, res.StatusCode(), res.Body, http.StatusNoContent) +} + func derefString(value *string) string { if value == nil { return "" diff --git a/cmd/internal/client/client_test.go b/cmd/internal/client/client_test.go index fefb38ef..0f636e93 100644 --- a/cmd/internal/client/client_test.go +++ b/cmd/internal/client/client_test.go @@ -6,6 +6,7 @@ import ( "net/http" "net/http/httptest" "reflect" + "strings" "testing" ) @@ -56,6 +57,112 @@ func TestCreateObjectUsesRenameConflictStrategy(t *testing.T) { } } +func TestDownloadTaskAccessors(t *testing.T) { + totalBytes := int64(2048) + runtime := &DownloadTaskRuntime{Engine: "http", Phase: "downloading"} + task := downloadTaskFixture("task-1", "downloading") + task.Spec.Source.Type = "bt" + task.Spec.Source.URI = "magnet:?xt=urn:btih:test" + task.Spec.Destination.Name = "movie.mkv" + task.Spec.Destination.Folder = "folder-1" + task.Spec.Labels.Category = "movies" + task.Spec.Labels.Tags = []string{"uhd", "hdr"} + task.Status.Attempt = 2 + task.Status.Progress.Download.TotalBytes = &totalBytes + task.Status.Runtime = runtime + + if task.SourceType() != "bt" || task.SourceURI() != "magnet:?xt=urn:btih:test" { + t.Fatalf("unexpected source accessors: %s %s", task.SourceType(), task.SourceURI()) + } + if task.Name() != "movie.mkv" || task.TargetFolder() != "folder-1" { + t.Fatalf("unexpected destination accessors: %s %s", task.Name(), task.TargetFolder()) + } + if task.Category() != "movies" || !reflect.DeepEqual(task.Tags(), []string{"uhd", "hdr"}) { + t.Fatalf("unexpected label accessors: %s %#v", task.Category(), task.Tags()) + } + if task.State() != "downloading" || task.Attempt() != 2 || task.Runtime() != runtime { + t.Fatalf("unexpected status accessors") + } + if task.UploadToken() != "upload-token" { + t.Fatalf("unexpected upload token: %q", task.UploadToken()) + } + task.Status.Assignment = nil + if task.UploadToken() != "" { + t.Fatalf("expected empty upload token without assignment, got %q", task.UploadToken()) + } +} + +func TestHeartbeatUsesGeneratedRequestShape(t *testing.T) { + var body map[string]any + var auth string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost || r.URL.Path != "/api/downloads/downloaders/me/heartbeats" { + t.Fatalf("unexpected request: %s %s", r.Method, r.URL.Path) + } + auth = r.Header.Get("Authorization") + if err := json.NewDecoder(r.Body).Decode(&body); err != nil { + t.Fatal(err) + } + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{ + "id": "downloader-1", + "name": "node-a", + "engine": "http", + "status": "online", + "enabled": true, + "version": "v1", + "hostname": "host-a", + "platform": "darwin", + "arch": "arm64", + "capabilities": []string{"http"}, + "maxConcurrentTasks": 2, + "currentTasks": 1, + "downloadBps": 100, + "uploadBps": 20, + "freeDiskBytes": 4096, + "remoteDownloadCreditBillingEnabled": false, + "remoteDownloadCreditPerUnit": 0, + "remoteDownloadCreditUnitBytes": 0, + "createdAt": "2026-01-01T00:00:00Z", + "createdBy": "user-1", + "updatedAt": "2026-01-01T00:00:00Z", + "nextPollAfterSeconds": 7, + "assignments": []DownloadTask{downloadTaskFixture("task-assigned", "assigned")}, + "controls": []DownloadTask{downloadTaskFixture("task-pausing", "pausing")}, + }) + })) + defer server.Close() + + result, err := mustClient(t, server.URL, "downloader-token").Heartbeat(context.Background(), Heartbeat{ + Version: "v1", + Hostname: "host-a", + Platform: "darwin", + Arch: "arm64", + Engine: "http", + Capabilities: []string{"http"}, + MaxConcurrentTasks: 2, + CurrentTasks: 1, + DownloadBps: 100, + UploadBps: 20, + FreeDiskBytes: 4096, + }) + if err != nil { + t.Fatal(err) + } + if auth != "Bearer downloader-token" { + t.Fatalf("unexpected auth header: %q", auth) + } + if body["engine"] != "http" || body["hostname"] != "host-a" || body["freeDiskBytes"] != float64(4096) { + t.Fatalf("unexpected heartbeat body: %#v", body) + } + if result.NextPollAfterSeconds != 7 || len(result.Assignments) != 1 || result.Assignments[0].ID != "task-assigned" { + t.Fatalf("unexpected heartbeat assignments: %#v", result) + } + if len(result.Controls) != 1 || result.Controls[0].ID != "task-pausing" { + t.Fatalf("unexpected heartbeat controls: %#v", result) + } +} + func TestAssignedTasksFetchesRunnableStatuses(t *testing.T) { var status string var requests int @@ -111,6 +218,31 @@ func TestAssignedControlTasksFetchesControlStatuses(t *testing.T) { } } +func TestLocalResultTasksFetchesRetryableStatuses(t *testing.T) { + var status string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Query().Get("assignedTo") != "me" { + t.Fatalf("expected assignedTo=me, got %q", r.URL.Query().Get("assignedTo")) + } + status = r.URL.Query().Get("status") + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(Page[DownloadTask]{Items: []DownloadTask{downloadTaskFixture("task-1", "failed")}}) + })) + defer server.Close() + + tasks, err := mustClient(t, server.URL, "token").LocalResultTasks(context.Background()) + if err != nil { + t.Fatal(err) + } + expected := "assigned,downloading,interrupted,uploading,pausing,paused,suspended,failed" + if status != expected { + t.Fatalf("expected status query %q, got %q", expected, status) + } + if len(tasks) != 1 || tasks[0].ID != "task-1" { + t.Fatalf("unexpected tasks: %#v", tasks) + } +} + func TestSeedingTasksFiltersCompletedSeedingPhase(t *testing.T) { var status string server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -136,6 +268,127 @@ func TestSeedingTasksFiltersCompletedSeedingPhase(t *testing.T) { } } +func TestDeviceAuthClientMethods(t *testing.T) { + var codeBody map[string]any + var tokenBody map[string]any + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + switch { + case r.Method == http.MethodPost && r.URL.Path == "/api/auth/device/code": + if err := json.NewDecoder(r.Body).Decode(&codeBody); err != nil { + t.Fatal(err) + } + _ = json.NewEncoder(w).Encode(map[string]any{ + "device_code": "device-1", + "user_code": "ABCD-EFGH", + "verification_uri": "https://zpan.test/device", + "verification_uri_complete": "https://zpan.test/device?user_code=ABCD-EFGH", + "expires_in": 600, + "interval": 5, + }) + case r.Method == http.MethodPost && r.URL.Path == "/api/auth/device/token": + if err := json.NewDecoder(r.Body).Decode(&tokenBody); err != nil { + t.Fatal(err) + } + _ = json.NewEncoder(w).Encode(map[string]any{ + "access_token": "access-token", + "token_type": "Bearer", + "expires_in": 3600, + "scope": "downloader:register", + }) + default: + t.Fatalf("unexpected request: %s %s", r.Method, r.URL.Path) + } + })) + defer server.Close() + + api := mustClient(t, server.URL, "") + code, err := api.RequestDeviceCode(context.Background()) + if err != nil { + t.Fatal(err) + } + if codeBody["client_id"] != "zpan-cli" || codeBody["scope"] != "downloader:register" { + t.Fatalf("unexpected device code body: %#v", codeBody) + } + if code.DeviceCode != "device-1" || code.UserCode != "ABCD-EFGH" || code.ExpiresIn != 600 || code.Interval != 5 { + t.Fatalf("unexpected device code: %#v", code) + } + + token, err := api.PollDeviceToken(context.Background(), "device-1") + if err != nil { + t.Fatal(err) + } + if tokenBody["client_id"] != "zpan-cli" || tokenBody["device_code"] != "device-1" { + t.Fatalf("unexpected device token body: %#v", tokenBody) + } + if tokenBody["grant_type"] != "urn:ietf:params:oauth:grant-type:device_code" { + t.Fatalf("unexpected grant type: %#v", tokenBody) + } + if token.AccessToken != "access-token" || token.TokenType != "Bearer" || token.ExpiresIn != 3600 || token.Scope != "downloader:register" { + t.Fatalf("unexpected device token: %#v", token) + } +} + +func TestCreateDownloaderUsesHeartbeatRequestShape(t *testing.T) { + var body map[string]any + var auth string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost || r.URL.Path != "/api/downloads/downloaders" { + t.Fatalf("unexpected request: %s %s", r.Method, r.URL.Path) + } + auth = r.Header.Get("Authorization") + if err := json.NewDecoder(r.Body).Decode(&body); err != nil { + t.Fatal(err) + } + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusCreated) + _ = json.NewEncoder(w).Encode(map[string]any{ + "downloader": map[string]any{ + "id": "downloader-1", + "name": "node-a", + "engine": "http", + "status": "online", + "enabled": true, + }, + "token": "downloader-token", + }) + })) + defer server.Close() + + out, err := mustClient(t, server.URL, "").CreateDownloader(context.Background(), "access-token", CreateDownloaderRequest{ + Name: "node-a", + Heartbeat: Heartbeat{ + Version: "v1", + Hostname: "host-a", + Platform: "darwin", + Arch: "arm64", + Engine: "http", + Capabilities: []string{"http"}, + MaxConcurrentTasks: 2, + CurrentTasks: 1, + DownloadBps: 100, + UploadBps: 20, + FreeDiskBytes: 4096, + }, + }) + if err != nil { + t.Fatal(err) + } + if auth != "Bearer access-token" { + t.Fatalf("unexpected auth header: %q", auth) + } + if body["name"] != "node-a" { + t.Fatalf("unexpected create downloader body: %#v", body) + } + heartbeat, ok := body["heartbeat"].(map[string]any) + if !ok || heartbeat["engine"] != "http" || heartbeat["downloadBps"] != float64(100) { + t.Fatalf("unexpected heartbeat body: %#v", body) + } + if out.Downloader.ID != "downloader-1" || out.Token != "downloader-token" { + t.Fatalf("unexpected create downloader response: %#v", out) + } +} + func TestUpdateTaskUsesGeneratedRequestShape(t *testing.T) { var body map[string]any server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -159,7 +412,7 @@ func TestUpdateTaskUsesGeneratedRequestShape(t *testing.T) { Download: &DownloadTaskTransferProgress{Bytes: downloadedBytes, TotalBytes: &totalBytes}, }, Runtime: &DownloadTaskRuntime{ - Engine: "builtin", + Engine: "http", Phase: "downloading", ETASeconds: &etaSeconds, Files: []DownloadTaskFile{{Path: "file.bin", Size: 2048, CompletedBytes: &downloadedBytes}}, @@ -183,7 +436,7 @@ func TestUpdateTaskUsesGeneratedRequestShape(t *testing.T) { t.Fatalf("unexpected progress body: %#v", body["progress"]) } runtime, ok := body["runtime"].(map[string]any) - if !ok || runtime["engine"] != "builtin" || runtime["phase"] != "downloading" || runtime["etaSeconds"] != float64(30) { + if !ok || runtime["engine"] != "http" || runtime["phase"] != "downloading" || runtime["etaSeconds"] != float64(30) { t.Fatalf("unexpected runtime body: %#v", body["runtime"]) } } @@ -192,6 +445,7 @@ func TestMultipartUploadSessionClientMethods(t *testing.T) { var presignBody map[string]any var completeBody map[string]any var abortCalled bool + var deleteCalled bool server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") switch { @@ -219,6 +473,9 @@ func TestMultipartUploadSessionClientMethods(t *testing.T) { case r.Method == http.MethodDelete && r.URL.Path == "/api/objects/object-1/uploads/session-1": abortCalled = true w.WriteHeader(http.StatusNoContent) + case r.Method == http.MethodDelete && r.URL.Path == "/api/objects/root-folder": + deleteCalled = true + w.WriteHeader(http.StatusNoContent) default: t.Fatalf("unexpected request: %s %s", r.Method, r.URL.Path) } @@ -241,6 +498,10 @@ func TestMultipartUploadSessionClientMethods(t *testing.T) { if err != nil { t.Fatal(err) } + err = api.DeleteObject(context.Background(), "upload-token", "root-folder") + if err != nil { + t.Fatal(err) + } if !reflect.DeepEqual(presignBody["partNumbers"], []any{float64(1), float64(2)}) { t.Fatalf("expected presign part numbers, got %#v", presignBody) @@ -256,6 +517,188 @@ func TestMultipartUploadSessionClientMethods(t *testing.T) { if !abortCalled { t.Fatalf("abort was not called") } + if !deleteCalled { + t.Fatalf("delete object was not called") + } +} + +func TestCreateObjectMapsUploadInstructions(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusCreated) + _ = json.NewEncoder(w).Encode(map[string]any{ + "id": "object-1", + "name": "movie.mkv", + "upload": map[string]any{ + "sessionId": "session-1", + "partSize": 1024, + "urls": []string{"https://s3/part-1"}, + }, + }) + })) + defer server.Close() + + draft, err := mustClient(t, server.URL, "token").CreateObject(context.Background(), "upload-token", "movie.mkv", 1024, "") + if err != nil { + t.Fatal(err) + } + if draft.Upload == nil { + t.Fatalf("expected upload instructions: %#v", draft) + } + if draft.Upload.SessionID != "session-1" || draft.Upload.PartSize != 1024 || !reflect.DeepEqual(draft.Upload.URLs, []string{"https://s3/part-1"}) { + t.Fatalf("unexpected upload instructions: %#v", draft.Upload) + } +} + +func TestCreateFolderUsesFolderMatterShape(t *testing.T) { + var body map[string]any + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost || r.URL.Path != "/api/objects" { + t.Fatalf("unexpected request: %s %s", r.Method, r.URL.Path) + } + if err := json.NewDecoder(r.Body).Decode(&body); err != nil { + t.Fatal(err) + } + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusCreated) + _ = json.NewEncoder(w).Encode(ObjectDraft{ID: "folder-1", Name: "Movies"}) + })) + defer server.Close() + + folder, err := mustClient(t, server.URL, "token").CreateFolder(context.Background(), "upload-token", "Movies", "parent-1") + if err != nil { + t.Fatal(err) + } + if folder.ID != "folder-1" || folder.Name != "Movies" { + t.Fatalf("unexpected folder draft: %#v", folder) + } + if body["name"] != "Movies" || body["parent"] != "parent-1" || body["dirtype"] != float64(dirTypeUserFolder) || body["type"] != "folder" { + t.Fatalf("unexpected folder body: %#v", body) + } + if body["size"] != float64(0) || body["onConflict"] != "rename" { + t.Fatalf("unexpected folder defaults: %#v", body) + } +} + +func TestClientErrorResponsesIncludeProblemBody(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + http.Error(w, `{"message":"not authorized"}`, http.StatusUnauthorized) + })) + defer server.Close() + + _, err := mustClient(t, server.URL, "token").AssignedTasks(context.Background()) + if err == nil { + t.Fatal("expected assigned tasks error") + } + if !strings.Contains(err.Error(), "GET /api/downloads/tasks failed") || !strings.Contains(err.Error(), "not authorized") { + t.Fatalf("unexpected error: %v", err) + } +} + +func TestClientResponseAndConversionHelpers(t *testing.T) { + if (TaskPatch{Status: "failed"}).State() != "failed" { + t.Fatal("unexpected task patch state") + } + if derefString(nil) != "" { + t.Fatal("nil string pointer should dereference to empty string") + } + if derefFloatToInt(nil) != 0 { + t.Fatal("nil float pointer should dereference to zero") + } + if responseError(nil) != "empty response body" { + t.Fatalf("unexpected empty response error") + } + if responseError([]byte(`{"error":"bad request"}`)) != "bad request" { + t.Fatalf("unexpected json error response") + } + if responseError([]byte(" plain text ")) != "plain text" { + t.Fatalf("unexpected text error response") + } + if _, err := downloadTaskFromOpenAPI(make(chan int)); err == nil { + t.Fatal("expected single task conversion error") + } + if _, err := downloadTasksFromOpenAPI(make(chan int)); err == nil { + t.Fatal("expected task list conversion error") + } +} + +func TestClientEmptyResponseBranches(t *testing.T) { + tests := []struct { + name string + call func(*Client) error + }{ + { + name: "heartbeat", + call: func(api *Client) error { + _, err := api.Heartbeat(context.Background(), Heartbeat{Engine: "http"}) + return err + }, + }, + { + name: "device code", + call: func(api *Client) error { + _, err := api.RequestDeviceCode(context.Background()) + return err + }, + }, + { + name: "device token", + call: func(api *Client) error { + _, err := api.PollDeviceToken(context.Background(), "device-1") + return err + }, + }, + { + name: "create downloader", + call: func(api *Client) error { + _, err := api.CreateDownloader(context.Background(), "access-token", CreateDownloaderRequest{Heartbeat: Heartbeat{Engine: "http"}}) + return err + }, + }, + { + name: "update task", + call: func(api *Client) error { + _, err := api.UpdateTask(context.Background(), "task-1", TaskPatch{Status: "failed"}) + return err + }, + }, + { + name: "create object", + call: func(api *Client) error { + _, err := api.CreateObject(context.Background(), "upload-token", "file.bin", 1, "") + return err + }, + }, + { + name: "presign parts", + call: func(api *Client) error { + _, err := api.PresignObjectUploadParts(context.Background(), "upload-token", "object-1", "session-1", []int{1}) + return err + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if tt.name == "create downloader" { + w.WriteHeader(http.StatusCreated) + return + } + w.WriteHeader(http.StatusOK) + })) + defer server.Close() + + err := tt.call(mustClient(t, server.URL, "token")) + if err == nil { + t.Fatal("expected empty response error") + } + if !strings.Contains(err.Error(), "empty response") { + t.Fatalf("unexpected error: %v", err) + } + }) + } } func mustClient(t *testing.T, baseURL string, token string) *Client { diff --git a/cmd/internal/config/config_test.go b/cmd/internal/config/config_test.go index 2b9caeb5..f9c10c68 100644 --- a/cmd/internal/config/config_test.go +++ b/cmd/internal/config/config_test.go @@ -220,6 +220,78 @@ func TestWriteConfigStoresGlobalTokenAndOmitsDefaultRuntimeBlocks(t *testing.T) } } +func TestConfigFormattingHelpers(t *testing.T) { + t.Setenv("XDG_DATA_HOME", "/xdg/data") + if got := defaultGeoIPDBPath("/home/me"); got != filepath.Join("/xdg/data", "zpan", "geoip.mmdb") { + t.Fatalf("unexpected xdg geoip path: %s", got) + } + t.Setenv("XDG_DATA_HOME", "") + if got := defaultGeoIPDBPath("/home/me"); got != filepath.Join("/home/me", ".local", "share", "zpan", "geoip.mmdb") { + t.Fatalf("unexpected default geoip path: %s", got) + } + if got := defaultStateDir(""); got != filepath.Join(".zpan", "downloader") { + t.Fatalf("unexpected empty-home state dir: %s", got) + } + if got := defaultStateDir("/home/me"); got != filepath.Join("/home/me", ".local", "state", "zpan", "downloader") { + t.Fatalf("unexpected state dir: %s", got) + } + if got := formatSeedCacheLimit(1234); got != "1234" { + t.Fatalf("unexpected seed cache limit: %s", got) + } + if got := formatDuration(0, "5s"); got != "5s" { + t.Fatalf("unexpected fallback duration: %s", got) + } + if got := formatDuration(time.Hour, "5s"); got != "1h" { + t.Fatalf("unexpected hour duration: %s", got) + } + if got := formatDuration(1500*time.Millisecond, "5s"); got != "1.5s" { + t.Fatalf("unexpected fractional duration: %s", got) + } + if got := DefaultConfigPath(); !strings.HasSuffix(got, filepath.Join(".config", "zpan", "config.yaml")) { + t.Fatalf("unexpected default config path: %s", got) + } +} + +func TestWriteRuntimeBlocksWhenConfigured(t *testing.T) { + path := filepath.Join(t.TempDir(), "config.yaml") + cfg := Config{ + ServerURL: "https://zpan.space", + Engine: "qbittorrent", + DownloadDir: "/downloads", + StateDir: "/state", + PollInterval: time.Second, + MaxConcurrentTasks: 1, + SeedDuration: time.Minute, + SeedMaxConcurrent: 1, + Aria2Configured: true, + Aria2Secret: "secret", + QBittorrentConfigured: true, + QBittorrentUser: "admin", + QBittorrentPass: "password", + } + if err := WriteConfig(path, cfg, "token"); err != nil { + t.Fatal(err) + } + text := readConfigFile(t, path) + if !hasConfigLine(text, " aria2:") || !strings.Contains(text, ` secret: "secret"`) { + t.Fatalf("expected aria2 runtime block, got:\n%s", text) + } + if !hasConfigLine(text, " qbittorrent:") || + !strings.Contains(text, ` username: "admin"`) || + !strings.Contains(text, ` password: "password"`) { + t.Fatalf("expected qbittorrent runtime block, got:\n%s", text) + } +} + +func readConfigFile(t *testing.T, path string) string { + t.Helper() + content, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + return string(content) +} + func hasConfigLine(text string, line string) bool { for _, candidate := range strings.Split(text, "\n") { if candidate == line { diff --git a/cmd/internal/downloader/api.go b/cmd/internal/downloader/api.go new file mode 100644 index 00000000..ac7b7992 --- /dev/null +++ b/cmd/internal/downloader/api.go @@ -0,0 +1,66 @@ +package downloader + +import ( + "context" + "fmt" + "log/slog" + "strings" + "time" +) + +const apiRetryAttempts = 3 + +func callAPI(ctx context.Context, logger *slog.Logger, operation string, call func(context.Context) error) error { + if logger == nil { + logger = slog.Default() + } + var last error + for attempt := 1; attempt <= apiRetryAttempts; attempt++ { + if err := ctx.Err(); err != nil { + return err + } + if err := call(ctx); err != nil { + last = err + if attempt == apiRetryAttempts || !isRetryableAPIError(err) { + return err + } + delay := time.Duration(attempt) * 500 * time.Millisecond + logger.Warn("retrying downloader api call", "operation", operation, "attempt", attempt, "delay", delay.String(), "error", err) + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(delay): + } + continue + } + return nil + } + return fmt.Errorf("%s failed: %w", operation, last) +} + +func isRetryableAPIError(err error) bool { + if err == nil { + return false + } + message := err.Error() + return containsAny(message, + "connection refused", + "connection reset", + "connection is shut down", + "timeout", + "temporary failure", + "Too Many Requests", + "Bad Gateway", + "Service Unavailable", + "Gateway Timeout", + ) +} + +func containsAny(value string, needles ...string) bool { + for _, needle := range needles { + if needle != "" && strings.Contains(value, needle) { + return true + } + } + return false +} diff --git a/cmd/internal/worker/api_test.go b/cmd/internal/downloader/api_test.go similarity index 68% rename from cmd/internal/worker/api_test.go rename to cmd/internal/downloader/api_test.go index f7f919aa..efab3e0e 100644 --- a/cmd/internal/worker/api_test.go +++ b/cmd/internal/downloader/api_test.go @@ -1,18 +1,15 @@ -package worker +package downloader import ( "context" "errors" "testing" - - "github.com/saltbo/zpan/internal/config" ) func TestCallAPIRetriesTransientErrors(t *testing.T) { - w := NewWithAPI(config.Config{}, nil) attempts := 0 - err := w.callAPI(context.Background(), "test", func(context.Context) error { + err := callAPI(context.Background(), nil, "test", func(context.Context) error { attempts++ if attempts < 3 { return errors.New("503 Service Unavailable") @@ -29,10 +26,9 @@ func TestCallAPIRetriesTransientErrors(t *testing.T) { } func TestCallAPIDoesNotRetryApplicationErrors(t *testing.T) { - w := NewWithAPI(config.Config{}, nil) attempts := 0 - err := w.callAPI(context.Background(), "test", func(context.Context) error { + err := callAPI(context.Background(), nil, "test", func(context.Context) error { attempts++ return errors.New("Task is paused") }) diff --git a/cmd/internal/worker/attempt_ledger.go b/cmd/internal/downloader/attempt_ledger.go similarity index 98% rename from cmd/internal/worker/attempt_ledger.go rename to cmd/internal/downloader/attempt_ledger.go index fe7449ff..b4a217fc 100644 --- a/cmd/internal/worker/attempt_ledger.go +++ b/cmd/internal/downloader/attempt_ledger.go @@ -1,4 +1,4 @@ -package worker +package downloader import ( "encoding/json" diff --git a/cmd/internal/downloader/contract.go b/cmd/internal/downloader/contract.go new file mode 100644 index 00000000..6fcdcfcd --- /dev/null +++ b/cmd/internal/downloader/contract.go @@ -0,0 +1,255 @@ +package downloader + +import ( + "context" + "time" + + "github.com/saltbo/zpan/pkg/geoip" +) + +type Config struct { + Engine string + DownloadDir string + StateDir string + BTListenPort int + MaxConcurrentDownloads int + SeedEnabled bool + SeedDuration time.Duration + SeedRatio float64 + Aria2 Aria2Config + QBittorrent QBittorrentConfig + GeoIP geoip.Resolver +} + +type Aria2Config struct { + URL string + Secret string + BtTrackers string + Configured bool +} + +type QBittorrentConfig struct { + URL string + Username string + Password string + Configured bool +} + +type DownloadTask struct { + ID string + Source Source + Destination Destination + Labels Labels + Status Status +} + +func (r DownloadTask) SourceType() string { + return r.Source.Type +} + +func (r DownloadTask) SourceURI() string { + return r.Source.URI +} + +func (r DownloadTask) Name() string { + return r.Destination.Name +} + +func (r DownloadTask) Category() string { + return r.Labels.Category +} + +func (r DownloadTask) Tags() []string { + return r.Labels.Tags +} + +func (r DownloadTask) State() string { + return r.Status.State +} + +func (r DownloadTask) Runtime() *TaskRuntime { + return r.Status.Runtime +} + +type Source struct { + Type string + URI string +} + +type Destination struct { + Name string +} + +type Labels struct { + Category string + Tags []string +} + +type Status struct { + State string + Progress TaskProgress + Runtime *TaskRuntime +} + +type TaskProgress struct { + Download TransferProgress +} + +type TransferProgress struct { + Bytes int64 + TotalBytes *int64 + Bps int64 +} + +type Result struct { + Path string + Name string + Size int64 + IsDir bool + Seed *Seed +} + +type Seed struct { + Engine string + ID string + InfoHash string + Path string + Snapshot func(context.Context) (SeedSnapshot, error) + Cleanup func(context.Context) error +} + +type SeedRef struct { + TaskID string + Engine string + ID string + InfoHash string + Path string +} + +type SeedSnapshot struct { + Downloaded int64 + Total *int64 + Bps int64 + Runtime *TaskRuntime +} + +type ProgressUpdate struct { + Downloaded int64 + Total *int64 + Bps int64 + Runtime *TaskRuntime +} + +type ProgressReporter func(ProgressUpdate) error + +type TaskRuntime struct { + Engine string + Phase string + State string + Message string + UpdatedAt string + Progress *RuntimeProgress + ETASeconds *int64 + Connections *int64 + Torrent *TorrentRuntime + Seeding *SeedingRuntime + Trackers []Tracker + Peers []Peer + Files []File +} + +type RuntimeProgress struct { + Download TransferProgress + Upload TransferProgress +} + +type TorrentRuntime struct { + InfoHash string + Name string + Seeders *int64 + Leechers *int64 + Peers *int64 +} + +type SeedingRuntime struct { + Enabled *bool + Active *bool + UploadedBytes *int64 + UploadBytesPerSecond *int64 + Ratio *float64 + StartedAt string + ExpiresAt string +} + +type Tracker struct { + URL string + Status string + Peers *int64 + Seeds *int64 + Leechers *int64 + Message string +} + +type Peer struct { + Address string + Client string + CountryCode string + RegionCode string + Progress *float64 + DownloadBps *int64 + UploadBps *int64 +} + +type File struct { + Path string + Size int64 + CompletedBytes *int64 + Selected *bool +} + +type TaskState string + +const ( + TaskStateDownloading TaskState = "downloading" + TaskStateCompleted TaskState = "completed" + TaskStateFailed TaskState = "failed" +) + +type TaskSnapshot struct { + State TaskState + Downloaded int64 + Total *int64 + Bps int64 + Runtime *TaskRuntime + Result *Result + Error string +} + +type Capabilities struct { + SourceTypes []string +} + +type Downloader interface { + Name() string + Capabilities() Capabilities + Start(ctx context.Context) error + Stop(ctx context.Context) error + Check(ctx context.Context) error + InspectTask(ctx context.Context, task DownloadTask) (TaskSnapshot, bool, error) + Download(ctx context.Context, task DownloadTask, progress ProgressReporter) (Result, error) +} + +type TaskResetter interface { + ResetTask(ctx context.Context, task DownloadTask) error +} + +type SeedRestorer interface { + RestoreSeed(ctx context.Context, ref SeedRef) (*Seed, error) +} + +// SeedLister enumerates every torrent the downloader is currently seeding, +// including ones the worker is no longer tracking. The worker uses this to +// reconcile orphaned seeds so they cannot occupy runtime slots forever. +type SeedLister interface { + ListSeeds(ctx context.Context) ([]Seed, error) +} diff --git a/cmd/internal/downloader/coverage_test.go b/cmd/internal/downloader/coverage_test.go new file mode 100644 index 00000000..8270f5d7 --- /dev/null +++ b/cmd/internal/downloader/coverage_test.go @@ -0,0 +1,972 @@ +package downloader + +import ( + "context" + "errors" + "io" + "log/slog" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "reflect" + "strings" + "sync" + "testing" + "time" + + "github.com/saltbo/zpan/internal/client" + "github.com/saltbo/zpan/internal/config" +) + +func TestDownloadTaskAccessors(t *testing.T) { + runtime := &TaskRuntime{Engine: "aria2", Phase: "downloading"} + task := DownloadTask{ + ID: "task-1", + Source: Source{Type: "magnet", URI: "magnet:?xt=urn:btih:abc"}, + Destination: Destination{Name: "movie.mkv"}, + Labels: Labels{Category: "movies", Tags: []string{"hd", "bt"}}, + Status: Status{State: "downloading", Runtime: runtime}, + } + + if task.SourceType() != "magnet" || task.SourceURI() == "" || task.Name() != "movie.mkv" { + t.Fatalf("unexpected source/name accessors: %#v", task) + } + if task.Category() != "movies" || !reflect.DeepEqual(task.Tags(), []string{"hd", "bt"}) { + t.Fatalf("unexpected labels: %#v", task.Labels) + } + if task.State() != "downloading" || task.Runtime() != runtime { + t.Fatalf("unexpected status accessors: %#v", task.Status) + } +} + +func TestRuntimeMapperPreservesFullDetail(t *testing.T) { + enabled := true + active := true + uploaded := int64(42) + ratio := 1.25 + total := int64(100) + completed := int64(80) + selected := true + progress := 0.5 + downBps := int64(20) + upBps := int64(3) + runtime := &TaskRuntime{ + Engine: "aria2", + Phase: "seeding", + State: "active", + Message: "ok", + UpdatedAt: time.Now().UTC().Format(time.RFC3339), + Progress: &RuntimeProgress{Download: TransferProgress{Bytes: 80, TotalBytes: &total, Bps: downBps}, Upload: TransferProgress{Bytes: 40, TotalBytes: &total, Bps: upBps}}, + ETASeconds: ptrInt64(5), + Connections: ptrInt64(9), + Torrent: &TorrentRuntime{InfoHash: "abc", Name: "torrent", Seeders: ptrInt64(10), Leechers: ptrInt64(2), Peers: ptrInt64(12)}, + Seeding: &SeedingRuntime{Enabled: &enabled, Active: &active, UploadedBytes: &uploaded, UploadBytesPerSecond: &upBps, Ratio: &ratio, StartedAt: "start", ExpiresAt: "end"}, + Trackers: []Tracker{{URL: "udp://tracker", Status: "working", Peers: ptrInt64(3), Seeds: ptrInt64(2), Leechers: ptrInt64(1), Message: "ok"}}, + Peers: []Peer{{Address: "1.1.1.1:6881", Client: "peer", CountryCode: "US", RegionCode: "CA", Progress: &progress, DownloadBps: &downBps, UploadBps: &upBps}}, + Files: []File{{Path: "movie.mkv", Size: total, CompletedBytes: &completed, Selected: &selected}}, + } + + zpan := zpanRuntime(runtime) + roundTrip := downloaderRuntime(zpan) + + if !reflect.DeepEqual(roundTrip, runtime) { + t.Fatalf("runtime mapping lost detail:\nwant %#v\ngot %#v", runtime, roundTrip) + } + if got := zpanTransferProgress(TransferProgress{Bytes: 1, TotalBytes: &total, Bps: 2}); got.Bytes != 1 || got.TotalBytes == nil || *got.TotalBytes != total || got.BytesPerSecond != 2 { + t.Fatalf("unexpected transfer progress mapping: %#v", got) + } +} + +func TestManagerLifecycleHelpers(t *testing.T) { + first := &lifecycleDownloader{name: "aria2", sourceTypes: []string{"magnet"}} + second := &lifecycleDownloader{name: "http", sourceTypes: []string{"http"}} + manager := NewManagerWithDownloaders(first, second) + + if manager.Name() != "aria2" { + t.Fatalf("expected BT manager name, got %q", manager.Name()) + } + if !manager.isStarted() { + t.Fatal("expected injected manager to be started") + } + manager.Stop(context.Background()) + if first.stopCalls != 1 || second.stopCalls != 1 { + t.Fatalf("expected both downloaders to stop, got %d/%d", first.stopCalls, second.stopCalls) + } + if manager.isStarted() { + t.Fatal("expected manager stopped") + } + + empty := NewManager(config.Config{Engine: "qbittorrent"}, nil, nil) + if empty.Name() != "qbittorrent" { + t.Fatalf("expected configured name before start, got %q", empty.Name()) + } + if !reflect.DeepEqual(empty.Capabilities(), []string{"http"}) { + t.Fatalf("expected default http capability before start, got %#v", empty.Capabilities()) + } + if len(NewManagerWithDownloaders().currentDownloaders()) != 0 { + t.Fatal("expected empty downloader list") + } + var nilManager *Manager + nilManager.Stop(context.Background()) + if nilManager.Name() != "auto" { + t.Fatalf("expected nil manager name auto, got %q", nilManager.Name()) + } + if err := nilManager.ready(); err == nil { + t.Fatal("expected nil manager not ready") + } +} + +func TestManagerStartupCleanupOnLaterDownloaderFailure(t *testing.T) { + registerTestDownloaders(t, + testRegistration{name: "aria2"}, + testRegistration{name: "http", fallback: true}, + ) + original := registeredDownloaders + registeredDownloaders = []registration{ + { + name: "aria2", + configured: func(Config) bool { return false }, + new: func(Config) (Downloader, error) { + return &lifecycleDownloader{name: "aria2", sourceTypes: []string{"magnet"}}, nil + }, + }, + { + name: "http", + fallback: true, + configured: func(Config) bool { return false }, + new: func(Config) (Downloader, error) { + return &lifecycleDownloader{name: "http", sourceTypes: []string{"http"}, checkErr: errors.New("http unavailable")}, nil + }, + }, + } + t.Cleanup(func() { registeredDownloaders = original }) + + manager := NewManager(config.Config{}, nil, slog.New(slog.NewTextHandler(io.Discard, nil))) + err := manager.Start(context.Background(), nil) + if err == nil || !strings.Contains(err.Error(), "http") { + t.Fatalf("expected startup failure from http downloader, got %v", err) + } +} + +func TestManagerStartDownloaderErrorBranches(t *testing.T) { + manager := NewManager(config.Config{}, nil, slog.New(slog.NewTextHandler(io.Discard, nil))) + startErr := errors.New("start failed") + if err := manager.startDownloader(context.Background(), &lifecycleDownloader{name: "bad", startErr: startErr}, false); !errors.Is(err, startErr) { + t.Fatalf("expected non-required start error, got %v", err) + } + if err := manager.startDownloader(context.Background(), &lifecycleDownloader{name: "bad", startErr: startErr}, true); err == nil || !strings.Contains(err.Error(), "start downloader") { + t.Fatalf("expected required start wrapper, got %v", err) + } + + checkErr := errors.New("check failed") + checkCtx, cancel := context.WithTimeout(context.Background(), time.Millisecond) + defer cancel() + if err := manager.startDownloader(checkCtx, &lifecycleDownloader{name: "bad", checkErr: checkErr}, false); err == nil { + t.Fatal("expected non-required check error") + } + checkCtx, cancel = context.WithTimeout(context.Background(), time.Millisecond) + defer cancel() + if err := manager.startDownloader(checkCtx, &lifecycleDownloader{name: "bad", checkErr: checkErr}, true); err == nil || !strings.Contains(err.Error(), "not available") { + t.Fatalf("expected required check wrapper, got %v", err) + } +} + +func TestTaskRunnerRunStartupFailures(t *testing.T) { + downloadDirFile := filepath.Join(t.TempDir(), "download-dir") + if err := os.WriteFile(downloadDirFile, []byte("not a dir"), 0o644); err != nil { + t.Fatal(err) + } + runner := NewTaskRunnerWithAPI(config.Config{DownloadDir: downloadDirFile}, &recordingAPI{}) + err := runner.Run(context.Background()) + if err == nil { + t.Fatal("expected download dir mkdir failure") + } + + geoIPFile := filepath.Join(t.TempDir(), "invalid.mmdb") + if err := os.WriteFile(geoIPFile, []byte("not a maxmind db"), 0o644); err != nil { + t.Fatal(err) + } + runner = NewTaskRunnerWithAPI(config.Config{DownloadDir: t.TempDir(), GeoIPDBPath: geoIPFile}, &recordingAPI{}) + err = runner.Run(context.Background()) + if err == nil || !strings.Contains(err.Error(), "open geoip database") { + t.Fatalf("expected geoip open failure, got %v", err) + } + + original := registeredDownloaders + registeredDownloaders = nil + t.Cleanup(func() { registeredDownloaders = original }) + runner = NewTaskRunnerWithAPI(config.Config{Engine: "http", DownloadDir: t.TempDir()}, &recordingAPI{}) + err = runner.Run(context.Background()) + if err == nil || !strings.Contains(err.Error(), "http downloader is not registered") { + t.Fatalf("expected missing http downloader error, got %v", err) + } +} + +func TestManagerWatchDownloaderExitPaths(t *testing.T) { + manager := NewManagerWithDownloaders(&lifecycleDownloader{name: "http", sourceTypes: []string{"http"}}) + manager.watchDownloader(context.Background(), &lifecycleDownloader{name: "http", sourceTypes: []string{"http"}}, nil) + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + manager.watchDownloader(ctx, &lifecycleDownloader{name: "http", sourceTypes: []string{"http"}}, func(error) { + t.Fatal("did not expect exit callback after context cancellation") + }) + + errCh := make(chan error, 1) + go manager.watchDownloader(context.Background(), &lifecycleDownloader{name: "http", sourceTypes: []string{"http"}, checkErr: errors.New("health failed")}, func(err error) { + errCh <- err + }) + select { + case err := <-errCh: + if err == nil || !strings.Contains(err.Error(), "health check failed") { + t.Fatalf("expected health check failure, got %v", err) + } + case <-time.After(6 * time.Second): + t.Fatal("timed out waiting for downloader health failure") + } +} + +func TestCleanupDownloadedResultBranches(t *testing.T) { + seedCleaned := false + err := cleanupDownloadedResult(context.Background(), clientTaskWithStatus("task-1", "completed"), Result{ + Seed: &Seed{Cleanup: func(context.Context) error { + seedCleaned = true + return nil + }}, + }) + if err != nil { + t.Fatal(err) + } + if !seedCleaned { + t.Fatal("expected seed cleanup to be used") + } + + file := writeTempFile(t, "payload") + err = cleanupDownloadedResult(context.Background(), clientTaskWithStatus("task-1", "completed"), Result{Path: file}) + if err != nil { + t.Fatal(err) + } + if _, statErr := os.Stat(file); !os.IsNotExist(statErr) { + t.Fatalf("expected file removal, got %v", statErr) + } +} + +func TestLedgerInvalidJSONAndEmptyFiles(t *testing.T) { + stateDir := t.TempDir() + if err := os.WriteFile(attemptLedgerPath(stateDir), []byte("{"), 0o644); err != nil { + t.Fatal(err) + } + if _, err := loadAttemptLedger(stateDir); err == nil { + t.Fatal("expected invalid attempt ledger error") + } + if err := os.WriteFile(seedLedgerPath(stateDir), []byte("{"), 0o644); err != nil { + t.Fatal(err) + } + if _, err := loadSeedLedger(stateDir); err == nil { + t.Fatal("expected invalid seed ledger error") + } + + emptyDir := t.TempDir() + if err := os.WriteFile(attemptLedgerPath(emptyDir), nil, 0o644); err != nil { + t.Fatal(err) + } + attempts, err := loadAttemptLedger(emptyDir) + if err != nil { + t.Fatal(err) + } + if len(attempts.Attempts) != 0 { + t.Fatalf("expected empty attempts, got %#v", attempts) + } + if err := os.WriteFile(seedLedgerPath(emptyDir), nil, 0o644); err != nil { + t.Fatal(err) + } + seeds, err := loadSeedLedger(emptyDir) + if err != nil { + t.Fatal(err) + } + if len(seeds.Seeds) != 0 { + t.Fatalf("expected empty seed ledger, got %#v", seeds) + } +} + +func TestTaskRunnerRunHappyPathDownloadsUploadsAndCompletes(t *testing.T) { + payload := "happy path payload" + var uploadedBody string + uploadServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, err := io.ReadAll(r.Body) + if err != nil { + t.Fatal(err) + } + uploadedBody = string(body) + w.Header().Set("ETag", `"etag-1"`) + w.WriteHeader(http.StatusOK) + })) + defer uploadServer.Close() + + ctx, cancel := context.WithCancel(context.Background()) + api := &happyPathAPI{ + task: clientHTTPTask("task-1", "assigned", "https://example.com/payload.bin", "payload.bin"), + draft: client.ObjectDraft{ + ID: "object-1", + Name: "payload.bin", + Upload: &client.ObjectUploadInstructions{SessionID: "session-1", PartSize: int64(len(payload)), URLs: []string{uploadServer.URL}}, + }, + onCompleted: cancel, + } + registerHappyPathDownloader(t, payload) + + runner := NewTaskRunnerWithAPI(config.Config{ + Engine: "http", + DownloadDir: t.TempDir(), + PollInterval: time.Millisecond, + MaxConcurrentTasks: 1, + }, api) + runner.logger = slog.New(slog.NewTextHandler(io.Discard, nil)) + + err := runner.Run(ctx) + if err != nil { + t.Fatal(err) + } + if uploadedBody != payload { + t.Fatalf("expected uploaded payload %q, got %q", payload, uploadedBody) + } + if !api.completed { + t.Fatal("expected task to complete") + } + if api.completedObjectID != "object-1" { + t.Fatalf("expected completed object id, got %q", api.completedObjectID) + } +} + +func TestNewTaskRunnerValidatesTokenAndBuildsClient(t *testing.T) { + if _, err := NewTaskRunner(config.Config{}); err == nil { + t.Fatal("expected missing token error") + } + runner, err := NewTaskRunner(config.Config{ServerURL: "https://zpan.example", Token: "token"}) + if err != nil { + t.Fatal(err) + } + if runner.api == nil || runner.uploader == nil || runner.seeds == nil { + t.Fatalf("expected runner dependencies to be initialized: %#v", runner) + } +} + +func TestTaskRunnerIntervalsAndControlBranches(t *testing.T) { + runner := NewTaskRunnerWithAPI(config.Config{PollInterval: 123 * time.Millisecond, MaxConcurrentTasks: 1}, &recordingAPI{}) + runner.logger = slog.New(slog.NewTextHandler(io.Discard, nil)) + if runner.localPollInterval() != 123*time.Millisecond { + t.Fatalf("expected configured local poll interval, got %s", runner.localPollInterval()) + } + if got := NewTaskRunnerWithAPI(config.Config{}, &recordingAPI{}).localPollInterval(); got != 5*time.Second { + t.Fatalf("expected default poll interval, got %s", got) + } + if got := runner.remotePollInterval(client.HeartbeatResult{NextPollAfterSeconds: 2}); got != 2*time.Second { + t.Fatalf("expected remote interval, got %s", got) + } + + task := clientTaskWithStatus("task-1", "suspended") + runner.ackStoppedControlTask(context.Background(), task) + runner.ackStoppedControlTask(context.Background(), clientTaskWithStatus("task-2", "pausing")) + + runner.cleanupDeletedTask(context.Background(), runner.logger, clientTaskWithStatus("task-1", "canceling")) + errEngine := &recordingEngine{resetErr: errors.New("reset failed")} + runner.downloader = NewManagerWithDownloader(errEngine) + runner.cleanupDeletedTask(context.Background(), runner.logger, withRuntime(clientTaskWithStatus("task-3", "canceling"), &client.DownloadTaskRuntime{State: deleteRequestedRuntimeState})) + + _, ok := runner.startTask(context.Background(), "busy") + if !ok { + t.Fatal("expected first task to start") + } + if _, ok := runner.startTask(context.Background(), "busy"); ok { + t.Fatal("expected duplicate task to be rejected") + } + if _, ok := runner.startTask(context.Background(), "other"); ok { + t.Fatal("expected max concurrency to reject task") + } + runner.finish("busy") + + ackAPI := &recordingAPI{updateErr: errors.New("update failed")} + ackRunner := NewTaskRunnerWithAPI(config.Config{}, ackAPI) + ackRunner.logger = slog.New(slog.NewTextHandler(io.Discard, nil)) + ackRunner.ackStoppedControlTask(context.Background(), clientTaskWithStatus("task-4", "pausing")) + ackRunner.ackStoppedControlTask(context.Background(), clientTaskWithStatus("task-5", "canceling")) +} + +func TestTickAndResetErrorBranches(t *testing.T) { + runner := NewTaskRunnerWithAPI(config.Config{MaxConcurrentTasks: 0}, &recordingAPI{ + assignedTasks: []client.DownloadTask{ + clientTaskWithStatus("missing-token", "assigned"), + clientTaskWithUploadToken("busy", "assigned"), + }, + nextPollAfterSeconds: 0, + }) + runner.logger = slog.New(slog.NewTextHandler(io.Discard, nil)) + next, err := runner.tickAndNextPoll(context.Background()) + if err != nil { + t.Fatal(err) + } + if next != runner.localPollInterval() { + t.Fatalf("expected local poll fallback, got %s", next) + } + + runner = NewTaskRunnerWithAPI(config.Config{}, &recordingAPI{heartbeatErr: errors.New("heartbeat failed")}) + if _, err := runner.tickAndNextPoll(context.Background()); err == nil { + t.Fatal("expected heartbeat error") + } + + runner = NewTaskRunnerWithAPI(config.Config{}, &recordingAPI{}) + badAttempt := clientTaskWithStatus("bad", "assigned") + badAttempt.Status.Attempt = 0 + if err := runner.resetTaskForAttempt(context.Background(), badAttempt, runner.logger); err == nil { + t.Fatal("expected invalid attempt error") + } + task := clientTaskWithStatus("task-1", "assigned") + runner.setMemoryAttempt(task.ID, task.Attempt()) + if err := runner.resetTaskForAttempt(context.Background(), task, runner.logger); err != nil { + t.Fatal(err) + } + resetErr := errors.New("reset failed") + runner.downloader = NewManagerWithDownloader(&recordingEngine{resetErr: resetErr}) + retry := clientTaskWithStatus("task-2", "assigned") + retry.Status.Attempt = 2 + if err := runner.resetTaskForAttempt(context.Background(), retry, runner.logger); !errors.Is(err, resetErr) { + t.Fatalf("expected reset error, got %v", err) + } + + ids, ok := runner.localResultTaskIDs(context.Background()) + if !ok || len(ids) != 0 { + t.Fatalf("expected empty local result ids, got %#v ok=%v", ids, ok) + } + runner = NewTaskRunnerWithAPI(config.Config{}, &recordingAPI{localResultErr: errors.New("list failed")}) + if _, ok := runner.localResultTaskIDs(context.Background()); ok { + t.Fatal("expected local result list failure") + } +} + +func TestRegisterValidationAndReplacement(t *testing.T) { + original := registeredDownloaders + registeredDownloaders = nil + t.Cleanup(func() { registeredDownloaders = original }) + + mustPanic(t, func() { + Register("", false, nil, func(Config) (Downloader, error) { return testDownloader{name: "x"}, nil }) + }) + mustPanic(t, func() { + Register("bad", false, nil, nil) + }) + + Register("http", true, nil, func(Config) (Downloader, error) { return testDownloader{name: "old"}, nil }) + Register("http", true, nil, func(Config) (Downloader, error) { return testDownloader{name: "new"}, nil }) + entries := registrations() + if len(entries) != 1 { + t.Fatalf("expected replacement, got %#v", entries) + } + d, err := entries[0].new(Config{}) + if err != nil { + t.Fatal(err) + } + if d.Name() != "new" { + t.Fatalf("expected replacement constructor, got %q", d.Name()) + } + if entries[0].configured(Config{}) { + t.Fatal("expected nil configured callback to default false") + } +} + +func TestDirectoryUploadSuccessCreatesTree(t *testing.T) { + var uploaded []string + uploadServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, err := io.ReadAll(r.Body) + if err != nil { + t.Fatal(err) + } + uploaded = append(uploaded, string(body)) + w.Header().Set("ETag", `"etag"`) + w.WriteHeader(http.StatusOK) + })) + defer uploadServer.Close() + + root := t.TempDir() + if err := os.MkdirAll(filepath.Join(root, "disc"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(root, "disc", "a.txt"), []byte("a"), 0o644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(root, "b.txt"), []byte("b"), 0o644); err != nil { + t.Fatal(err) + } + api := &recordingAPI{ + createFolderDrafts: []client.ObjectDraft{ + {ID: "root-id", Name: "album"}, + {ID: "sub-id", Name: "disc"}, + }, + createObjectDraft: client.ObjectDraft{ + ID: "object-id", + Name: "file", + Upload: &client.ObjectUploadInstructions{SessionID: "session", PartSize: 1, URLs: []string{uploadServer.URL}}, + }, + } + uploader := NewUploader(api, nil) + id, err := uploader.Upload(context.Background(), slog.New(slog.NewTextHandler(io.Discard, nil)), clientTaskWithUploadToken("task-1", "uploading"), Result{ + Path: root, + Name: "album", + Size: 2, + IsDir: true, + }) + if err != nil { + t.Fatal(err) + } + if id != "root-id" { + t.Fatalf("expected root id, got %q", id) + } + if !reflect.DeepEqual(uploaded, []string{"b", "a"}) { + t.Fatalf("unexpected uploaded files: %#v", uploaded) + } + if len(api.deletedObjects) != 0 { + t.Fatalf("did not expect cleanup after success, got %#v", api.deletedObjects) + } +} + +func TestCollectDirectoryEntriesSkipsHiddenTrees(t *testing.T) { + root := t.TempDir() + if err := os.MkdirAll(filepath.Join(root, ".hidden"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(root, ".hidden", "secret.txt"), []byte("secret"), 0o644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(root, ".hidden-file"), []byte("secret"), 0o644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(root, "visible.txt"), []byte("visible"), 0o644); err != nil { + t.Fatal(err) + } + entries, err := collectDirectoryEntries(root) + if err != nil { + t.Fatal(err) + } + if len(entries) != 1 || entries[0].name != "visible.txt" { + t.Fatalf("expected only visible file, got %#v", entries) + } +} + +func TestUploadObjectSlicesTailPartAndMissingETag(t *testing.T) { + var contentLengths []string + var bodies []string + uploadServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, err := io.ReadAll(r.Body) + if err != nil { + t.Fatal(err) + } + contentLengths = append(contentLengths, r.Header.Get("Content-Length")) + bodies = append(bodies, string(body)) + if len(bodies) == 1 { + w.Header().Set("ETag", `"etag-1"`) + } + w.WriteHeader(http.StatusOK) + })) + defer uploadServer.Close() + + path := writeTempFile(t, "hello") + uploader := NewUploader(&recordingAPI{}, nil) + err := uploader.uploadObjectSlices(context.Background(), slog.New(slog.NewTextHandler(io.Discard, nil)), clientTaskWithUploadToken("task-1", "uploading"), client.ObjectDraft{ + ID: "object-1", + Upload: &client.ObjectUploadInstructions{ + SessionID: "session", + PartSize: 3, + URLs: []string{uploadServer.URL, uploadServer.URL}, + }, + }, path, 5, &uploadProgress{totalBytes: 5, lastAt: time.Now()}) + if err == nil || !strings.Contains(err.Error(), "missing ETag") { + t.Fatalf("expected missing etag on tail part, got %v", err) + } + if !reflect.DeepEqual(contentLengths, []string{"3", "2"}) || !reflect.DeepEqual(bodies, []string{"hel", "lo"}) { + t.Fatalf("unexpected multipart reads: lengths=%v bodies=%v", contentLengths, bodies) + } +} + +func TestUploadProgressReaderReturnsProgressError(t *testing.T) { + reader := &uploadProgressReader{ + reader: strings.NewReader("payload"), + progress: func(int64) error { + return errors.New("progress failed") + }, + } + buf := make([]byte, 3) + n, err := reader.Read(buf) + if n != 3 || err == nil || !strings.Contains(err.Error(), "progress failed") { + t.Fatalf("expected progress error after read, n=%d err=%v", n, err) + } +} + +func TestUploadAndCompleteHandlesCanceledUploadStates(t *testing.T) { + cases := []struct { + name string + cancel func(context.CancelCauseFunc) + wantStatus string + wantState string + }{ + {name: "paused", cancel: func(cancel context.CancelCauseFunc) { cancel(errTaskPausing) }, wantStatus: "paused"}, + {name: "canceled", cancel: func(cancel context.CancelCauseFunc) { cancel(errTaskCanceling) }, wantStatus: "canceled"}, + {name: "suspended", cancel: func(cancel context.CancelCauseFunc) { cancel(errTaskSuspended) }}, + {name: "interrupted", cancel: func(cancel context.CancelCauseFunc) { cancel(context.Canceled) }, wantStatus: "interrupted", wantState: "Interrupted because the downloader stopped"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + api := &recordingAPI{} + runner := NewTaskRunnerWithAPI(config.Config{}, api) + runner.logger = slog.New(slog.NewTextHandler(io.Discard, nil)) + ctx, cancel := context.WithCancelCause(context.Background()) + tc.cancel(cancel) + + runner.uploadAndComplete(ctx, runner.logger, clientTaskWithUploadToken("task-1", "downloading"), Result{ + Path: writeTempFile(t, "payload"), + Name: "payload.bin", + Size: 7, + }, nil) + + if tc.wantStatus == "" { + if len(api.patches) != 0 { + t.Fatalf("expected no status mutation, got %#v", api.patches) + } + return + } + last := api.patches[len(api.patches)-1] + if last.State() != tc.wantStatus { + t.Fatalf("expected status %q, got patch %#v", tc.wantStatus, last) + } + if tc.wantState != "" && (last.Runtime == nil || last.Runtime.Message != tc.wantState) { + t.Fatalf("expected interrupted runtime, got %#v", last.Runtime) + } + }) + } +} + +func TestUploadAndCompleteSuccessWithNilRuntimeCleansLocalFile(t *testing.T) { + var uploaded string + uploadServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, err := io.ReadAll(r.Body) + if err != nil { + t.Fatal(err) + } + uploaded = string(body) + w.Header().Set("ETag", `"etag"`) + w.WriteHeader(http.StatusOK) + })) + defer uploadServer.Close() + api := &recordingAPI{ + createObjectDraft: client.ObjectDraft{ + ID: "object-1", + Name: "payload.bin", + Upload: &client.ObjectUploadInstructions{SessionID: "session", PartSize: 1024, URLs: []string{uploadServer.URL}}, + }, + } + runner := NewTaskRunnerWithAPI(config.Config{}, api) + runner.logger = slog.New(slog.NewTextHandler(io.Discard, nil)) + path := writeTempFile(t, "payload") + + runner.uploadAndComplete(context.Background(), runner.logger, clientTaskWithUploadToken("task-1", "downloading"), Result{ + Path: path, + Name: "payload.bin", + Size: int64(len("payload")), + }, nil) + + if uploaded != "payload" { + t.Fatalf("expected upload body, got %q", uploaded) + } + last := api.patches[len(api.patches)-1] + if last.State() != "completed" || last.Runtime == nil || last.Runtime.Phase != "completed" { + t.Fatalf("expected completed patch, got %#v", last) + } + if _, err := os.Stat(path); !os.IsNotExist(err) { + t.Fatalf("expected local file cleanup, got %v", err) + } +} + +func TestSmallRuntimeHelpers(t *testing.T) { + total := int64(10) + detail := withDownloadRuntime(nil, 5, &total, 1) + if detail == nil || detail.Progress == nil || detail.Progress.Download.Bytes != 5 { + t.Fatalf("expected runtime progress, got %#v", detail) + } + bytes, gotTotal := downloadCheckpoint(detail) + if bytes != 5 || gotTotal == nil || *gotTotal != total { + t.Fatalf("unexpected checkpoint: %d %#v", bytes, gotTotal) + } + if optionalTime(time.Time{}) != nil { + t.Fatal("expected zero time to map to nil") + } +} + +func TestSeedManagerFallbackAccessorsAndCleanupTask(t *testing.T) { + manager := NewSeedManager(config.Config{}, &recordingAPI{}, nil, nil, nil, nil) + if manager.log() == nil { + t.Fatal("expected default logger") + } + if manager.manager() != nil { + t.Fatal("expected nil manager") + } + + cleaned := false + manager.retainedSeeds = []retainedSeed{{ + taskID: "task-1", + engine: "aria2", + seedID: "seed-1", + cleanup: func(context.Context) error { + cleaned = true + return nil + }, + }} + manager.CleanupTask(context.Background(), "task-1", "test") + if !cleaned { + t.Fatal("expected retained seed cleanup") + } + if len(manager.retainedSeedSnapshot()) != 0 { + t.Fatalf("expected seed removed, got %#v", manager.retainedSeedSnapshot()) + } +} + +func TestSeedManagerRestoreBranches(t *testing.T) { + stateDir := t.TempDir() + manager := NewTaskRunnerWithAPI(config.Config{SeedEnabled: true, StateDir: stateDir}, &recordingAPI{}) + manager.downloader = NewManagerWithDownloader(testDownloader{name: "http", sourceType: []string{"http"}}) + manager.seeds.Restore(context.Background()) + + expiredPath := t.TempDir() + if err := saveSeedLedger(stateDir, seedLedger{Seeds: []seedLedgerEntry{{ + TaskID: "expired", + Engine: "aria2", + SeedID: "gid", + Path: expiredPath, + ExpiresAt: time.Now().Add(-time.Second), + }}}); err != nil { + t.Fatal(err) + } + manager.seeds.Restore(context.Background()) + if _, err := os.Stat(expiredPath); !os.IsNotExist(err) { + t.Fatalf("expected expired seed path removed, got %v", err) + } + + existingPath := t.TempDir() + if err := saveSeedLedger(stateDir, seedLedger{Seeds: []seedLedgerEntry{{ + TaskID: "pending", + Engine: "aria2", + SeedID: "gid", + Path: existingPath, + ExpiresAt: time.Now().Add(time.Hour), + }}}); err != nil { + t.Fatal(err) + } + engine := &recordingEngine{name: "aria2"} + manager.downloader = NewManagerWithDownloader(engine) + manager.seeds.Restore(context.Background()) + ledger, err := loadSeedLedger(stateDir) + if err != nil { + t.Fatal(err) + } + if len(ledger.Seeds) != 1 || ledger.Seeds[0].TaskID != "pending" { + t.Fatalf("expected pending seed to be kept, got %#v", ledger.Seeds) + } + + engine.restoreErr = errors.New("restore failed") + manager.seeds.Restore(context.Background()) +} + +func TestSeedManagerReportAndReconcileBranches(t *testing.T) { + api := &recordingAPI{localResultErr: errors.New("local result list failed")} + runner := NewTaskRunnerWithAPI(config.Config{SeedEnabled: true}, api) + runner.downloader = NewManagerWithDownloader(&recordingEngine{listSeedsErr: errors.New("list failed")}) + runner.seeds.Reconcile(context.Background()) + + runner.downloader = NewManagerWithDownloader(testDownloader{name: "http", sourceType: []string{"http"}}) + runner.seeds.Reconcile(context.Background()) + + runner.seeds.retainedSeeds = []retainedSeed{ + { + taskID: "nil-runtime", + engine: "aria2", + seedID: "seed-1", + size: 10, + snapshot: func(context.Context) (SeedSnapshot, error) { + return SeedSnapshot{}, nil + }, + cleanup: func(context.Context) error { return nil }, + }, + { + taskID: "snapshot-error", + engine: "aria2", + seedID: "seed-2", + size: 10, + snapshot: func(context.Context) (SeedSnapshot, error) { + return SeedSnapshot{}, errors.New("temporary snapshot failure") + }, + cleanup: func(context.Context) error { return nil }, + }, + } + runner.seeds.Report(context.Background()) + runner.seeds.ReportStopped(context.Background()) +} + +type lifecycleDownloader struct { + name string + sourceTypes []string + startErr error + checkErr error + stopErr error + stopCalls int +} + +func (d *lifecycleDownloader) Name() string { return d.name } + +func (d *lifecycleDownloader) Capabilities() Capabilities { + return Capabilities{SourceTypes: d.sourceTypes} +} + +func (d *lifecycleDownloader) Start(context.Context) error { return d.startErr } + +func (d *lifecycleDownloader) Stop(context.Context) error { + d.stopCalls++ + return d.stopErr +} + +func (d *lifecycleDownloader) Check(context.Context) error { return d.checkErr } + +func (d *lifecycleDownloader) InspectTask(context.Context, DownloadTask) (TaskSnapshot, bool, error) { + return TaskSnapshot{}, false, nil +} + +func (d *lifecycleDownloader) Download(context.Context, DownloadTask, ProgressReporter) (Result, error) { + return Result{}, nil +} + +func registerHappyPathDownloader(t *testing.T, payload string) { + t.Helper() + original := registeredDownloaders + registeredDownloaders = nil + Register("http", true, func(Config) bool { return true }, func(cfg Config) (Downloader, error) { + return &happyPathDownloader{dir: cfg.DownloadDir, payload: payload}, nil + }) + t.Cleanup(func() { registeredDownloaders = original }) +} + +type happyPathDownloader struct { + dir string + payload string +} + +func (d *happyPathDownloader) Name() string { return "http" } + +func (d *happyPathDownloader) Capabilities() Capabilities { + return Capabilities{SourceTypes: []string{"http"}} +} + +func (d *happyPathDownloader) Start(context.Context) error { return nil } +func (d *happyPathDownloader) Stop(context.Context) error { return nil } +func (d *happyPathDownloader) Check(context.Context) error { return nil } + +func (d *happyPathDownloader) InspectTask(context.Context, DownloadTask) (TaskSnapshot, bool, error) { + return TaskSnapshot{}, false, nil +} + +func (d *happyPathDownloader) Download(_ context.Context, task DownloadTask, progress ProgressReporter) (Result, error) { + taskDir := filepath.Join(d.dir, task.ID) + if err := os.MkdirAll(taskDir, 0o755); err != nil { + return Result{}, err + } + path := filepath.Join(taskDir, task.Name()) + if err := os.WriteFile(path, []byte(d.payload), 0o644); err != nil { + return Result{}, err + } + size := int64(len(d.payload)) + if progress != nil { + if err := progress(ProgressUpdate{Downloaded: size, Total: &size, Runtime: &TaskRuntime{Engine: "http", Phase: "downloading"}}); err != nil { + return Result{}, err + } + } + return Result{Path: path, Name: task.Name(), Size: size}, nil +} + +type happyPathAPI struct { + mu sync.Mutex + task client.DownloadTask + draft client.ObjectDraft + onCompleted context.CancelFunc + assigned bool + completed bool + completedObjectID string +} + +func (a *happyPathAPI) Heartbeat(context.Context, client.Heartbeat) (client.HeartbeatResult, error) { + a.mu.Lock() + defer a.mu.Unlock() + if a.assigned { + return client.HeartbeatResult{NextPollAfterSeconds: 1}, nil + } + a.assigned = true + return client.HeartbeatResult{Assignments: []client.DownloadTask{a.task}, NextPollAfterSeconds: 1}, nil +} + +func (a *happyPathAPI) AssignedTasks(context.Context) ([]client.DownloadTask, error) { + return nil, nil +} + +func (a *happyPathAPI) LocalResultTasks(context.Context) ([]client.DownloadTask, error) { + return nil, nil +} + +func (a *happyPathAPI) SeedingTasks(context.Context) ([]client.DownloadTask, error) { + return nil, nil +} + +func (a *happyPathAPI) UpdateTask(_ context.Context, id string, patch client.TaskPatch) (client.DownloadTask, error) { + a.mu.Lock() + defer a.mu.Unlock() + task := a.task + task.ID = id + task = applyTaskPatch(task, patch) + if patch.ResultObjectID != nil { + a.completed = true + a.completedObjectID = *patch.ResultObjectID + if a.onCompleted != nil { + a.onCompleted() + } + } + a.task = task + return task, nil +} + +func (a *happyPathAPI) CreateFolder(context.Context, string, string, string) (client.ObjectDraft, error) { + return client.ObjectDraft{}, errors.New("unexpected folder creation") +} + +func (a *happyPathAPI) CreateObject(context.Context, string, string, int64, string) (client.ObjectDraft, error) { + return a.draft, nil +} + +func (a *happyPathAPI) CompleteObjectUpload(context.Context, string, string, string, []client.CompletedObjectUploadPart) error { + return nil +} + +func (a *happyPathAPI) AbortObjectUploadSession(context.Context, string, string, string) error { + return nil +} + +func (a *happyPathAPI) DeleteObject(context.Context, string, string) error { + return nil +} + +func ptrInt64(value int64) *int64 { + return &value +} + +func mustPanic(t *testing.T, fn func()) { + t.Helper() + defer func() { + if recover() == nil { + t.Fatal("expected panic") + } + }() + fn() +} diff --git a/cmd/internal/downloader/manager.go b/cmd/internal/downloader/manager.go new file mode 100644 index 00000000..b5283610 --- /dev/null +++ b/cmd/internal/downloader/manager.go @@ -0,0 +1,473 @@ +package downloader + +import ( + "context" + "errors" + "fmt" + "log/slog" + "sort" + "strings" + "sync" + "time" + + "github.com/saltbo/zpan/internal/config" + "github.com/saltbo/zpan/pkg/geoip" +) + +const defaultBTDownloader = "aria2" + +type Manager struct { + cfg config.Config + geoIP geoip.Resolver + logger *slog.Logger + downloaders []Downloader + started bool + mu sync.Mutex +} + +func NewManager(cfg config.Config, geoIP geoip.Resolver, logger *slog.Logger) *Manager { + if logger == nil { + logger = slog.Default() + } + return &Manager{cfg: cfg, geoIP: geoIP, logger: logger} +} + +func NewManagerWithDownloader(downloader Downloader) *Manager { + return NewManagerWithDownloaders(downloader) +} + +func NewManagerWithDownloaders(downloaders ...Downloader) *Manager { + return &Manager{downloaders: append([]Downloader(nil), downloaders...), logger: slog.Default(), started: true} +} + +func (m *Manager) Start(ctx context.Context, onDownloaderExit func(error)) error { + if m == nil { + return errors.New("downloader manager is nil") + } + m.mu.Lock() + if m.started { + m.mu.Unlock() + return nil + } + m.mu.Unlock() + + downloaders, err := m.resolveDownloaders(ctx) + if err != nil { + return err + } + if len(downloaders) == 0 { + return errors.New("no downloader is available") + } + + m.mu.Lock() + m.downloaders = downloaders + m.started = true + m.mu.Unlock() + for _, downloader := range downloaders { + go m.watchDownloader(ctx, downloader, onDownloaderExit) + } + m.logger.Info("downloaders started", "downloaders", m.Name(), "capabilities", m.Capabilities()) + return nil +} + +func (m *Manager) Stop(ctx context.Context) { + if m == nil { + return + } + m.mu.Lock() + m.started = false + m.mu.Unlock() + for _, downloader := range m.currentDownloaders() { + stopCtx, cancel := context.WithTimeout(ctx, 10*time.Second) + if err := downloader.Stop(stopCtx); err != nil { + m.logger.Warn("failed to stop downloader", "downloader", downloader.Name(), "error", err) + } + cancel() + } +} + +func (m *Manager) Name() string { + if m == nil { + return "auto" + } + downloaders := m.currentDownloaders() + if len(downloaders) == 0 { + if m.cfg.Engine != "" { + return m.cfg.Engine + } + return "auto" + } + names := make([]string, 0, len(downloaders)) + var fallbackName string + for _, downloader := range downloaders { + if isHTTPDownloader(downloader) { + fallbackName = downloader.Name() + continue + } + names = append(names, downloader.Name()) + } + if len(names) > 0 { + return strings.Join(names, ",") + } + if fallbackName != "" { + return fallbackName + } + return "auto" +} + +func (m *Manager) Capabilities() []string { + downloaders := m.currentDownloaders() + if len(downloaders) == 0 { + return []string{"http"} + } + seen := map[string]struct{}{} + var out []string + for _, downloader := range downloaders { + for _, sourceType := range downloader.Capabilities().SourceTypes { + if _, ok := seen[sourceType]; ok { + continue + } + seen[sourceType] = struct{}{} + out = append(out, sourceType) + } + } + sort.Strings(out) + return out +} + +func (m *Manager) Download(ctx context.Context, task DownloadTask, progress ProgressReporter) (Result, error) { + downloader, err := m.selectDownloader(task) + if err != nil { + return Result{}, err + } + return downloader.Download(ctx, task, progress) +} + +func (m *Manager) InspectTask(ctx context.Context, task DownloadTask) (TaskSnapshot, bool, error) { + downloader, err := m.selectDownloader(task) + if err != nil { + return TaskSnapshot{}, false, err + } + return downloader.InspectTask(ctx, task) +} + +func (m *Manager) ResetTask(ctx context.Context, task DownloadTask) error { + downloader, err := m.selectDownloader(task) + if err != nil { + return err + } + resetter, ok := downloader.(TaskResetter) + if !ok { + return fmt.Errorf("downloader %s does not support task reset", downloader.Name()) + } + return resetter.ResetTask(ctx, task) +} + +func (m *Manager) RestoreSeed(ctx context.Context, ref SeedRef) (*Seed, bool, error) { + if err := m.ready(); err != nil { + return nil, false, err + } + for _, downloader := range m.currentDownloaders() { + restorer, ok := downloader.(SeedRestorer) + if !ok { + continue + } + if ref.Engine != "" && downloader.Name() != ref.Engine { + continue + } + seed, err := restorer.RestoreSeed(ctx, ref) + return seed, true, err + } + return nil, false, nil +} + +func (m *Manager) ListSeeds(ctx context.Context) ([]Seed, bool, error) { + if err := m.ready(); err != nil { + return nil, false, err + } + var out []Seed + supported := false + for _, downloader := range m.currentDownloaders() { + lister, ok := downloader.(SeedLister) + if !ok { + continue + } + supported = true + seeds, err := lister.ListSeeds(ctx) + if err != nil { + return nil, true, err + } + out = append(out, seeds...) + } + return out, supported, nil +} + +func (m *Manager) ready() error { + if m == nil || len(m.currentDownloaders()) == 0 { + return errors.New("downloader manager is not started") + } + return nil +} + +func (m *Manager) selectDownloader(task DownloadTask) (Downloader, error) { + if err := m.ready(); err != nil { + return nil, err + } + sourceType := task.SourceType() + var candidates []Downloader + for _, downloader := range m.currentDownloaders() { + if supportsSourceType(downloader.Capabilities(), sourceType) { + candidates = append(candidates, downloader) + } + } + if len(candidates) == 0 { + return nil, fmt.Errorf("no downloader supports source type %q", sourceType) + } + return candidates[0], nil +} + +func supportsSourceType(capabilities Capabilities, sourceType string) bool { + for _, supported := range capabilities.SourceTypes { + if supported == sourceType { + return true + } + } + return false +} + +func (m *Manager) currentDownloaders() []Downloader { + if m == nil { + return nil + } + m.mu.Lock() + defer m.mu.Unlock() + return append([]Downloader(nil), m.downloaders...) +} + +func (m *Manager) resolveDownloaders(ctx context.Context) ([]Downloader, error) { + cfg := m.downloaderConfig() + var entries []registration + btEntry, hasBT, err := m.btRegistration(cfg) + if err != nil { + return nil, err + } + if hasBT { + entries = append(entries, btEntry) + } + httpEntry, err := fallbackRegistration() + if err != nil { + return nil, err + } + entries = append(entries, httpEntry) + + started := make([]Downloader, 0, len(entries)) + for _, entry := range entries { + downloader, err := entry.new(cfg) + if err != nil { + m.stopStartedDownloaders(ctx, started) + return nil, err + } + if err := m.startDownloader(ctx, downloader, true); err != nil { + m.stopStartedDownloaders(ctx, started) + return nil, err + } + started = append(started, downloader) + } + return started, nil +} + +func (m *Manager) startDownloader(ctx context.Context, downloader Downloader, required bool) error { + m.logger.Info("starting downloader", "downloader", downloader.Name()) + if err := downloader.Start(ctx); err != nil { + if required { + return fmt.Errorf("start downloader %q: %w", downloader.Name(), err) + } + return err + } + if err := waitForDownloader(ctx, downloader); err != nil { + downloader.Stop(context.Background()) + if required { + return fmt.Errorf("downloader %q is not available: %w", downloader.Name(), err) + } + return err + } + m.logger.Info("downloader started", "downloader", downloader.Name()) + return nil +} + +func (m *Manager) btRegistration(cfg Config) (registration, bool, error) { + engine := strings.ToLower(strings.TrimSpace(cfg.Engine)) + switch engine { + case "", "auto": + configured := configuredExternalRegistrations(cfg) + if len(configured) > 1 { + return registration{}, false, fmt.Errorf("multiple BT downloaders are configured; set downloader.engine to one of: %s", strings.Join(registrationNames(configured), ", ")) + } + if len(configured) == 1 { + return configured[0], true, nil + } + entry, ok := registrationByName(defaultBTDownloader) + if !ok { + return registration{}, false, fmt.Errorf("default BT downloader %q is not registered", defaultBTDownloader) + } + return entry, true, nil + case "http": + return registration{}, false, nil + default: + entry, ok := registrationByName(engine) + if !ok || entry.fallback { + return registration{}, false, fmt.Errorf("unsupported BT downloader %q; expected auto, http, or one of: %s", cfg.Engine, strings.Join(externalDownloaderNames(), ", ")) + } + return entry, true, nil + } +} + +func configuredExternalRegistrations(cfg Config) []registration { + var out []registration + for _, entry := range externalRegistrations() { + if entry.configured(cfg) { + out = append(out, entry) + } + } + return out +} + +func fallbackRegistration() (registration, error) { + for _, entry := range registrations() { + if entry.fallback { + return entry, nil + } + } + return registration{}, errors.New("http downloader is not registered") +} + +func registrationByName(name string) (registration, bool) { + for _, entry := range registrations() { + if entry.name == name { + return entry, true + } + } + return registration{}, false +} + +func registrationNames(entries []registration) []string { + names := make([]string, 0, len(entries)) + for _, entry := range entries { + names = append(names, entry.name) + } + return names +} + +func externalDownloaderNames() []string { + return registrationNames(externalRegistrations()) +} + +func (m *Manager) downloaderConfig() Config { + return Config{ + Engine: m.cfg.Engine, + DownloadDir: m.cfg.DownloadDir, + StateDir: m.cfg.StateDir, + BTListenPort: m.cfg.BTListenPort, + MaxConcurrentDownloads: aria2MaxConcurrentDownloads(m.cfg), + SeedEnabled: m.cfg.SeedEnabled, + SeedDuration: m.cfg.SeedDuration, + SeedRatio: m.cfg.SeedRatio, + Aria2: Aria2Config{ + URL: m.cfg.Aria2URL, + Secret: m.cfg.Aria2Secret, + Configured: m.cfg.Aria2Configured, + }, + QBittorrent: QBittorrentConfig{ + URL: m.cfg.QBittorrentURL, + Username: m.cfg.QBittorrentUser, + Password: m.cfg.QBittorrentPass, + Configured: m.cfg.QBittorrentConfigured, + }, + GeoIP: m.geoIP, + } +} + +func externalRegistrations() []registration { + var out []registration + for _, entry := range registrations() { + if entry.fallback { + continue + } + out = append(out, entry) + } + return out +} + +func (m *Manager) stopStartedDownloaders(ctx context.Context, downloaders []Downloader) { + for _, downloader := range downloaders { + stopCtx, cancel := context.WithTimeout(ctx, 10*time.Second) + if err := downloader.Stop(stopCtx); err != nil { + m.logger.Warn("failed to stop downloader after startup error", "downloader", downloader.Name(), "error", err) + } + cancel() + } +} + +func isHTTPDownloader(downloader Downloader) bool { + return downloader.Name() == "http" +} + +func aria2MaxConcurrentDownloads(cfg config.Config) int { + limit := cfg.MaxConcurrentTasks + if cfg.SeedEnabled { + limit += cfg.SeedMaxConcurrent + } + return limit +} + +func waitForDownloader(ctx context.Context, downloader Downloader) error { + deadline := time.Now().Add(8 * time.Second) + var lastErr error + for time.Now().Before(deadline) { + checkCtx, cancel := context.WithTimeout(ctx, time.Second) + err := downloader.Check(checkCtx) + cancel() + if err == nil { + return nil + } + lastErr = err + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(500 * time.Millisecond): + } + } + return lastErr +} + +func (m *Manager) watchDownloader(ctx context.Context, downloader Downloader, onDownloaderExit func(error)) { + if onDownloaderExit == nil { + return + } + ticker := time.NewTicker(5 * time.Second) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + if !m.isStarted() { + return + } + checkCtx, cancel := context.WithTimeout(ctx, 2*time.Second) + err := downloader.Check(checkCtx) + cancel() + if err == nil { + continue + } + onDownloaderExit(fmt.Errorf("%s health check failed: %w", downloader.Name(), err)) + return + } + } +} + +func (m *Manager) isStarted() bool { + m.mu.Lock() + defer m.mu.Unlock() + return m.started +} diff --git a/cmd/internal/downloader/registry.go b/cmd/internal/downloader/registry.go new file mode 100644 index 00000000..7f92bcad --- /dev/null +++ b/cmd/internal/downloader/registry.go @@ -0,0 +1,39 @@ +package downloader + +type registration struct { + name string + fallback bool + configured func(Config) bool + new func(Config) (Downloader, error) +} + +var registeredDownloaders []registration + +func Register( + name string, + fallback bool, + configured func(Config) bool, + new func(Config) (Downloader, error), +) { + if name == "" { + panic("downloader: register downloader with empty name") + } + if configured == nil { + configured = func(Config) bool { return false } + } + if new == nil { + panic("downloader: register downloader with nil constructor") + } + entry := registration{name: name, fallback: fallback, configured: configured, new: new} + for i, existing := range registeredDownloaders { + if existing.name == name { + registeredDownloaders[i] = entry + return + } + } + registeredDownloaders = append(registeredDownloaders, entry) +} + +func registrations() []registration { + return append([]registration(nil), registeredDownloaders...) +} diff --git a/cmd/internal/downloader/registry_test.go b/cmd/internal/downloader/registry_test.go new file mode 100644 index 00000000..512de73e --- /dev/null +++ b/cmd/internal/downloader/registry_test.go @@ -0,0 +1,201 @@ +package downloader + +import ( + "context" + "strings" + "testing" + + "github.com/saltbo/zpan/internal/config" +) + +type testRegistration struct { + name string + fallback bool + configured bool +} + +type testDownloader struct { + name string + sourceType []string +} + +type seedRestoringTestDownloader struct { + testDownloader + restoreCalls int +} + +func (d testDownloader) Name() string { + return d.name +} + +func (d testDownloader) Capabilities() Capabilities { + sourceTypes := d.sourceType + if len(sourceTypes) == 0 { + sourceTypes = []string{"http", "magnet", "torrent", "torrent_url"} + } + return Capabilities{SourceTypes: sourceTypes} +} + +func (d testDownloader) Start(context.Context) error { + return nil +} + +func (d testDownloader) Stop(context.Context) error { + return nil +} + +func (d testDownloader) Check(context.Context) error { + return nil +} + +func (d testDownloader) InspectTask(context.Context, DownloadTask) (TaskSnapshot, bool, error) { + return TaskSnapshot{}, false, nil +} + +func (d testDownloader) Download(context.Context, DownloadTask, ProgressReporter) (Result, error) { + return Result{Name: d.name}, nil +} + +func (d *seedRestoringTestDownloader) RestoreSeed(context.Context, SeedRef) (*Seed, error) { + d.restoreCalls++ + return &Seed{Engine: d.name, ID: d.name + "-seed"}, nil +} + +func TestAutoManagerStartsOneConfiguredBTAndHTTPDownloaders(t *testing.T) { + registerTestDownloaders(t, + testRegistration{name: "aria2"}, + testRegistration{name: "qbittorrent", configured: true}, + testRegistration{name: "http", fallback: true}, + ) + + manager := NewManager(config.Config{}, nil, nil) + if err := manager.Start(context.Background(), nil); err != nil { + t.Fatal(err) + } + downloaders := manager.currentDownloaders() + if len(downloaders) != 2 { + t.Fatalf("expected one BT downloader plus HTTP, got %d", len(downloaders)) + } + if manager.Name() != "qbittorrent" { + t.Fatalf("expected selected BT name, got %q", manager.Name()) + } +} + +func TestManagerRejectsUnknownBTDownloader(t *testing.T) { + registerTestDownloaders(t, testRegistration{name: "aria2"}, testRegistration{name: "http", fallback: true}) + + manager := NewManager(config.Config{Engine: "bad-engine"}, nil, nil) + + err := manager.Start(context.Background(), nil) + if err == nil { + t.Fatal("expected unsupported downloader error") + } + if !strings.Contains(err.Error(), "bad-engine") { + t.Fatalf("expected error to mention configured downloader, got %v", err) + } +} + +func TestHTTPOnlyManagerRejectsBTTask(t *testing.T) { + manager := NewManagerWithDownloaders(testDownloader{name: "http", sourceType: []string{"http"}}) + _, err := manager.Download(context.Background(), DownloadTask{Source: Source{Type: "magnet"}}, nil) + if err == nil || !strings.Contains(err.Error(), "no downloader supports") { + t.Fatalf("expected unsupported source type error, got %v", err) + } +} + +func TestAutoManagerRejectsMultipleConfiguredBTDownloaders(t *testing.T) { + registerTestDownloaders(t, + testRegistration{name: "aria2", configured: true}, + testRegistration{name: "qbittorrent", configured: true}, + testRegistration{name: "http", fallback: true}, + ) + + manager := NewManager(config.Config{}, nil, nil) + err := manager.Start(context.Background(), nil) + if err == nil || !strings.Contains(err.Error(), "multiple BT downloaders") { + t.Fatalf("expected multiple BT downloader error, got %v", err) + } +} + +func TestManagerSelectsDownloaderBySourceType(t *testing.T) { + manager := NewManagerWithDownloaders( + testDownloader{name: "aria2", sourceType: []string{"magnet", "torrent_url"}}, + testDownloader{name: "http", sourceType: []string{"http"}}, + ) + + result, err := manager.Download(context.Background(), DownloadTask{Source: Source{Type: "http"}}, nil) + if err != nil { + t.Fatal(err) + } + if result.Name != "http" { + t.Fatalf("expected http downloader, got %q", result.Name) + } + result, err = manager.Download(context.Background(), DownloadTask{Source: Source{Type: "magnet"}}, nil) + if err != nil { + t.Fatal(err) + } + if result.Name != "aria2" { + t.Fatalf("expected aria2 downloader, got %q", result.Name) + } +} + +func TestManagerRestoreSeedRequiresMatchingEngine(t *testing.T) { + aria2 := &seedRestoringTestDownloader{testDownloader: testDownloader{name: "aria2"}} + qbit := &seedRestoringTestDownloader{testDownloader: testDownloader{name: "qbittorrent"}} + manager := NewManagerWithDownloaders(aria2, qbit) + + seed, supported, err := manager.RestoreSeed(context.Background(), SeedRef{Engine: "qbittorrent", ID: "seed-1"}) + if err != nil { + t.Fatal(err) + } + if !supported { + t.Fatal("expected matching engine to support restore") + } + if seed == nil || seed.Engine != "qbittorrent" { + t.Fatalf("expected qbittorrent seed, got %#v", seed) + } + if aria2.restoreCalls != 0 { + t.Fatalf("expected aria2 not to restore qbit seed, got %d calls", aria2.restoreCalls) + } + if qbit.restoreCalls != 1 { + t.Fatalf("expected qbit restore once, got %d", qbit.restoreCalls) + } +} + +func TestManagerRestoreSeedDoesNotFallbackWhenEngineIsMissing(t *testing.T) { + aria2 := &seedRestoringTestDownloader{testDownloader: testDownloader{name: "aria2"}} + manager := NewManagerWithDownloaders(aria2) + + seed, supported, err := manager.RestoreSeed(context.Background(), SeedRef{Engine: "qbittorrent", ID: "seed-1"}) + if err != nil { + t.Fatal(err) + } + if supported { + t.Fatalf("expected missing explicit engine to be unsupported, got seed %#v", seed) + } + if seed != nil { + t.Fatalf("expected no seed, got %#v", seed) + } + if aria2.restoreCalls != 0 { + t.Fatalf("expected aria2 not to restore qbit seed, got %d calls", aria2.restoreCalls) + } +} + +func registerTestDownloaders(t *testing.T, entries ...testRegistration) { + t.Helper() + original := registeredDownloaders + registeredDownloaders = nil + for _, entry := range entries { + name := entry.name + configured := entry.configured + Register( + entry.name, + entry.fallback, + func(Config) bool { return configured }, + func(Config) (Downloader, error) { return testDownloader{name: name}, nil }, + ) + } + t.Cleanup(func() { + registeredDownloaders = original + }) +} diff --git a/cmd/internal/worker/seed_ledger.go b/cmd/internal/downloader/seed_ledger.go similarity index 98% rename from cmd/internal/worker/seed_ledger.go rename to cmd/internal/downloader/seed_ledger.go index f4747cbb..e9596f36 100644 --- a/cmd/internal/worker/seed_ledger.go +++ b/cmd/internal/downloader/seed_ledger.go @@ -1,4 +1,4 @@ -package worker +package downloader import ( "encoding/json" diff --git a/cmd/internal/worker/seeds.go b/cmd/internal/downloader/seeds.go similarity index 59% rename from cmd/internal/worker/seeds.go rename to cmd/internal/downloader/seeds.go index c45bcd04..b1195038 100644 --- a/cmd/internal/worker/seeds.go +++ b/cmd/internal/downloader/seeds.go @@ -1,4 +1,4 @@ -package worker +package downloader import ( "context" @@ -8,10 +8,12 @@ import ( "sort" "strconv" "strings" + "sync" "time" "github.com/saltbo/zpan/internal/client" - "github.com/saltbo/zpan/internal/engine" + "github.com/saltbo/zpan/internal/config" + "github.com/saltbo/zpan/pkg/system" ) const retainedSeedReportInterval = 5 * time.Second @@ -28,11 +30,68 @@ type retainedSeed struct { retainedAt time.Time expiresAt time.Time reportKey string - snapshot func(context.Context) (engine.SeedSnapshot, error) + snapshot func(context.Context) (SeedSnapshot, error) cleanup func(context.Context) error } -func cleanupDownloadedResult(ctx context.Context, task client.DownloadTask, result engine.Result) error { +type SeedManager struct { + cfg config.Config + api apiClient + logger func() *slog.Logger + downloader func() *Manager + runningTaskIDs func() map[string]struct{} + localResultTaskIDs func(context.Context) (map[string]struct{}, bool) + retainedSeeds []retainedSeed + mu sync.Mutex +} + +func NewSeedManager( + cfg config.Config, + api apiClient, + logger func() *slog.Logger, + downloader func() *Manager, + runningTaskIDs func() map[string]struct{}, + localResultTaskIDs func(context.Context) (map[string]struct{}, bool), +) *SeedManager { + return &SeedManager{ + cfg: cfg, + api: api, + logger: logger, + downloader: downloader, + runningTaskIDs: runningTaskIDs, + localResultTaskIDs: localResultTaskIDs, + } +} + +func (s *SeedManager) log() *slog.Logger { + if s.logger == nil { + return slog.Default() + } + logger := s.logger() + if logger == nil { + return slog.Default() + } + return logger +} + +func (s *SeedManager) manager() *Manager { + if s.downloader == nil { + return nil + } + return s.downloader() +} + +func (s *SeedManager) updateTask(ctx context.Context, id string, patch client.TaskPatch) (client.DownloadTask, error) { + var task client.DownloadTask + err := callAPI(ctx, s.log(), "update task", func(ctx context.Context) error { + var err error + task, err = s.api.UpdateTask(ctx, id, patch) + return err + }) + return task, err +} + +func cleanupDownloadedResult(ctx context.Context, task client.DownloadTask, result Result) error { if result.Seed != nil && result.Seed.Cleanup != nil { return result.Seed.Cleanup(ctx) } @@ -49,8 +108,8 @@ func cleanupDownloadedResult(ctx context.Context, task client.DownloadTask, resu return nil } -func (w *Worker) retainSeed(task client.DownloadTask, result engine.Result, log *slog.Logger) bool { - if !w.cfg.SeedEnabled || result.Seed == nil || result.Seed.Cleanup == nil || result.Seed.Snapshot == nil { +func (s *SeedManager) Retain(ctx context.Context, task client.DownloadTask, result Result, log *slog.Logger) bool { + if !s.cfg.SeedEnabled || result.Seed == nil || result.Seed.Cleanup == nil || result.Seed.Snapshot == nil { return false } now := time.Now() @@ -66,10 +125,12 @@ func (w *Worker) retainSeed(task client.DownloadTask, result engine.Result, log snapshot: result.Seed.Snapshot, cleanup: result.Seed.Cleanup, } - if w.cfg.SeedDuration > 0 { - seed.expiresAt = now.Add(w.cfg.SeedDuration) + if s.cfg.SeedDuration > 0 { + seed.expiresAt = now.Add(s.cfg.SeedDuration) } - if snapshot, err := result.Seed.Snapshot(context.Background()); err == nil && + snapshotCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + if snapshot, err := result.Seed.Snapshot(snapshotCtx); err == nil && snapshot.Runtime != nil && snapshot.Runtime.Seeding != nil && snapshot.Runtime.Seeding.UploadedBytes != nil { @@ -77,11 +138,11 @@ func (w *Worker) retainSeed(task client.DownloadTask, result engine.Result, log } else if err != nil { log.Warn("failed to record retained bt seed upload baseline", "error", err) } - w.mu.Lock() - w.retainedSeeds = append(w.retainedSeeds, seed) - count := len(w.retainedSeeds) - w.mu.Unlock() - if err := w.upsertSeedLedger(seed, result.Size); err != nil { + s.mu.Lock() + s.retainedSeeds = append(s.retainedSeeds, seed) + count := len(s.retainedSeeds) + s.mu.Unlock() + if err := s.upsertSeedLedger(seed, result.Size); err != nil { log.Warn("failed to persist retained bt seed", "error", err) } @@ -95,17 +156,13 @@ func (w *Worker) retainSeed(task client.DownloadTask, result engine.Result, log return true } -func (w *Worker) restoreRetainedSeeds(ctx context.Context) { - if !w.cfg.SeedEnabled || w.cfg.StateDir == "" { +func (s *SeedManager) Restore(ctx context.Context) { + if !s.cfg.SeedEnabled || s.cfg.StateDir == "" { return } - restorer, ok := w.engine.(engine.SeedRestorer) - if !ok { - return - } - ledger, err := loadSeedLedger(w.cfg.StateDir) + ledger, err := loadSeedLedger(s.cfg.StateDir) if err != nil { - w.logger.Warn("failed to load retained seed ledger", "error", err) + s.log().Warn("failed to load retained seed ledger", "error", err) return } if len(ledger.Seeds) == 0 { @@ -115,7 +172,7 @@ func (w *Worker) restoreRetainedSeeds(ctx context.Context) { var kept []seedLedgerEntry now := time.Now() existing := map[string]struct{}{} - for _, seed := range w.retainedSeedSnapshot() { + for _, seed := range s.retainedSeedSnapshot() { existing[seed.taskID] = struct{}{} } for _, entry := range ledger.Seeds { @@ -127,21 +184,24 @@ func (w *Worker) restoreRetainedSeeds(ctx context.Context) { _ = os.RemoveAll(entry.Path) continue } - seed, err := restorer.RestoreSeed(ctx, engine.SeedRef{ + seed, supported, err := s.manager().RestoreSeed(ctx, SeedRef{ TaskID: entry.TaskID, Engine: entry.Engine, ID: entry.SeedID, InfoHash: entry.InfoHash, Path: entry.Path, }) + if !supported { + return + } if err != nil { - w.logger.Warn("failed to restore retained bt seed", "task_id", entry.TaskID, "engine", entry.Engine, "seed_id", entry.SeedID, "error", err) + s.log().Warn("failed to restore retained bt seed", "task_id", entry.TaskID, "engine", entry.Engine, "seed_id", entry.SeedID, "error", err) kept = append(kept, entry) continue } if seed == nil { if _, statErr := os.Stat(entry.Path); statErr == nil { - w.logger.Debug("retained bt seed runtime is not ready; keeping local ledger entry", "task_id", entry.TaskID, "engine", entry.Engine, "path", entry.Path) + s.log().Debug("retained bt seed runtime is not ready; keeping local ledger entry", "task_id", entry.TaskID, "engine", entry.Engine, "path", entry.Path) kept = append(kept, entry) } continue @@ -166,72 +226,73 @@ func (w *Worker) restoreRetainedSeeds(ctx context.Context) { kept = append(kept, entry) } if len(restored) > 0 { - w.mu.Lock() - w.retainedSeeds = append(w.retainedSeeds, restored...) - count := len(w.retainedSeeds) - w.mu.Unlock() - w.logger.Info("restored retained bt seeds", "count", len(restored), "retained_seeds", count) + s.mu.Lock() + s.retainedSeeds = append(s.retainedSeeds, restored...) + count := len(s.retainedSeeds) + s.mu.Unlock() + s.log().Info("restored retained bt seeds", "count", len(restored), "retained_seeds", count) } - if err := saveSeedLedger(w.cfg.StateDir, seedLedger{Seeds: kept}); err != nil { - w.logger.Warn("failed to save retained seed ledger", "error", err) + if err := saveSeedLedger(s.cfg.StateDir, seedLedger{Seeds: kept}); err != nil { + s.log().Warn("failed to save retained seed ledger", "error", err) } } -func (w *Worker) reportRetainedSeeds(ctx context.Context) { - for _, seed := range w.retainedSeedSnapshot() { - log := w.logger.With("task_id", seed.taskID, "engine", seed.engine, "seed_id", seed.seedID) +func (s *SeedManager) Report(ctx context.Context) { + for _, seed := range s.retainedSeedSnapshot() { + log := s.log().With("task_id", seed.taskID, "engine", seed.engine, "seed_id", seed.seedID) snapshot, err := seed.snapshot(ctx) if err != nil { if isMissingRetainedSeedError(err) { - w.cleanupRetainedSeed(ctx, seed, "missing") + s.cleanupRetainedSeed(ctx, seed, "missing") continue } log.Warn("failed to inspect retained bt seed", "error", err) continue } - if snapshot.Runtime == nil { + runtime := zpanRuntime(snapshot.Runtime) + if runtime == nil { continue } - snapshot.Runtime.Phase = "seeding" - snapshot.Runtime.ETASeconds = nil - if snapshot.Runtime.Seeding == nil { - snapshot.Runtime.Seeding = &client.DownloadTaskSeedingRuntime{} + runtime.Phase = "seeding" + runtime.ETASeconds = nil + if runtime.Seeding == nil { + runtime.Seeding = &client.DownloadTaskSeedingRuntime{} } active := true - snapshot.Runtime.Seeding.Active = &active + runtime.Seeding.Active = &active reportKey := retainedSeedReportKey(seed, snapshot, true) if reportKey == seed.reportKey { continue } - snapshot.Runtime.Progress = &client.DownloadTaskProgress{ + runtime.Progress = &client.DownloadTaskProgress{ Download: *transferProgress(snapshot.Downloaded, snapshot.Total, 0), Upload: *transferProgress(seed.size, &seed.size, 0), } - _, err = w.updateTask(ctx, seed.taskID, client.TaskPatch{ + _, err = s.updateTask(ctx, seed.taskID, client.TaskPatch{ Progress: &client.DownloadTaskProgressPatch{ Download: transferProgress(snapshot.Downloaded, snapshot.Total, 0), Upload: transferProgress(seed.size, &seed.size, 0), }, - Runtime: snapshot.Runtime, + Runtime: runtime, }) if err != nil { log.Warn("failed to report retained bt seed", "error", err) continue } - w.markRetainedSeedReported(seed.taskID, reportKey) + s.markRetainedSeedReported(seed.taskID, reportKey) log.Debug("reported retained bt seed", "downloaded_bytes", snapshot.Downloaded, "bps", snapshot.Bps) } } -func (w *Worker) reportRetainedSeedsStopped(ctx context.Context) { - seeds := w.retainedSeedSnapshot() +func (s *SeedManager) ReportStopped(ctx context.Context) { + seeds := s.retainedSeedSnapshot() if len(seeds) == 0 { return } reportCtx, cancel := context.WithTimeout(ctx, 10*time.Second) defer cancel() for _, seed := range seeds { - log := w.logger.With("task_id", seed.taskID, "engine", seed.engine, "seed_id", seed.seedID) + log := s.log().With("task_id", seed.taskID, "engine", seed.engine, "seed_id", seed.seedID) runtime := &client.DownloadTaskRuntime{ Engine: seed.engine, Phase: "completed", @@ -248,7 +309,7 @@ func (w *Worker) reportRetainedSeedsStopped(ctx context.Context) { snapshot, err := seed.snapshot(reportCtx) if err != nil { if isMissingRetainedSeedError(err) { - w.cleanupRetainedSeed(reportCtx, seed, "missing") + s.cleanupRetainedSeed(reportCtx, seed, "missing") continue } log.Warn("failed to inspect retained bt seed before shutdown", "error", err) @@ -257,8 +318,8 @@ func (w *Worker) reportRetainedSeedsStopped(ctx context.Context) { if snapshot.Total != nil { totalPtr = snapshot.Total } - if snapshot.Runtime != nil { - runtime = snapshot.Runtime + if converted := zpanRuntime(snapshot.Runtime); converted != nil { + runtime = converted } } runtime.Phase = "completed" @@ -274,7 +335,7 @@ func (w *Worker) reportRetainedSeedsStopped(ctx context.Context) { Download: *transferProgress(downloaded, totalPtr, 0), Upload: *transferProgress(seed.size, &seed.size, 0), } - _, err = w.updateTask(reportCtx, seed.taskID, client.TaskPatch{ + _, err = s.updateTask(reportCtx, seed.taskID, client.TaskPatch{ Progress: &client.DownloadTaskProgressPatch{ Download: transferProgress(downloaded, totalPtr, 0), Upload: transferProgress(seed.size, &seed.size, 0), @@ -295,8 +356,8 @@ func isMissingRetainedSeedError(err error) bool { return strings.Contains(message, "gid") && strings.Contains(message, "not found") } -func (w *Worker) cleanupRetainedSeeds(ctx context.Context) { - seeds := w.retainedSeedSnapshot() +func (s *SeedManager) Cleanup(ctx context.Context) { + seeds := s.retainedSeedSnapshot() if len(seeds) == 0 { return } @@ -308,14 +369,14 @@ func (w *Worker) cleanupRetainedSeeds(ctx context.Context) { reasons[seed.taskID] = "expired" } } - if w.cfg.SeedRatio > 0 { + if s.cfg.SeedRatio > 0 { for _, seed := range seeds { if reasons[seed.taskID] != "" || seed.downloaded <= 0 { continue } snapshot, err := seed.snapshot(ctx) if err != nil { - w.logger.Warn("failed to inspect retained seed ratio", "task_id", seed.taskID, "path", seed.path, "error", err) + s.log().Warn("failed to inspect retained seed ratio", "task_id", seed.taskID, "path", seed.path, "error", err) continue } if snapshot.Runtime == nil || @@ -324,13 +385,13 @@ func (w *Worker) cleanupRetainedSeeds(ctx context.Context) { continue } uploaded := *snapshot.Runtime.Seeding.UploadedBytes - seed.uploadBase - if uploaded >= int64(float64(seed.downloaded)*w.cfg.SeedRatio) { + if uploaded >= int64(float64(seed.downloaded)*s.cfg.SeedRatio) { reasons[seed.taskID] = "ratio" } } } - if w.cfg.SeedCacheLimit > 0 { + if s.cfg.SeedCacheLimit > 0 { type seedSize struct { seed retainedSeed size int64 @@ -341,9 +402,9 @@ func (w *Worker) cleanupRetainedSeeds(ctx context.Context) { if reasons[seed.taskID] != "" { continue } - size, err := directorySize(seed.path) + size, err := system.DirectorySize(seed.path) if err != nil { - w.logger.Warn("failed to inspect retained seed size", "task_id", seed.taskID, "path", seed.path, "error", err) + s.log().Warn("failed to inspect retained seed size", "task_id", seed.taskID, "path", seed.path, "error", err) continue } total += size @@ -353,7 +414,7 @@ func (w *Worker) cleanupRetainedSeeds(ctx context.Context) { return sized[i].seed.retainedAt.Before(sized[j].seed.retainedAt) }) for _, item := range sized { - if total <= w.cfg.SeedCacheLimit { + if total <= s.cfg.SeedCacheLimit { break } reasons[item.seed.taskID] = "cache_limit" @@ -369,38 +430,38 @@ func (w *Worker) cleanupRetainedSeeds(ctx context.Context) { continue } if protected == nil { - protected, protectedOK = w.localResultTaskIDs(ctx) + protected, protectedOK = s.localResultTaskIDs(ctx) } if !protectedOK { - w.logger.Debug("skipping retained seed cleanup because task state is unavailable", "task_id", seed.taskID, "reason", reason) + s.log().Debug("skipping retained seed cleanup because task state is unavailable", "task_id", seed.taskID, "reason", reason) continue } if _, ok := protected[seed.taskID]; ok { - w.logger.Debug("skipping retained seed cleanup for incomplete task", "task_id", seed.taskID, "reason", reason) + s.log().Debug("skipping retained seed cleanup for incomplete task", "task_id", seed.taskID, "reason", reason) continue } - w.cleanupRetainedSeed(ctx, seed, reason) + s.cleanupRetainedSeed(ctx, seed, reason) } } -func (w *Worker) retainedSeedSnapshot() []retainedSeed { - w.mu.Lock() - defer w.mu.Unlock() - return append([]retainedSeed(nil), w.retainedSeeds...) +func (s *SeedManager) retainedSeedSnapshot() []retainedSeed { + s.mu.Lock() + defer s.mu.Unlock() + return append([]retainedSeed(nil), s.retainedSeeds...) } -func (w *Worker) markRetainedSeedReported(taskID string, reportKey string) { - w.mu.Lock() - defer w.mu.Unlock() - for i := range w.retainedSeeds { - if w.retainedSeeds[i].taskID == taskID { - w.retainedSeeds[i].reportKey = reportKey +func (s *SeedManager) markRetainedSeedReported(taskID string, reportKey string) { + s.mu.Lock() + defer s.mu.Unlock() + for i := range s.retainedSeeds { + if s.retainedSeeds[i].taskID == taskID { + s.retainedSeeds[i].reportKey = reportKey return } } } -func retainedSeedReportKey(seed retainedSeed, snapshot engine.SeedSnapshot, active bool) string { +func retainedSeedReportKey(seed retainedSeed, snapshot SeedSnapshot, active bool) string { total := int64(-1) if snapshot.Total != nil { total = *snapshot.Total @@ -418,65 +479,38 @@ func retainedSeedReportKey(seed retainedSeed, snapshot engine.SeedSnapshot, acti }, ":") } -func (w *Worker) runningTaskIDs() map[string]struct{} { - w.mu.Lock() - defer w.mu.Unlock() - ids := make(map[string]struct{}, len(w.running)) - for id := range w.running { - ids[id] = struct{}{} - } - return ids -} - -// localResultTaskIDs lists tasks whose local completed download may still be -// needed by this downloader, so seed cleanup cannot delete it before retrying -// an upload failure or resuming a paused/suspended upload. -func (w *Worker) localResultTaskIDs(ctx context.Context) (map[string]struct{}, bool) { - ids := map[string]struct{}{} - tasks, err := w.api.LocalResultTasks(ctx) - if err != nil { - w.logger.Warn("failed to list local-result tasks for seed reconciliation", "error", err) - return ids, false - } - for _, task := range tasks { - ids[task.ID] = struct{}{} - } - return ids, true -} - // reconcileEngineSeeds adopts torrents the engine is still seeding but the // worker no longer tracks (orphans left behind by restarts or ledger drift), // so the normal time/ratio/cache cleanup applies to them instead of letting // them seed forever and hold runtime slots. Tasks still mid-flight (running or // already tracked) are skipped so reconciliation never races their cleanup. -func (w *Worker) reconcileEngineSeeds(ctx context.Context) { - if !w.cfg.SeedEnabled { +func (s *SeedManager) Reconcile(ctx context.Context) { + if !s.cfg.SeedEnabled { return } - lister, ok := w.engine.(engine.SeedLister) - if !ok { + seeds, supported, err := s.manager().ListSeeds(ctx) + if !supported { return } - seeds, err := lister.ListSeeds(ctx) if err != nil { - w.logger.Warn("failed to list engine seeds for reconciliation", "error", err) + s.log().Warn("failed to list engine seeds for reconciliation", "error", err) return } if len(seeds) == 0 { return } tracked := map[string]struct{}{} - for _, seed := range w.retainedSeedSnapshot() { + for _, seed := range s.retainedSeedSnapshot() { tracked[seed.taskID] = struct{}{} } - running := w.runningTaskIDs() + running := s.runningTaskIDs() // Tasks still assigned to us (downloading/interrupted/uploading) are owned by // the task loop, which will upload then seed them. The reconciler runs at // startup before the loop marks them running, so without this an // auto-seeding-but-not-yet-uploaded torrent gets adopted as a done seed and // its upload is skipped (file lost when the seed expires). now := time.Now() - candidates := make([]engine.Seed, 0, len(seeds)) + candidates := make([]Seed, 0, len(seeds)) for _, seed := range seeds { taskID := filepath.Base(seed.Path) if taskID == "" || taskID == "." || taskID == string(filepath.Separator) { @@ -493,7 +527,7 @@ func (w *Worker) reconcileEngineSeeds(ctx context.Context) { if len(candidates) == 0 { return } - protected, ok := w.localResultTaskIDs(ctx) + protected, ok := s.localResultTaskIDs(ctx) if !ok { return } @@ -503,9 +537,9 @@ func (w *Worker) reconcileEngineSeeds(ctx context.Context) { if _, ok := protected[taskID]; ok { continue } - size, err := directorySize(seed.Path) + size, err := system.DirectorySize(seed.Path) if err != nil { - w.logger.Warn("failed to size adopted seed", "task_id", taskID, "path", seed.Path, "error", err) + s.log().Warn("failed to size adopted seed", "task_id", taskID, "path", seed.Path, "error", err) continue } adoptedSeed := retainedSeed{ @@ -520,27 +554,27 @@ func (w *Worker) reconcileEngineSeeds(ctx context.Context) { snapshot: seed.Snapshot, cleanup: seed.Cleanup, } - if w.cfg.SeedDuration > 0 { - adoptedSeed.expiresAt = now.Add(w.cfg.SeedDuration) + if s.cfg.SeedDuration > 0 { + adoptedSeed.expiresAt = now.Add(s.cfg.SeedDuration) } adopted = append(adopted, adoptedSeed) tracked[taskID] = struct{}{} - if err := w.upsertSeedLedger(adoptedSeed, size); err != nil { - w.logger.Warn("failed to persist adopted seed", "task_id", taskID, "error", err) + if err := s.upsertSeedLedger(adoptedSeed, size); err != nil { + s.log().Warn("failed to persist adopted seed", "task_id", taskID, "error", err) } } if len(adopted) == 0 { return } - w.mu.Lock() - w.retainedSeeds = append(w.retainedSeeds, adopted...) - count := len(w.retainedSeeds) - w.mu.Unlock() - w.logger.Info("adopted untracked engine seeds for managed expiry", "count", len(adopted), "retained_seeds", count) + s.mu.Lock() + s.retainedSeeds = append(s.retainedSeeds, adopted...) + count := len(s.retainedSeeds) + s.mu.Unlock() + s.log().Info("adopted untracked engine seeds for managed expiry", "count", len(adopted), "retained_seeds", count) } -func (w *Worker) cleanupRetainedSeed(ctx context.Context, seed retainedSeed, reason string) { - w.logger.Info("cleaning retained bt seed", +func (s *SeedManager) cleanupRetainedSeed(ctx context.Context, seed retainedSeed, reason string) { + s.log().Info("cleaning retained bt seed", "task_id", seed.taskID, "engine", seed.engine, "seed_id", seed.seedID, @@ -548,7 +582,7 @@ func (w *Worker) cleanupRetainedSeed(ctx context.Context, seed retainedSeed, rea "reason", reason, ) if err := seed.cleanup(ctx); err != nil { - w.logger.Warn("failed to clean retained bt seed", + s.log().Warn("failed to clean retained bt seed", "task_id", seed.taskID, "engine", seed.engine, "seed_id", seed.seedID, @@ -557,33 +591,33 @@ func (w *Worker) cleanupRetainedSeed(ctx context.Context, seed retainedSeed, rea ) return } - w.mu.Lock() - next := w.retainedSeeds[:0] - for _, retained := range w.retainedSeeds { + s.mu.Lock() + next := s.retainedSeeds[:0] + for _, retained := range s.retainedSeeds { if retained.taskID != seed.taskID { next = append(next, retained) } } - w.retainedSeeds = next - w.mu.Unlock() - if err := w.removeSeedLedger(seed.taskID); err != nil { - w.logger.Warn("failed to remove retained seed ledger entry", "task_id", seed.taskID, "error", err) + s.retainedSeeds = next + s.mu.Unlock() + if err := s.removeSeedLedger(seed.taskID); err != nil { + s.log().Warn("failed to remove retained seed ledger entry", "task_id", seed.taskID, "error", err) } // Tell the server the task is no longer seeding; otherwise its last runtime // report (phase=seeding) sticks and the UI shows it seeding forever. - w.reportSeedingStopped(ctx, seed.taskID, seed.engine) + s.reportSeedingStopped(ctx, seed.taskID, seed.engine) } // reportSeedingStopped flips a completed task's runtime out of the seeding phase // so the dashboard stops showing it as actively seeding. The task status stays // completed; only the runtime phase/seeding flags change. -func (w *Worker) reportSeedingStopped(ctx context.Context, taskID string, engineName string) { - if engineName == "" && w.engine != nil { - engineName = w.engine.Name() +func (s *SeedManager) reportSeedingStopped(ctx context.Context, taskID string, engineName string) { + if engineName == "" && s.manager() != nil { + engineName = s.manager().Name() } active := false zero := int64(0) - if _, err := w.updateTask(ctx, taskID, client.TaskPatch{ + if _, err := s.updateTask(ctx, taskID, client.TaskPatch{ Runtime: &client.DownloadTaskRuntime{ Engine: engineName, Phase: "completed", @@ -593,7 +627,7 @@ func (w *Worker) reportSeedingStopped(ctx context.Context, taskID string, engine }, }, }); err != nil { - w.logger.Warn("failed to report seeding stopped", "task_id", taskID, "error", err) + s.log().Warn("failed to report seeding stopped", "task_id", taskID, "error", err) } } @@ -601,17 +635,17 @@ func (w *Worker) reportSeedingStopped(ctx context.Context, taskID string, engine // reports as seeding but the worker is no longer tracking (a seed cleaned up // before this fix shipped, or by a worker that never reported it stopped) gets // a one-time stopped report so the dashboard clears it. -func (w *Worker) clearStaleSeedingReports(ctx context.Context) { - tasks, err := w.api.SeedingTasks(ctx) +func (s *SeedManager) ClearStaleReports(ctx context.Context) { + tasks, err := s.api.SeedingTasks(ctx) if err != nil { - w.logger.Warn("failed to list seeding tasks for reconciliation", "error", err) + s.log().Warn("failed to list seeding tasks for reconciliation", "error", err) return } if len(tasks) == 0 { return } tracked := map[string]struct{}{} - for _, seed := range w.retainedSeedSnapshot() { + for _, seed := range s.retainedSeedSnapshot() { tracked[seed.taskID] = struct{}{} } cleared := 0 @@ -619,31 +653,31 @@ func (w *Worker) clearStaleSeedingReports(ctx context.Context) { if _, ok := tracked[task.ID]; ok { continue } - w.reportSeedingStopped(ctx, task.ID, "") + s.reportSeedingStopped(ctx, task.ID, "") cleared++ } if cleared > 0 { - w.logger.Info("cleared stale seeding reports", "count", cleared) + s.log().Info("cleared stale seeding reports", "count", cleared) } } -func (w *Worker) cleanupRetainedSeedForTask(ctx context.Context, taskID string, reason string) { - for _, seed := range w.retainedSeedSnapshot() { +func (s *SeedManager) CleanupTask(ctx context.Context, taskID string, reason string) { + for _, seed := range s.retainedSeedSnapshot() { if seed.taskID == taskID { - w.cleanupRetainedSeed(ctx, seed, reason) + s.cleanupRetainedSeed(ctx, seed, reason) break } } - if err := w.removeSeedLedger(taskID); err != nil { - w.logger.Warn("failed to remove retained seed ledger entry", "task_id", taskID, "error", err) + if err := s.removeSeedLedger(taskID); err != nil { + s.log().Warn("failed to remove retained seed ledger entry", "task_id", taskID, "error", err) } } -func (w *Worker) upsertSeedLedger(seed retainedSeed, size int64) error { - if w.cfg.StateDir == "" { +func (s *SeedManager) upsertSeedLedger(seed retainedSeed, size int64) error { + if s.cfg.StateDir == "" { return nil } - ledger, err := loadSeedLedger(w.cfg.StateDir) + ledger, err := loadSeedLedger(s.cfg.StateDir) if err != nil { return err } @@ -658,8 +692,8 @@ func (w *Worker) upsertSeedLedger(seed retainedSeed, size int64) error { ExpiresAt: seed.expiresAt, Downloaded: size, UploadBase: seed.uploadBase, - SeedDuration: w.cfg.SeedDuration.String(), - SeedRatio: w.cfg.SeedRatio, + SeedDuration: s.cfg.SeedDuration.String(), + SeedRatio: s.cfg.SeedRatio, } replaced := false for i := range ledger.Seeds { @@ -672,14 +706,14 @@ func (w *Worker) upsertSeedLedger(seed retainedSeed, size int64) error { if !replaced { ledger.Seeds = append(ledger.Seeds, entry) } - return saveSeedLedger(w.cfg.StateDir, ledger) + return saveSeedLedger(s.cfg.StateDir, ledger) } -func (w *Worker) removeSeedLedger(taskID string) error { - if w.cfg.StateDir == "" { +func (s *SeedManager) removeSeedLedger(taskID string) error { + if s.cfg.StateDir == "" { return nil } - ledger, err := loadSeedLedger(w.cfg.StateDir) + ledger, err := loadSeedLedger(s.cfg.StateDir) if err != nil { return err } @@ -690,5 +724,5 @@ func (w *Worker) removeSeedLedger(taskID string) error { } } ledger.Seeds = next - return saveSeedLedger(w.cfg.StateDir, ledger) + return saveSeedLedger(s.cfg.StateDir, ledger) } diff --git a/cmd/internal/downloader/task_mapper.go b/cmd/internal/downloader/task_mapper.go new file mode 100644 index 00000000..537ad4e9 --- /dev/null +++ b/cmd/internal/downloader/task_mapper.go @@ -0,0 +1,273 @@ +package downloader + +import ( + "github.com/saltbo/zpan/internal/client" +) + +func downloadTask(task client.DownloadTask) DownloadTask { + return DownloadTask{ + ID: task.ID, + Source: Source{ + Type: task.SourceType(), + URI: task.SourceURI(), + }, + Destination: Destination{ + Name: task.Name(), + }, + Labels: Labels{ + Category: task.Category(), + Tags: task.Tags(), + }, + Status: Status{ + State: task.State(), + Progress: TaskProgress{ + Download: TransferProgress{ + Bytes: task.Status.Progress.Download.Bytes, + TotalBytes: task.Status.Progress.Download.TotalBytes, + Bps: task.Status.Progress.Download.BytesPerSecond, + }, + }, + Runtime: downloaderRuntime(task.Runtime()), + }, + } +} + +func zpanRuntime(runtime *TaskRuntime) *client.DownloadTaskRuntime { + if runtime == nil { + return nil + } + return &client.DownloadTaskRuntime{ + Engine: runtime.Engine, + Phase: runtime.Phase, + State: runtime.State, + Message: runtime.Message, + UpdatedAt: runtime.UpdatedAt, + Progress: zpanRuntimeProgress(runtime.Progress), + ETASeconds: runtime.ETASeconds, + Connections: runtime.Connections, + Torrent: zpanTorrentRuntime(runtime.Torrent), + Seeding: zpanSeedingRuntime(runtime.Seeding), + Trackers: zpanTrackers(runtime.Trackers), + Peers: zpanPeers(runtime.Peers), + Files: zpanFiles(runtime.Files), + } +} + +func downloaderRuntime(runtime *client.DownloadTaskRuntime) *TaskRuntime { + if runtime == nil { + return nil + } + return &TaskRuntime{ + Engine: runtime.Engine, + Phase: runtime.Phase, + State: runtime.State, + Message: runtime.Message, + UpdatedAt: runtime.UpdatedAt, + Progress: downloaderRuntimeProgress(runtime.Progress), + ETASeconds: runtime.ETASeconds, + Connections: runtime.Connections, + Torrent: downloaderTorrentRuntime(runtime.Torrent), + Seeding: downloaderSeedingRuntime(runtime.Seeding), + Trackers: downloaderTrackers(runtime.Trackers), + Peers: downloaderPeers(runtime.Peers), + Files: downloaderFiles(runtime.Files), + } +} + +func zpanRuntimeProgress(progress *RuntimeProgress) *client.DownloadTaskProgress { + if progress == nil { + return nil + } + return &client.DownloadTaskProgress{ + Download: zpanTransferProgress(progress.Download), + Upload: zpanTransferProgress(progress.Upload), + } +} + +func downloaderRuntimeProgress(progress *client.DownloadTaskProgress) *RuntimeProgress { + if progress == nil { + return nil + } + return &RuntimeProgress{ + Download: downloaderTransferProgress(progress.Download), + Upload: downloaderTransferProgress(progress.Upload), + } +} + +func zpanTransferProgress(progress TransferProgress) client.DownloadTaskTransferProgress { + return client.DownloadTaskTransferProgress{ + Bytes: progress.Bytes, + TotalBytes: progress.TotalBytes, + BytesPerSecond: progress.Bps, + } +} + +func downloaderTransferProgress(progress client.DownloadTaskTransferProgress) TransferProgress { + return TransferProgress{ + Bytes: progress.Bytes, + TotalBytes: progress.TotalBytes, + Bps: progress.BytesPerSecond, + } +} + +func zpanTorrentRuntime(torrent *TorrentRuntime) *client.DownloadTaskTorrentRuntime { + if torrent == nil { + return nil + } + return &client.DownloadTaskTorrentRuntime{ + InfoHash: torrent.InfoHash, + Name: torrent.Name, + Seeders: torrent.Seeders, + Leechers: torrent.Leechers, + Peers: torrent.Peers, + } +} + +func downloaderTorrentRuntime(torrent *client.DownloadTaskTorrentRuntime) *TorrentRuntime { + if torrent == nil { + return nil + } + return &TorrentRuntime{ + InfoHash: torrent.InfoHash, + Name: torrent.Name, + Seeders: torrent.Seeders, + Leechers: torrent.Leechers, + Peers: torrent.Peers, + } +} + +func zpanSeedingRuntime(seeding *SeedingRuntime) *client.DownloadTaskSeedingRuntime { + if seeding == nil { + return nil + } + return &client.DownloadTaskSeedingRuntime{ + Enabled: seeding.Enabled, + Active: seeding.Active, + UploadedBytes: seeding.UploadedBytes, + UploadBytesPerSecond: seeding.UploadBytesPerSecond, + Ratio: seeding.Ratio, + StartedAt: seeding.StartedAt, + ExpiresAt: seeding.ExpiresAt, + } +} + +func downloaderSeedingRuntime(seeding *client.DownloadTaskSeedingRuntime) *SeedingRuntime { + if seeding == nil { + return nil + } + return &SeedingRuntime{ + Enabled: seeding.Enabled, + Active: seeding.Active, + UploadedBytes: seeding.UploadedBytes, + UploadBytesPerSecond: seeding.UploadBytesPerSecond, + Ratio: seeding.Ratio, + StartedAt: seeding.StartedAt, + ExpiresAt: seeding.ExpiresAt, + } +} + +func zpanTrackers(trackers []Tracker) []client.DownloadTaskTracker { + if len(trackers) == 0 { + return nil + } + out := make([]client.DownloadTaskTracker, 0, len(trackers)) + for _, tracker := range trackers { + out = append(out, client.DownloadTaskTracker{ + URL: tracker.URL, + Status: tracker.Status, + Peers: tracker.Peers, + Seeds: tracker.Seeds, + Leechers: tracker.Leechers, + Message: tracker.Message, + }) + } + return out +} + +func downloaderTrackers(trackers []client.DownloadTaskTracker) []Tracker { + if len(trackers) == 0 { + return nil + } + out := make([]Tracker, 0, len(trackers)) + for _, tracker := range trackers { + out = append(out, Tracker{ + URL: tracker.URL, + Status: tracker.Status, + Peers: tracker.Peers, + Seeds: tracker.Seeds, + Leechers: tracker.Leechers, + Message: tracker.Message, + }) + } + return out +} + +func zpanPeers(peers []Peer) []client.DownloadTaskPeer { + if len(peers) == 0 { + return nil + } + out := make([]client.DownloadTaskPeer, 0, len(peers)) + for _, peer := range peers { + out = append(out, client.DownloadTaskPeer{ + Address: peer.Address, + Client: peer.Client, + CountryCode: peer.CountryCode, + RegionCode: peer.RegionCode, + Progress: peer.Progress, + DownloadBps: peer.DownloadBps, + UploadBps: peer.UploadBps, + }) + } + return out +} + +func downloaderPeers(peers []client.DownloadTaskPeer) []Peer { + if len(peers) == 0 { + return nil + } + out := make([]Peer, 0, len(peers)) + for _, peer := range peers { + out = append(out, Peer{ + Address: peer.Address, + Client: peer.Client, + CountryCode: peer.CountryCode, + RegionCode: peer.RegionCode, + Progress: peer.Progress, + DownloadBps: peer.DownloadBps, + UploadBps: peer.UploadBps, + }) + } + return out +} + +func zpanFiles(files []File) []client.DownloadTaskFile { + if len(files) == 0 { + return nil + } + out := make([]client.DownloadTaskFile, 0, len(files)) + for _, file := range files { + out = append(out, client.DownloadTaskFile{ + Path: file.Path, + Size: file.Size, + CompletedBytes: file.CompletedBytes, + Selected: file.Selected, + }) + } + return out +} + +func downloaderFiles(files []client.DownloadTaskFile) []File { + if len(files) == 0 { + return nil + } + out := make([]File, 0, len(files)) + for _, file := range files { + out = append(out, File{ + Path: file.Path, + Size: file.Size, + CompletedBytes: file.CompletedBytes, + Selected: file.Selected, + }) + } + return out +} diff --git a/cmd/internal/worker/worker.go b/cmd/internal/downloader/task_runner.go similarity index 74% rename from cmd/internal/worker/worker.go rename to cmd/internal/downloader/task_runner.go index 4cae8498..14f93745 100644 --- a/cmd/internal/worker/worker.go +++ b/cmd/internal/downloader/task_runner.go @@ -1,4 +1,4 @@ -package worker +package downloader import ( "context" @@ -6,19 +6,15 @@ import ( "fmt" "log/slog" "os" - "os/exec" - "path/filepath" "runtime" "runtime/debug" - "sort" - "strings" "sync" "time" "github.com/saltbo/zpan/internal/client" "github.com/saltbo/zpan/internal/config" - "github.com/saltbo/zpan/internal/engine" - "github.com/saltbo/zpan/internal/host" + "github.com/saltbo/zpan/pkg/geoip" + "github.com/saltbo/zpan/pkg/system" ) const Version = "0.1.0" @@ -38,21 +34,19 @@ const ( taskWorkStageUploadExistingResult ) -type Worker struct { - cfg config.Config - api apiClient - engine engine.Engine - geoIP *engine.GeoIPResolver - logger *slog.Logger - running map[string]context.CancelCauseFunc - speeds map[string]transferSpeeds - retainedSeeds []retainedSeed - attempts map[string]int - started []*exec.Cmd - cancelRun context.CancelCauseFunc - stopping bool - wg sync.WaitGroup - mu sync.Mutex +type TaskRunner struct { + cfg config.Config + api apiClient + downloader *Manager + uploader *Uploader + seeds *SeedManager + geoIP *geoip.DB + logger *slog.Logger + running map[string]context.CancelCauseFunc + speeds map[string]transferSpeeds + attempts map[string]int + wg sync.WaitGroup + mu sync.Mutex } type transferSpeeds struct { @@ -70,9 +64,10 @@ type apiClient interface { CreateObject(context.Context, string, string, int64, string) (client.ObjectDraft, error) CompleteObjectUpload(context.Context, string, string, string, []client.CompletedObjectUploadPart) error AbortObjectUploadSession(context.Context, string, string, string) error + DeleteObject(context.Context, string, string) error } -func New(cfg config.Config) (*Worker, error) { +func NewTaskRunner(cfg config.Config) (*TaskRunner, error) { if cfg.Token == "" { return nil, errors.New("token is required") } @@ -80,11 +75,11 @@ func New(cfg config.Config) (*Worker, error) { if err != nil { return nil, err } - return NewWithAPI(cfg, api), nil + return NewTaskRunnerWithAPI(cfg, api), nil } -func NewWithAPI(cfg config.Config, api apiClient) *Worker { - return &Worker{ +func NewTaskRunnerWithAPI(cfg config.Config, api apiClient) *TaskRunner { + runner := &TaskRunner{ cfg: cfg, api: api, logger: slog.Default(), @@ -92,9 +87,12 @@ func NewWithAPI(cfg config.Config, api apiClient) *Worker { speeds: map[string]transferSpeeds{}, attempts: map[string]int{}, } + runner.uploader = NewUploader(api, runner.setTaskTransferSpeed) + runner.seeds = NewSeedManager(cfg, api, func() *slog.Logger { return runner.logger }, func() *Manager { return runner.downloader }, runner.runningTaskIDs, runner.localResultTaskIDs) + return runner } -func (w *Worker) Run(ctx context.Context) error { +func (w *TaskRunner) Run(ctx context.Context) error { if err := os.MkdirAll(w.cfg.DownloadDir, 0o755); err != nil { return err } @@ -111,7 +109,7 @@ func (w *Worker) Run(ctx context.Context) error { "seed_cache_limit", w.cfg.SeedCacheLimit, "seed_ratio", w.cfg.SeedRatio, ) - geoIP, err := engine.OpenGeoIPResolver(w.cfg.GeoIPDBPath) + geoIP, err := geoip.Open(w.cfg.GeoIPDBPath) if err != nil { return fmt.Errorf("open geoip database: %w", err) } @@ -121,30 +119,23 @@ func (w *Worker) Run(ctx context.Context) error { defer w.geoIP.Close() } // runCtx is cancelled either by the parent ctx (signal-driven shutdown) or - // by watchEngineProcess when a managed engine subprocess dies. The latter - // surfaces errEngineExited as the cancel cause so Run returns a non-nil - // error and the process exits non-zero for the supervisor to restart. + // by the downloader manager when a managed subprocess dies. The latter + // surfaces errEngineExited so the supervisor restarts this process. runCtx, cancelRun := context.WithCancelCause(ctx) defer cancelRun(nil) - w.cancelRun = cancelRun - if err := w.resolveEngine(runCtx); err != nil { + w.downloader = NewManager(w.cfg, w.geoIP, w.logger) + if err := w.downloader.Start(runCtx, func(err error) { + cancelRun(fmt.Errorf("%w: %v", errEngineExited, err)) + }); err != nil { return err } - defer w.stopStartedEngines() + defer w.downloader.Stop(context.Background()) - checkCtx, cancel := context.WithTimeout(runCtx, 10*time.Second) - defer cancel() - w.logger.Info("checking downloader engine", "engine", w.cfg.Engine) - if err := w.engine.Check(checkCtx); err != nil { - w.logger.Error("downloader engine check failed", "engine", w.cfg.Engine, "error", err) - return fmt.Errorf("engine %q is not available: %w", w.cfg.Engine, err) - } - w.logger.Info("downloader engine check passed", "engine", w.cfg.Engine) - w.logger.Info("downloader started", "engine", w.cfg.Engine) - w.restoreRetainedSeeds(runCtx) - w.reconcileEngineSeeds(runCtx) - w.clearStaleSeedingReports(runCtx) + w.logger.Info("downloader started", "engine", w.downloader.Name()) + w.seeds.Restore(runCtx) + w.seeds.Reconcile(runCtx) + w.seeds.ClearStaleReports(runCtx) pollTimer := time.NewTimer(0) defer pollTimer.Stop() seedCleanupTicker := time.NewTicker(time.Minute) @@ -156,10 +147,10 @@ func (w *Worker) Run(ctx context.Context) error { select { case <-runCtx.Done(): cause := context.Cause(runCtx) - w.reportRetainedSeedsStopped(context.WithoutCancel(runCtx)) + w.seeds.ReportStopped(context.WithoutCancel(runCtx)) w.waitForTasks() if errors.Is(cause, errEngineExited) { - // watchEngineProcess already logged the exit at error level. + // The downloader manager already logged the process exit. return cause } w.logger.Info("downloader stopped", "reason", cause) @@ -172,23 +163,23 @@ func (w *Worker) Run(ctx context.Context) error { } pollTimer.Reset(nextPoll) case <-seedCleanupTicker.C: - w.cleanupRetainedSeeds(runCtx) + w.seeds.Cleanup(runCtx) case <-seedReportTicker.C: - w.restoreRetainedSeeds(runCtx) - w.reconcileEngineSeeds(runCtx) - w.reportRetainedSeeds(runCtx) + w.seeds.Restore(runCtx) + w.seeds.Reconcile(runCtx) + w.seeds.Report(runCtx) } } } -func (w *Worker) tick(ctx context.Context) error { +func (w *TaskRunner) tick(ctx context.Context) error { _, err := w.tickAndNextPoll(ctx) return err } -func (w *Worker) tickAndNextPoll(ctx context.Context) (time.Duration, error) { +func (w *TaskRunner) tickAndNextPoll(ctx context.Context) (time.Duration, error) { var heartbeat client.HeartbeatResult - if err := w.callAPI(ctx, "heartbeat", func(ctx context.Context) error { + if err := callAPI(ctx, w.logger, "heartbeat", func(ctx context.Context) error { var err error heartbeat, err = w.api.Heartbeat(ctx, w.heartbeat()) return err @@ -218,21 +209,31 @@ func (w *Worker) tickAndNextPoll(ctx context.Context) (time.Duration, error) { return w.remotePollInterval(heartbeat), nil } -func (w *Worker) remotePollInterval(heartbeat client.HeartbeatResult) time.Duration { +func (w *TaskRunner) updateTask(ctx context.Context, id string, patch client.TaskPatch) (client.DownloadTask, error) { + var task client.DownloadTask + err := callAPI(ctx, w.logger, "update task", func(ctx context.Context) error { + var err error + task, err = w.api.UpdateTask(ctx, id, patch) + return err + }) + return task, err +} + +func (w *TaskRunner) remotePollInterval(heartbeat client.HeartbeatResult) time.Duration { if heartbeat.NextPollAfterSeconds <= 0 { return w.localPollInterval() } return time.Duration(heartbeat.NextPollAfterSeconds) * time.Second } -func (w *Worker) localPollInterval() time.Duration { +func (w *TaskRunner) localPollInterval() time.Duration { if w.cfg.PollInterval > 0 { return w.cfg.PollInterval } return 5 * time.Second } -func (w *Worker) process(ctx context.Context, task client.DownloadTask) { +func (w *TaskRunner) process(ctx context.Context, task client.DownloadTask) { defer w.finish(task.ID) log := w.taskLogger(task) defer w.recoverTaskPanic(ctx, task.ID, log) @@ -254,7 +255,7 @@ func (w *Worker) process(ctx context.Context, task client.DownloadTask) { w.downloadThenUpload(ctx, log, task, currentDetail) } -func (w *Worker) downloadThenUpload( +func (w *TaskRunner) downloadThenUpload( ctx context.Context, log *slog.Logger, task client.DownloadTask, @@ -262,7 +263,7 @@ func (w *Worker) downloadThenUpload( ) { // Marking the task downloading is also the credit gate: the server charges the // first unit on this transition and answers with the authoritative status. If - // it comes back suspended, don't pull a single byte. (Progress reports below + // it comes back suspended, don't pull a single byte. (ProgressReporter reports below // stay pure telemetry — control still flows through the poll.) if updated, err := w.updateTask(ctx, task.ID, client.TaskPatch{Status: "downloading"}); err != nil { log.Warn("failed to mark task downloading", "error", err) @@ -272,18 +273,22 @@ func (w *Worker) downloadThenUpload( } var lastProgressLog time.Time - // Progress reporting is pure telemetry: it never decides whether to stop. + // ProgressReporter reporting is pure telemetry: it never decides whether to stop. // Control transitions (pause/cancel/suspend) arrive through the task poll // and cancel this context; a failed report just gets logged and retried. - result, err := w.engine.Download(ctx, task, func(downloaded int64, total *int64, bps int64, detail *client.DownloadTaskRuntime) error { + result, err := w.downloader.Download(ctx, downloadTask(task), func(update ProgressUpdate) error { + downloaded := update.Downloaded + total := update.Total + bps := update.Bps + detail := update.Runtime w.setTaskTransferSpeed(task.ID, transferSpeeds{downloadBps: bps}) - detail = withDownloadRuntime(detail, downloaded, total, bps) - if detail != nil { - currentDetail = detail + zpanDetail := withDownloadRuntime(zpanRuntime(detail), downloaded, total, bps) + if zpanDetail != nil { + currentDetail = zpanDetail } if _, updateErr := w.updateTask(ctx, task.ID, client.TaskPatch{ Progress: downloadProgressPatch(downloaded, total, bps), - Runtime: detail, + Runtime: zpanDetail, }); updateErr != nil { log.Warn("failed to report task progress", "downloaded_bytes", downloaded, "total_bytes", optionalInt64(total), "bps", bps, "error", updateErr) } @@ -339,13 +344,13 @@ func (w *Worker) downloadThenUpload( w.uploadAndComplete(ctx, log, task, result, currentDetail) } -func (w *Worker) uploadExistingResult( +func (w *TaskRunner) uploadExistingResult( ctx context.Context, log *slog.Logger, task client.DownloadTask, currentDetail *client.DownloadTaskRuntime, ) { - snapshot, found, err := w.engine.InspectTask(ctx, task) + snapshot, found, err := w.downloader.InspectTask(ctx, downloadTask(task)) if err != nil { msg := taskErrorMessage(err) log.Error("failed to inspect downloader runtime task", "error", err) @@ -362,7 +367,7 @@ func (w *Worker) uploadExistingResult( } return } - if snapshot.State != engine.TaskStateCompleted || snapshot.Result == nil { + if snapshot.State != TaskStateCompleted || snapshot.Result == nil { // The server checkpoint routed us here to upload an already-finished // download, but the engine isn't reporting it complete yet — e.g. aria2 // is re-checking on-disk files after a restart, or the download was lost @@ -382,7 +387,7 @@ func (w *Worker) uploadExistingResult( w.uploadAndComplete(ctx, log, task, *snapshot.Result, currentDetail) } -func (w *Worker) recoverTaskPanic(ctx context.Context, taskID string, log *slog.Logger) { +func (w *TaskRunner) recoverTaskPanic(ctx context.Context, taskID string, log *slog.Logger) { value := recover() if value == nil { return @@ -420,7 +425,7 @@ func nextTaskWorkStage(task client.DownloadTask) taskWorkStage { return taskWorkStageDownload } -func (w *Worker) resetTaskForAttempt(ctx context.Context, task client.DownloadTask, log *slog.Logger) error { +func (w *TaskRunner) resetTaskForAttempt(ctx context.Context, task client.DownloadTask, log *slog.Logger) error { attempt := task.Attempt() if attempt <= 0 { return fmt.Errorf("download task has invalid attempt %d", attempt) @@ -458,13 +463,13 @@ func (w *Worker) resetTaskForAttempt(ctx context.Context, task client.DownloadTa return nil } -func (w *Worker) memoryAttempt(taskID string) int { +func (w *TaskRunner) memoryAttempt(taskID string) int { w.mu.Lock() defer w.mu.Unlock() return w.attempts[taskID] } -func (w *Worker) setMemoryAttempt(taskID string, attempt int) { +func (w *TaskRunner) setMemoryAttempt(taskID string, attempt int) { w.mu.Lock() defer w.mu.Unlock() if w.attempts == nil { @@ -473,43 +478,34 @@ func (w *Worker) setMemoryAttempt(taskID string, attempt int) { w.attempts[taskID] = attempt } -func (w *Worker) resetRuntimeTask(ctx context.Context, task client.DownloadTask, log *slog.Logger) error { - resetter, ok := w.engine.(engine.TaskResetter) - if !ok { - return fmt.Errorf("engine %s does not support task reset", w.engine.Name()) - } - w.cleanupRetainedSeedForTask(ctx, task.ID, "restart") +func (w *TaskRunner) resetRuntimeTask(ctx context.Context, task client.DownloadTask, log *slog.Logger) error { + w.seeds.CleanupTask(ctx, task.ID, "restart") log.Info("resetting downloader runtime task", "attempt", task.Attempt()) - if err := resetter.ResetTask(ctx, task); err != nil { + if err := w.downloader.ResetTask(ctx, downloadTask(task)); err != nil { return fmt.Errorf("reset downloader runtime task: %w", err) } return nil } -func (w *Worker) cleanupDeletedTask(ctx context.Context, log *slog.Logger, task client.DownloadTask) { +func (w *TaskRunner) cleanupDeletedTask(ctx context.Context, log *slog.Logger, task client.DownloadTask) { reason := "deleted" - w.cleanupRetainedSeedForTask(ctx, task.ID, reason) - if w.engine == nil { + w.seeds.CleanupTask(ctx, task.ID, reason) + if w.downloader == nil { log.Warn("downloader engine is unavailable for terminal cleanup", "reason", reason) return } - resetter, ok := w.engine.(engine.TaskResetter) - if !ok { - log.Warn("downloader engine does not support terminal cleanup", "engine", w.engine.Name(), "reason", reason) - return - } - if err := resetter.ResetTask(ctx, task); err != nil { + if err := w.downloader.ResetTask(ctx, downloadTask(task)); err != nil { log.Warn("failed to clean terminal downloader task", "reason", reason, "error", err) return } log.Info("cleaned terminal downloader task", "reason", reason) } -func (w *Worker) uploadAndComplete( +func (w *TaskRunner) uploadAndComplete( ctx context.Context, log *slog.Logger, task client.DownloadTask, - result engine.Result, + result Result, currentDetail *client.DownloadTaskRuntime, ) { zero := int64(0) @@ -522,7 +518,7 @@ func (w *Worker) uploadAndComplete( task.Status.Progress.Download = *transferProgress(result.Size, &result.Size, zero) task.Status.Progress.Upload = *transferProgress(0, &result.Size, zero) task.Status.Runtime = currentDetail - resultObjectID, err := w.uploadResult(ctx, log, task, result) + resultObjectID, err := w.uploader.Upload(ctx, log, task, result) if err != nil { downloadedBytes := result.Size if errors.Is(err, context.Canceled) { @@ -599,9 +595,9 @@ func (w *Worker) uploadAndComplete( return } log.Debug("task completed", "object_id", resultObjectID) - if w.retainSeed(task, result, log) { - w.reportRetainedSeeds(ctx) - w.cleanupRetainedSeeds(ctx) + if w.seeds.Retain(ctx, task, result, log) { + w.seeds.Report(ctx) + w.seeds.Cleanup(ctx) return } if err := cleanupDownloadedResult(ctx, task, result); err != nil { @@ -672,85 +668,7 @@ func uploadETA(progress *uploadProgress, bps int64) *int64 { return &eta } -type directoryEntry struct { - path string - relativePath string - name string - size int64 - isDir bool -} - -func collectDirectoryEntries(root string) ([]directoryEntry, error) { - var entries []directoryEntry - err := filepath.WalkDir(root, func(path string, entry os.DirEntry, err error) error { - if err != nil { - return err - } - if path == root { - return nil - } - if strings.HasPrefix(entry.Name(), ".") { - if entry.IsDir() { - return filepath.SkipDir - } - return nil - } - relativePath, err := filepath.Rel(root, path) - if err != nil { - return err - } - if !entry.IsDir() && isDownloadSidecarPath(relativePath) { - return nil - } - item := directoryEntry{ - path: path, - relativePath: filepath.ToSlash(relativePath), - name: entry.Name(), - isDir: entry.IsDir(), - } - if !entry.IsDir() { - info, err := entry.Info() - if err != nil { - return err - } - item.size = info.Size() - } - entries = append(entries, item) - return nil - }) - sort.Slice(entries, func(i, j int) bool { - if entries[i].isDir != entries[j].isDir { - return entries[i].isDir - } - return entries[i].relativePath < entries[j].relativePath - }) - return entries, err -} - -func isDownloadSidecarPath(path string) bool { - base := filepath.Base(path) - ext := filepath.Ext(base) - return strings.HasPrefix(path, "[MEMORY]") || - strings.HasPrefix(path, "[METADATA]") || - strings.HasPrefix(base, "[MEMORY]") || - strings.HasPrefix(base, "[METADATA]") || - strings.EqualFold(ext, ".torrent") || - strings.EqualFold(ext, ".aria2") -} - -func joinObjectPath(parent string, name string) string { - name = strings.Trim(filepath.ToSlash(name), "/") - if name == "" || name == "." { - return strings.Trim(parent, "/") - } - parent = strings.Trim(parent, "/") - if parent == "" { - return name - } - return parent + "/" + name -} - -func (w *Worker) startTask(ctx context.Context, taskID string) (context.Context, bool) { +func (w *TaskRunner) startTask(ctx context.Context, taskID string) (context.Context, bool) { w.mu.Lock() defer w.mu.Unlock() if _, exists := w.running[taskID]; exists { @@ -766,7 +684,7 @@ func (w *Worker) startTask(ctx context.Context, taskID string) (context.Context, return taskCtx, true } -func (w *Worker) finish(taskID string) { +func (w *TaskRunner) finish(taskID string) { w.mu.Lock() _, exists := w.running[taskID] if exists { @@ -779,21 +697,7 @@ func (w *Worker) finish(taskID string) { } } -// markStopping records that the worker is shutting down on purpose so -// watchEngineProcess can tell a deliberate engine kill from a crash. -func (w *Worker) markStopping() { - w.mu.Lock() - w.stopping = true - w.mu.Unlock() -} - -func (w *Worker) isStopping() bool { - w.mu.Lock() - defer w.mu.Unlock() - return w.stopping -} - -func (w *Worker) waitForTasks() { +func (w *TaskRunner) waitForTasks() { done := make(chan struct{}) go func() { w.wg.Wait() @@ -806,7 +710,7 @@ func (w *Worker) waitForTasks() { } } -func (w *Worker) cancelRunning(task client.DownloadTask) bool { +func (w *TaskRunner) cancelRunning(task client.DownloadTask) bool { w.mu.Lock() cancel := w.running[task.ID] w.mu.Unlock() @@ -825,7 +729,7 @@ func (w *Worker) cancelRunning(task client.DownloadTask) bool { return true } -func (w *Worker) ackStoppedControlTask(ctx context.Context, task client.DownloadTask) { +func (w *TaskRunner) ackStoppedControlTask(ctx context.Context, task client.DownloadTask) { log := w.taskLogger(task) if task.State() == "pausing" { if _, err := w.updateTask(ctx, task.ID, client.TaskPatch{Status: "paused"}); err != nil { @@ -851,21 +755,21 @@ func (w *Worker) ackStoppedControlTask(ctx context.Context, task client.Download } } -func (w *Worker) heartbeat() client.Heartbeat { +func (w *TaskRunner) heartbeat() client.Heartbeat { engineName := w.cfg.Engine capabilities := []string{"http"} - if w.engine != nil { - engineName = w.engine.Name() - capabilities = w.engine.Capabilities() + if w.downloader != nil { + engineName = w.downloader.Name() + capabilities = w.downloader.Capabilities() } speeds := w.currentTransferSpeeds() - freeDiskBytes, err := freeDiskBytes(w.cfg.DownloadDir) + freeDiskBytes, err := system.FreeDiskBytes(w.cfg.DownloadDir) if err != nil { w.logger.Warn("failed to inspect downloader free disk space", "download_dir", w.cfg.DownloadDir, "error", err) } return client.Heartbeat{ Version: Version, - Hostname: host.DownloaderHostname(), + Hostname: system.DownloaderHostname(), Platform: runtime.GOOS, Arch: runtime.GOARCH, Engine: engineName, @@ -878,13 +782,39 @@ func (w *Worker) heartbeat() client.Heartbeat { } } -func (w *Worker) currentTasks() int { +func (w *TaskRunner) currentTasks() int { w.mu.Lock() defer w.mu.Unlock() return len(w.running) } -func (w *Worker) setTaskTransferSpeed(taskID string, speeds transferSpeeds) { +func (w *TaskRunner) runningTaskIDs() map[string]struct{} { + w.mu.Lock() + defer w.mu.Unlock() + ids := make(map[string]struct{}, len(w.running)) + for id := range w.running { + ids[id] = struct{}{} + } + return ids +} + +// localResultTaskIDs lists tasks whose local completed download may still be +// needed by this downloader, so seed cleanup cannot delete it before retrying +// an upload failure or resuming a paused/suspended upload. +func (w *TaskRunner) localResultTaskIDs(ctx context.Context) (map[string]struct{}, bool) { + ids := map[string]struct{}{} + tasks, err := w.api.LocalResultTasks(ctx) + if err != nil { + w.logger.Warn("failed to list local-result tasks for seed reconciliation", "error", err) + return ids, false + } + for _, task := range tasks { + ids[task.ID] = struct{}{} + } + return ids, true +} + +func (w *TaskRunner) setTaskTransferSpeed(taskID string, speeds transferSpeeds) { w.mu.Lock() defer w.mu.Unlock() if _, exists := w.running[taskID]; !exists { @@ -893,7 +823,7 @@ func (w *Worker) setTaskTransferSpeed(taskID string, speeds transferSpeeds) { w.speeds[taskID] = speeds } -func (w *Worker) currentTransferSpeeds() transferSpeeds { +func (w *TaskRunner) currentTransferSpeeds() transferSpeeds { w.mu.Lock() defer w.mu.Unlock() var total transferSpeeds @@ -912,7 +842,7 @@ func taskErrorMessage(err error) string { return msg[:maxTaskErrorMessageLength-3] + "..." } -func (w *Worker) taskLogger(task client.DownloadTask) *slog.Logger { +func (w *TaskRunner) taskLogger(task client.DownloadTask) *slog.Logger { return w.logger.With( "task_id", task.ID, "source_type", task.SourceType(), @@ -934,31 +864,3 @@ func optionalTime(value time.Time) any { } return value.Format(time.RFC3339) } - -func directorySize(root string) (int64, error) { - var total int64 - err := filepath.WalkDir(root, func(_ string, entry os.DirEntry, err error) error { - if err != nil { - if errors.Is(err, os.ErrNotExist) { - return nil - } - return err - } - if entry.IsDir() { - return nil - } - info, err := entry.Info() - if err != nil { - if errors.Is(err, os.ErrNotExist) { - return nil - } - return err - } - total += info.Size() - return nil - }) - if errors.Is(err, os.ErrNotExist) { - return 0, nil - } - return total, err -} diff --git a/cmd/internal/worker/worker_test.go b/cmd/internal/downloader/task_runner_test.go similarity index 79% rename from cmd/internal/worker/worker_test.go rename to cmd/internal/downloader/task_runner_test.go index 64649b49..5bca636d 100644 --- a/cmd/internal/worker/worker_test.go +++ b/cmd/internal/downloader/task_runner_test.go @@ -1,4 +1,4 @@ -package worker +package downloader import ( "context" @@ -8,8 +8,8 @@ import ( "net/http" "net/http/httptest" "os" - "os/exec" "path/filepath" + "reflect" "strconv" "strings" "testing" @@ -17,7 +17,6 @@ import ( "github.com/saltbo/zpan/internal/client" "github.com/saltbo/zpan/internal/config" - "github.com/saltbo/zpan/internal/engine" ) func TestCancelRunningUsesCauseForControlState(t *testing.T) { @@ -31,7 +30,7 @@ func TestCancelRunningUsesCauseForControlState(t *testing.T) { } for _, tc := range cases { t.Run(tc.state, func(t *testing.T) { - w := NewWithAPI(config.Config{MaxConcurrentTasks: 5}, &recordingAPI{}) + w := NewTaskRunnerWithAPI(config.Config{MaxConcurrentTasks: 5}, &recordingAPI{}) w.logger = slog.New(slog.NewTextHandler(io.Discard, nil)) taskCtx, ok := w.startTask(context.Background(), "task-1") if !ok { @@ -52,10 +51,10 @@ func TestCancelRunningUsesCauseForControlState(t *testing.T) { func TestDownloadThenUploadStopsWhenSuspendedAtStart(t *testing.T) { api := &recordingAPI{suspendDownloading: true} - w := NewWithAPI(config.Config{}, api) + w := NewTaskRunnerWithAPI(config.Config{}, api) w.logger = slog.New(slog.NewTextHandler(io.Discard, nil)) eng := &recordingEngine{} - w.engine = eng + w.downloader = NewManagerWithDownloader(eng) w.downloadThenUpload(context.Background(), w.logger, clientTaskWithStatus("task-1", "assigned"), nil) @@ -67,9 +66,9 @@ func TestDownloadThenUploadStopsWhenSuspendedAtStart(t *testing.T) { func TestCanceledDownloadPreservesRuntimeAndMarksCanceled(t *testing.T) { api := &recordingAPI{} eng := &recordingEngine{downloadErr: context.Canceled} - w := NewWithAPI(config.Config{}, api) + w := NewTaskRunnerWithAPI(config.Config{}, api) w.logger = slog.New(slog.NewTextHandler(io.Discard, nil)) - w.engine = eng + w.downloader = NewManagerWithDownloader(eng) ctx, cancel := context.WithCancelCause(context.Background()) cancel(errTaskCanceling) @@ -87,9 +86,9 @@ func TestCanceledDownloadPreservesRuntimeAndMarksCanceled(t *testing.T) { func TestSuspendedDownloadPreservesRuntimeWithoutStatusChange(t *testing.T) { api := &recordingAPI{} eng := &recordingEngine{downloadErr: context.Canceled} - w := NewWithAPI(config.Config{}, api) + w := NewTaskRunnerWithAPI(config.Config{}, api) w.logger = slog.New(slog.NewTextHandler(io.Discard, nil)) - w.engine = eng + w.downloader = NewManagerWithDownloader(eng) ctx, cancel := context.WithCancelCause(context.Background()) cancel(errTaskSuspended) @@ -113,9 +112,9 @@ func TestTickSuspendedControlTaskPreservesRuntime(t *testing.T) { controlTasks: []client.DownloadTask{clientTaskWithStatus("task-1", "suspended")}, } eng := &recordingEngine{} - w := NewWithAPI(config.Config{}, api) + w := NewTaskRunnerWithAPI(config.Config{}, api) w.logger = slog.New(slog.NewTextHandler(io.Discard, nil)) - w.engine = eng + w.downloader = NewManagerWithDownloader(eng) if err := w.tick(context.Background()); err != nil { t.Fatalf("first tick: %v", err) @@ -143,7 +142,7 @@ func TestTickSuspendedControlTaskPreservesRuntime(t *testing.T) { func TestTickUsesHeartbeatNextPollInterval(t *testing.T) { api := &recordingAPI{nextPollAfterSeconds: 17} - w := NewWithAPI(config.Config{PollInterval: time.Second}, api) + w := NewTaskRunnerWithAPI(config.Config{PollInterval: time.Second}, api) w.logger = slog.New(slog.NewTextHandler(io.Discard, nil)) nextPoll, err := w.tickAndNextPoll(context.Background()) @@ -162,9 +161,9 @@ func TestDeleteRequestedControlTaskCleansRuntimeAndAcksCanceled(t *testing.T) { }, } eng := &recordingEngine{} - w := NewWithAPI(config.Config{}, api) + w := NewTaskRunnerWithAPI(config.Config{}, api) w.logger = slog.New(slog.NewTextHandler(io.Discard, nil)) - w.engine = eng + w.downloader = NewManagerWithDownloader(eng) if err := w.tick(context.Background()); err != nil { t.Fatalf("tick: %v", err) @@ -183,9 +182,9 @@ func TestCancelingControlTaskWithoutDeleteRequestPreservesRuntime(t *testing.T) controlTasks: []client.DownloadTask{clientTaskWithStatus("task-1", "canceling")}, } eng := &recordingEngine{} - w := NewWithAPI(config.Config{}, api) + w := NewTaskRunnerWithAPI(config.Config{}, api) w.logger = slog.New(slog.NewTextHandler(io.Discard, nil)) - w.engine = eng + w.downloader = NewManagerWithDownloader(eng) if err := w.tick(context.Background()); err != nil { t.Fatalf("tick: %v", err) @@ -202,9 +201,9 @@ func TestCancelingControlTaskWithoutDeleteRequestPreservesRuntime(t *testing.T) func TestFailedDownloadPreservesRuntimeAndMarksFailed(t *testing.T) { api := &recordingAPI{} eng := &recordingEngine{downloadErr: errors.New("disk write failed")} - w := NewWithAPI(config.Config{}, api) + w := NewTaskRunnerWithAPI(config.Config{}, api) w.logger = slog.New(slog.NewTextHandler(io.Discard, nil)) - w.engine = eng + w.downloader = NewManagerWithDownloader(eng) w.downloadThenUpload(context.Background(), w.logger, clientTaskWithStatus("task-1", "downloading"), nil) @@ -249,27 +248,27 @@ func TestTerminalDownloadStopsPreservePartialFiles(t *testing.T) { partialPath := filepath.Join(downloadDir, "task-1", "payload.bin") ready := make(chan struct{}, 1) eng := &recordingEngine{ - downloadFunc: func(ctx context.Context, task client.DownloadTask, progress engine.Progress) (engine.Result, error) { + downloadFunc: func(ctx context.Context, task DownloadTask, progress ProgressReporter) (Result, error) { if err := os.MkdirAll(filepath.Dir(partialPath), 0o755); err != nil { - return engine.Result{}, err + return Result{}, err } if err := os.WriteFile(partialPath, []byte("partial"), 0o644); err != nil { - return engine.Result{}, err + return Result{}, err } ready <- struct{}{} if tc.failedMessage != "" { - return engine.Result{}, errors.New(tc.failedMessage) + return Result{}, errors.New(tc.failedMessage) } <-ctx.Done() - return engine.Result{}, ctx.Err() + return Result{}, ctx.Err() }, - resetTaskFn: func(context.Context, client.DownloadTask) error { + resetTaskFn: func(context.Context, DownloadTask) error { return os.RemoveAll(filepath.Join(downloadDir, "task-1")) }, } - w := NewWithAPI(config.Config{DownloadDir: downloadDir}, api) + w := NewTaskRunnerWithAPI(config.Config{DownloadDir: downloadDir}, api) w.logger = slog.New(slog.NewTextHandler(io.Discard, nil)) - w.engine = eng + w.downloader = NewManagerWithDownloader(eng) task := clientHTTPTask("task-1", "downloading", "https://example.com/payload.bin", "payload.bin") if tc.cancelCause != nil { @@ -308,67 +307,17 @@ func TestTerminalDownloadStopsPreservePartialFiles(t *testing.T) { } } -func TestWatchEngineProcessFatalOnUnexpectedExit(t *testing.T) { - w := NewWithAPI(config.Config{}, nil) - w.logger = slog.New(slog.NewTextHandler(io.Discard, nil)) - runCtx, cancel := context.WithCancelCause(context.Background()) - defer cancel(nil) - w.cancelRun = cancel - - cmd := exec.Command("sleep", "30") - if err := cmd.Start(); err != nil { - t.Fatalf("start managed process stub: %v", err) - } - done := make(chan struct{}) - go func() { - w.watchEngineProcess("aria2", cmd) - close(done) - }() - // Simulate the engine dying out from under the worker. - _ = cmd.Process.Kill() - - select { - case <-runCtx.Done(): - case <-time.After(5 * time.Second): - t.Fatal("expected run context to be cancelled after the engine exits") - } - if cause := context.Cause(runCtx); !errors.Is(cause, errEngineExited) { - t.Fatalf("expected errEngineExited cause, got %v", cause) - } - <-done -} - -func TestWatchEngineProcessQuietOnDeliberateStop(t *testing.T) { - w := NewWithAPI(config.Config{}, nil) - w.logger = slog.New(slog.NewTextHandler(io.Discard, nil)) - runCtx, cancel := context.WithCancelCause(context.Background()) - defer cancel(nil) - w.cancelRun = cancel - w.markStopping() - - cmd := exec.Command("sleep", "30") - if err := cmd.Start(); err != nil { - t.Fatalf("start managed process stub: %v", err) - } - _ = cmd.Process.Kill() - w.watchEngineProcess("aria2", cmd) - - if cause := context.Cause(runCtx); cause != nil { - t.Fatalf("expected run context to stay live during a deliberate stop, got %v", cause) - } -} - func TestClearStaleSeedingReportsClearsUntrackedOnly(t *testing.T) { api := &recordingAPI{seedingTasks: []client.DownloadTask{ clientTaskWithStatus("stale-task", "completed"), clientTaskWithStatus("live-seed", "completed"), }} - w := NewWithAPI(config.Config{SeedEnabled: true}, api) + w := NewTaskRunnerWithAPI(config.Config{SeedEnabled: true}, api) w.logger = slog.New(slog.NewTextHandler(io.Discard, nil)) - w.engine = &recordingEngine{} - w.retainedSeeds = []retainedSeed{{taskID: "live-seed"}} + w.downloader = NewManagerWithDownloader(&recordingEngine{}) + w.seeds.retainedSeeds = []retainedSeed{{taskID: "live-seed"}} - w.clearStaleSeedingReports(context.Background()) + w.seeds.ClearStaleReports(context.Background()) if len(api.patchedIDs) != 1 || api.patchedIDs[0] != "stale-task" { t.Fatalf("expected exactly the untracked task to be cleared, got %v", api.patchedIDs) @@ -384,17 +333,17 @@ func TestClearStaleSeedingReportsClearsUntrackedOnly(t *testing.T) { func TestCleanupRetainedSeedReportsStopped(t *testing.T) { api := &recordingAPI{} - w := NewWithAPI(config.Config{SeedEnabled: true}, api) + w := NewTaskRunnerWithAPI(config.Config{SeedEnabled: true}, api) w.logger = slog.New(slog.NewTextHandler(io.Discard, nil)) - w.engine = &recordingEngine{} + w.downloader = NewManagerWithDownloader(&recordingEngine{}) cleaned := false - w.retainedSeeds = []retainedSeed{{ + w.seeds.retainedSeeds = []retainedSeed{{ taskID: "seed-task", engine: "aria2", cleanup: func(context.Context) error { cleaned = true; return nil }, }} - w.cleanupRetainedSeed(context.Background(), w.retainedSeeds[0], "expired") + w.seeds.cleanupRetainedSeed(context.Background(), w.seeds.retainedSeeds[0], "expired") if !cleaned { t.Fatal("expected engine cleanup to run") @@ -410,15 +359,15 @@ func TestCleanupRetainedSeedReportsStopped(t *testing.T) { func TestCleanupRetainedSeedsKeepsFailedUploadLocalResult(t *testing.T) { api := &recordingAPI{localResultTasks: []client.DownloadTask{clientTaskWithStatus("task-1", "failed")}} - w := NewWithAPI(config.Config{SeedEnabled: true, SeedCacheLimit: 1}, api) + w := NewTaskRunnerWithAPI(config.Config{SeedEnabled: true, SeedCacheLimit: 1}, api) w.logger = slog.New(slog.NewTextHandler(io.Discard, nil)) - w.engine = &recordingEngine{} + w.downloader = NewManagerWithDownloader(&recordingEngine{}) cleaned := false seedDir := t.TempDir() if err := os.WriteFile(filepath.Join(seedDir, "payload.bin"), []byte("downloaded payload"), 0o644); err != nil { t.Fatal(err) } - w.retainedSeeds = []retainedSeed{{ + w.seeds.retainedSeeds = []retainedSeed{{ taskID: "task-1", engine: "aria2", path: seedDir, @@ -426,13 +375,13 @@ func TestCleanupRetainedSeedsKeepsFailedUploadLocalResult(t *testing.T) { cleanup: func(context.Context) error { cleaned = true; return nil }, }} - w.cleanupRetainedSeeds(context.Background()) + w.seeds.Cleanup(context.Background()) if cleaned { t.Fatal("expected failed upload local result to be kept for retry") } - if len(w.retainedSeedSnapshot()) != 1 { - t.Fatalf("expected retained seed to remain tracked, got %d", len(w.retainedSeedSnapshot())) + if len(w.seeds.retainedSeedSnapshot()) != 1 { + t.Fatalf("expected retained seed to remain tracked, got %d", len(w.seeds.retainedSeedSnapshot())) } if api.localResultTasksCalls != 1 { t.Fatalf("expected local-result task lookup, got %d", api.localResultTasksCalls) @@ -441,11 +390,11 @@ func TestCleanupRetainedSeedsKeepsFailedUploadLocalResult(t *testing.T) { func TestReconcileEngineSeedsDoesNotFetchLocalResultTasksWithoutLocalSeeds(t *testing.T) { api := &recordingAPI{} - w := NewWithAPI(config.Config{SeedEnabled: true}, api) + w := NewTaskRunnerWithAPI(config.Config{SeedEnabled: true}, api) w.logger = slog.New(slog.NewTextHandler(io.Discard, nil)) - w.engine = &recordingEngine{} + w.downloader = NewManagerWithDownloader(&recordingEngine{}) - w.reconcileEngineSeeds(context.Background()) + w.seeds.Reconcile(context.Background()) if api.localResultTasksCalls != 0 { t.Fatalf("expected no local-result task fetch without local seeds, got %d", api.localResultTasksCalls) @@ -471,7 +420,7 @@ func TestReconcileEngineSeedsAdoptsUntrackedOrphans(t *testing.T) { } } - eng := &recordingEngine{listSeeds: []engine.Seed{ + eng := &recordingEngine{listSeeds: []Seed{ {Engine: "aria2", ID: "g1", InfoHash: "AAAA", Path: orphanDir}, {Engine: "aria2", ID: "g2", InfoHash: "BBBB", Path: trackedDir}, {Engine: "aria2", ID: "g3", InfoHash: "CCCC", Path: runningDir}, @@ -480,20 +429,20 @@ func TestReconcileEngineSeedsAdoptsUntrackedOrphans(t *testing.T) { }} // 'assigned-task' is still assigned/unfinished — it auto-seeds but hasn't been // uploaded yet, so the reconciler must NOT adopt it as a done seed. - w := NewWithAPI(config.Config{SeedEnabled: true, SeedDuration: time.Hour}, &recordingAPI{ + w := NewTaskRunnerWithAPI(config.Config{SeedEnabled: true, SeedDuration: time.Hour}, &recordingAPI{ localResultTasks: []client.DownloadTask{ clientTaskWithStatus("assigned-task", "assigned"), clientTaskWithStatus("failed-task", "failed"), }, }) w.logger = slog.New(slog.NewTextHandler(io.Discard, nil)) - w.engine = eng - w.retainedSeeds = []retainedSeed{{taskID: "tracked-task"}} + w.downloader = NewManagerWithDownloader(eng) + w.seeds.retainedSeeds = []retainedSeed{{taskID: "tracked-task"}} w.running["running-task"] = func(error) {} - w.reconcileEngineSeeds(context.Background()) + w.seeds.Reconcile(context.Background()) - for _, seed := range w.retainedSeedSnapshot() { + for _, seed := range w.seeds.retainedSeedSnapshot() { if seed.taskID == "assigned-task" { t.Fatal("expected an assigned (not-yet-uploaded) task's seed to be skipped, not adopted") } @@ -504,7 +453,7 @@ func TestReconcileEngineSeedsAdoptsUntrackedOrphans(t *testing.T) { got := map[string]retainedSeed{} trackedCount := 0 - for _, seed := range w.retainedSeedSnapshot() { + for _, seed := range w.seeds.retainedSeedSnapshot() { got[seed.taskID] = seed if seed.taskID == "tracked-task" { trackedCount++ @@ -529,7 +478,7 @@ func TestReconcileEngineSeedsAdoptsUntrackedOrphans(t *testing.T) { } func TestHeartbeatReportsAggregateTransferSpeeds(t *testing.T) { - w := NewWithAPI(config.Config{Engine: "auto", MaxConcurrentTasks: 5, DownloadDir: t.TempDir()}, &recordingAPI{}) + w := NewTaskRunnerWithAPI(config.Config{Engine: "auto", MaxConcurrentTasks: 5, DownloadDir: t.TempDir()}, &recordingAPI{}) if _, ok := w.startTask(context.Background(), "task-1"); !ok { t.Fatal("expected task-1 to start") @@ -562,47 +511,6 @@ func TestHeartbeatReportsAggregateTransferSpeeds(t *testing.T) { w.finish("task-2") } -func TestResolveEngineRejectsUnknownConfiguredEngine(t *testing.T) { - w := NewWithAPI(config.Config{Engine: "bad-engine"}, nil) - - err := w.resolveEngine(context.Background()) - if err == nil { - t.Fatal("expected unsupported engine error") - } - if !strings.Contains(err.Error(), "bad-engine") { - t.Fatalf("expected error to mention configured engine, got %v", err) - } -} - -func TestExplicitlyConfiguredExternalEngineRejectsAmbiguousRuntimeConfig(t *testing.T) { - _, _, err := explicitlyConfiguredExternalEngine(config.Config{ - Aria2Configured: true, - QBittorrentConfigured: true, - Aria2URL: config.DefaultAria2URL, - QBittorrentURL: config.DefaultQBittorrentURL, - }, nil) - if err == nil { - t.Fatal("expected ambiguous external runtime config error") - } -} - -func TestExplicitlyConfiguredExternalEngineSelectsConfiguredRuntime(t *testing.T) { - downloader, ok, err := explicitlyConfiguredExternalEngine(config.Config{ - Aria2Configured: true, - Aria2URL: "ws://aria2:6800/jsonrpc", - DownloadDir: t.TempDir(), - }, nil) - if err != nil { - t.Fatal(err) - } - if !ok { - t.Fatal("expected configured external runtime") - } - if downloader.Name() != "aria2" { - t.Fatalf("expected aria2 runtime, got %q", downloader.Name()) - } -} - func TestUploadFilePartSendsContentLength(t *testing.T) { path := writeTempFile(t, "hello world") file, err := os.Open(path) @@ -758,7 +666,7 @@ func TestCleanupDownloadedResultRemovesTaskDirForNestedDirectoryResult(t *testin t.Fatal(err) } - err := cleanupDownloadedResult(context.Background(), clientTaskWithStatus("task-1", "downloading"), engine.Result{ + err := cleanupDownloadedResult(context.Background(), clientTaskWithStatus("task-1", "downloading"), Result{ Path: resultDir, Name: "payload", IsDir: true, @@ -773,9 +681,9 @@ func TestCleanupDownloadedResultRemovesTaskDirForNestedDirectoryResult(t *testin func TestUploadFailurePersistsDownloadCheckpoint(t *testing.T) { api := &recordingAPI{createFolderErr: errors.New("unauthorized")} - w := NewWithAPI(config.Config{}, api) + w := NewTaskRunnerWithAPI(config.Config{}, api) resultPath := t.TempDir() - result := engine.Result{ + result := Result{ Path: resultPath, Name: "album", Size: 1234, @@ -811,6 +719,39 @@ func TestUploadFailurePersistsDownloadCheckpoint(t *testing.T) { } } +func TestDirectoryUploadFailureDeletesRemoteRootFolder(t *testing.T) { + api := &recordingAPI{ + createFolderDrafts: []client.ObjectDraft{{ID: "root-folder", Name: "album"}}, + createObjectDraft: client.ObjectDraft{ID: "file-object"}, + } + w := NewTaskRunnerWithAPI(config.Config{}, api) + resultPath := t.TempDir() + if err := os.WriteFile(filepath.Join(resultPath, "track.flac"), []byte("audio"), 0o644); err != nil { + t.Fatal(err) + } + + w.uploadAndComplete( + context.Background(), + slog.New(slog.NewTextHandler(io.Discard, nil)), + clientTaskWithUploadToken("task-1", "downloading"), + Result{ + Path: resultPath, + Name: "album", + Size: 5, + IsDir: true, + }, + nil, + ) + + if !reflect.DeepEqual(api.deletedObjects, []string{"root-folder"}) { + t.Fatalf("expected remote root folder cleanup, got %#v", api.deletedObjects) + } + failed := api.patches[len(api.patches)-1] + if failed.State() != "failed" { + t.Fatalf("expected failed status, got %q", failed.State()) + } +} + func TestWorkerLifecycleUploadFailurePreservesLocalResult(t *testing.T) { payloadPath := writeTempFile(t, "downloaded payload") payloadSize := int64(len("downloaded payload")) @@ -830,16 +771,16 @@ func TestWorkerLifecycleUploadFailurePreservesLocalResult(t *testing.T) { completeErrs: []error{errors.New("unauthorized"), nil}, } eng := &recordingEngine{ - downloadResult: engine.Result{Path: payloadPath, Name: "payload.bin", Size: payloadSize}, - taskSnapshot: engine.TaskSnapshot{ - State: engine.TaskStateCompleted, - Result: &engine.Result{Path: payloadPath, Name: "payload.bin", Size: payloadSize}, + downloadResult: Result{Path: payloadPath, Name: "payload.bin", Size: payloadSize}, + taskSnapshot: TaskSnapshot{ + State: TaskStateCompleted, + Result: &Result{Path: payloadPath, Name: "payload.bin", Size: payloadSize}, }, taskFound: true, } - first := NewWithAPI(config.Config{}, api) - first.engine = eng + first := NewTaskRunnerWithAPI(config.Config{}, api) + first.downloader = NewManagerWithDownloader(eng) first.process(context.Background(), clientHTTPTask("task-1", "assigned", "https://example.com/payload.bin", "payload.bin")) failed := lastPatchWithStatus(t, api.patches, "failed") if failed.Progress == nil || failed.Progress.Download == nil || failed.Progress.Download.Bytes != payloadSize { @@ -884,8 +825,8 @@ func TestWorkerLifecycleHTTPUploadFailurePreservesLocalResult(t *testing.T) { } downloadDir := t.TempDir() - first := NewWithAPI(config.Config{}, api) - first.engine = engine.HTTP{Dir: downloadDir} + first := NewTaskRunnerWithAPI(config.Config{}, api) + first.downloader = NewManagerWithDownloader(preservingHTTPDownloadEngine{dir: downloadDir}) first.process(context.Background(), clientHTTPTask("task-1", "assigned", downloadServer.URL+"/payload.bin", "payload.bin")) failed := lastPatchWithStatus(t, api.patches, "failed") if failed.Progress == nil || failed.Progress.Download == nil || failed.Progress.Download.Bytes != payloadSize { @@ -901,8 +842,8 @@ func TestWorkerLifecycleHTTPUploadFailurePreservesLocalResult(t *testing.T) { t.Fatalf("expected failed upload to preserve local task directory, stat err=%v", err) } - second := NewWithAPI(config.Config{}, api) - second.engine = engine.HTTP{Dir: downloadDir} + second := NewTaskRunnerWithAPI(config.Config{}, api) + second.downloader = NewManagerWithDownloader(preservingHTTPDownloadEngine{dir: downloadDir}) failedRuntime := failed.Runtime second.process(context.Background(), withRuntime( withDownloadCheckpoint(clientHTTPTask("task-1", "assigned", downloadServer.URL+"/payload.bin", "payload.bin"), payloadSize, &payloadSize), @@ -924,8 +865,8 @@ func TestWorkerLifecycleHTTPUploadFailurePreservesLocalResult(t *testing.T) { func TestUploadExistingResultInspectErrorFailsWithoutRedownloading(t *testing.T) { api := &recordingAPI{} eng := &recordingEngine{inspectErr: errors.New("runtime state is inconsistent")} - w := NewWithAPI(config.Config{}, api) - w.engine = eng + w := NewTaskRunnerWithAPI(config.Config{}, api) + w.downloader = NewManagerWithDownloader(eng) total := int64(100) w.process(context.Background(), withRuntime( @@ -945,8 +886,8 @@ func TestUploadExistingResultInspectErrorFailsWithoutRedownloading(t *testing.T) func TestUploadExistingResultInspectPanicFailsWithoutRedownloading(t *testing.T) { api := &recordingAPI{} eng := &recordingEngine{inspectPanic: "runtime invariant violated"} - w := NewWithAPI(config.Config{}, api) - w.engine = eng + w := NewTaskRunnerWithAPI(config.Config{}, api) + w.downloader = NewManagerWithDownloader(eng) total := int64(100) w.process(context.Background(), withRuntime( @@ -966,8 +907,8 @@ func TestUploadExistingResultInspectPanicFailsWithoutRedownloading(t *testing.T) func TestUploadExistingResultMissingRuntimeFailsWithoutRedownloading(t *testing.T) { api := &recordingAPI{} eng := &recordingEngine{taskFound: false} - w := NewWithAPI(config.Config{}, api) - w.engine = eng + w := NewTaskRunnerWithAPI(config.Config{}, api) + w.downloader = NewManagerWithDownloader(eng) total := int64(100) w.process(context.Background(), withRuntime( @@ -991,12 +932,12 @@ func TestUploadExistingResultIncompleteRuntimeResumesDownload(t *testing.T) { // panic/fail the task. api := &recordingAPI{} eng := &recordingEngine{ - taskSnapshot: engine.TaskSnapshot{State: engine.TaskStateDownloading, Downloaded: 10}, + taskSnapshot: TaskSnapshot{State: TaskStateDownloading, Downloaded: 10}, taskFound: true, downloadErr: errors.New("resumed via download path"), } - w := NewWithAPI(config.Config{}, api) - w.engine = eng + w := NewTaskRunnerWithAPI(config.Config{}, api) + w.downloader = NewManagerWithDownloader(eng) total := int64(100) w.process(context.Background(), withRuntime( @@ -1017,8 +958,8 @@ func TestUploadExistingResultIncompleteRuntimeResumesDownload(t *testing.T) { func TestDownloadShutdownMarksTaskInterrupted(t *testing.T) { api := &recordingAPI{} eng := &recordingEngine{downloadErr: context.Canceled} - w := NewWithAPI(config.Config{}, api) - w.engine = eng + w := NewTaskRunnerWithAPI(config.Config{}, api) + w.downloader = NewManagerWithDownloader(eng) w.process(context.Background(), clientTaskWithStatus("task-1", "downloading")) @@ -1040,7 +981,7 @@ func TestUploadShutdownMarksTaskInterrupted(t *testing.T) { api := &recordingAPI{ createObjectDraft: client.ObjectDraft{ID: "object-1", Name: "payload.bin", Upload: &client.ObjectUploadInstructions{SessionID: "session-1", PartSize: payloadSize, URLs: []string{"http://127.0.0.1:1"}}}, } - w := NewWithAPI(config.Config{}, api) + w := NewTaskRunnerWithAPI(config.Config{}, api) ctx, cancel := context.WithCancel(context.Background()) cancel() @@ -1048,7 +989,7 @@ func TestUploadShutdownMarksTaskInterrupted(t *testing.T) { ctx, slog.New(slog.NewTextHandler(io.Discard, nil)), clientTaskWithUploadToken("task-1", "downloading"), - engine.Result{Path: payloadPath, Name: "payload.bin", Size: int64(len("downloaded payload"))}, + Result{Path: payloadPath, Name: "payload.bin", Size: int64(len("downloaded payload"))}, nil, ) @@ -1079,7 +1020,7 @@ func TestSuspendedUploadPreservesLocalResult(t *testing.T) { api := &recordingAPI{ createObjectDraft: client.ObjectDraft{ID: "object-1", Name: "payload.bin", Upload: &client.ObjectUploadInstructions{SessionID: "session-1", PartSize: payloadSize, URLs: []string{"http://127.0.0.1:1"}}}, } - w := NewWithAPI(config.Config{}, api) + w := NewTaskRunnerWithAPI(config.Config{}, api) ctx, cancel := context.WithCancelCause(context.Background()) cancel(errTaskSuspended) @@ -1087,7 +1028,7 @@ func TestSuspendedUploadPreservesLocalResult(t *testing.T) { ctx, slog.New(slog.NewTextHandler(io.Discard, nil)), clientTaskWithUploadToken("task-1", "downloading"), - engine.Result{Path: payloadPath, Name: "payload.bin", Size: payloadSize}, + Result{Path: payloadPath, Name: "payload.bin", Size: payloadSize}, &client.DownloadTaskRuntime{Phase: "uploading"}, ) @@ -1107,9 +1048,9 @@ func TestSuspendedUploadPreservesLocalResult(t *testing.T) { func TestProcessRedownloadsWhenLocalResultWasCleaned(t *testing.T) { api := &recordingAPI{} eng := &recordingEngine{downloadErr: errors.New("redownload missing local result")} - w := NewWithAPI(config.Config{}, api) + w := NewTaskRunnerWithAPI(config.Config{}, api) w.logger = slog.New(slog.NewTextHandler(io.Discard, nil)) - w.engine = eng + w.downloader = NewManagerWithDownloader(eng) total := int64(100) w.process(context.Background(), withRuntime( @@ -1158,25 +1099,25 @@ func TestPausedAndInterruptedDownloadsPreservePartialFiles(t *testing.T) { path := filepath.Join(downloadDir, "task-1", "payload.bin") ready := make(chan struct{}, 1) eng := &recordingEngine{ - downloadFunc: func(ctx context.Context, task client.DownloadTask, progress engine.Progress) (engine.Result, error) { + downloadFunc: func(ctx context.Context, task DownloadTask, progress ProgressReporter) (Result, error) { if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { - return engine.Result{}, err + return Result{}, err } if err := os.WriteFile(path, []byte("partial"), 0o644); err != nil { - return engine.Result{}, err + return Result{}, err } ready <- struct{}{} <-ctx.Done() - return engine.Result{}, ctx.Err() + return Result{}, ctx.Err() }, - resetTaskFn: func(context.Context, client.DownloadTask) error { + resetTaskFn: func(context.Context, DownloadTask) error { return os.RemoveAll(filepath.Join(downloadDir, "task-1")) }, } - w := NewWithAPI(config.Config{DownloadDir: downloadDir}, api) + w := NewTaskRunnerWithAPI(config.Config{DownloadDir: downloadDir}, api) w.logger = slog.New(slog.NewTextHandler(io.Discard, nil)) - w.engine = eng + w.downloader = NewManagerWithDownloader(eng) ctx, cancel := context.WithCancel(context.Background()) ctxWithCause, cancelCause := context.WithCancelCause(ctx) @@ -1216,12 +1157,12 @@ func TestUploadETARoundsRemainingSeconds(t *testing.T) { func TestWithDownloadETAAddsFallback(t *testing.T) { total := int64(100) - detail := withDownloadRuntime(&client.DownloadTaskRuntime{Engine: "builtin"}, 25, &total, 20) + detail := withDownloadRuntime(&client.DownloadTaskRuntime{Engine: "http"}, 25, &total, 20) if detail == nil || detail.ETASeconds == nil || *detail.ETASeconds != 4 { t.Fatalf("expected fallback ETA 4, got %#v", detail) } - if detail.Engine != "builtin" { + if detail.Engine != "http" { t.Fatalf("expected existing detail fields to be preserved, got %#v", detail) } } @@ -1361,8 +1302,8 @@ func TestResetTaskForRestartAttemptResetsRuntimeAndRecordsAttempt(t *testing.T) task := clientTaskWithStatus("task-1", "assigned") task.Status.Attempt = 2 eng := &recordingEngine{} - w := NewWithAPI(config.Config{StateDir: stateDir}, nil) - w.engine = eng + w := NewTaskRunnerWithAPI(config.Config{StateDir: stateDir}, nil) + w.downloader = NewManagerWithDownloader(eng) if err := w.resetTaskForAttempt(context.Background(), task, w.logger); err != nil { t.Fatal(err) @@ -1395,8 +1336,8 @@ func TestResetTaskForRestartAttemptSkipsAlreadyRecordedAttempt(t *testing.T) { task := clientTaskWithStatus("task-1", "assigned") task.Status.Attempt = 2 eng := &recordingEngine{} - w := NewWithAPI(config.Config{StateDir: stateDir}, nil) - w.engine = eng + w := NewTaskRunnerWithAPI(config.Config{StateDir: stateDir}, nil) + w.downloader = NewManagerWithDownloader(eng) if err := w.resetTaskForAttempt(context.Background(), task, w.logger); err != nil { t.Fatal(err) @@ -1410,20 +1351,21 @@ func TestRetainSeedKeepsDownloadedResult(t *testing.T) { dir := t.TempDir() stateDir := t.TempDir() cleaned := false - w := NewWithAPI(config.Config{SeedEnabled: true, SeedDuration: time.Hour, StateDir: stateDir}, nil) + w := NewTaskRunnerWithAPI(config.Config{SeedEnabled: true, SeedDuration: time.Hour, StateDir: stateDir}, nil) - retained := w.retainSeed( + retained := w.seeds.Retain( + context.Background(), clientTask("task-1"), - engine.Result{ + Result{ Path: filepath.Join(dir, "result"), Size: 123, - Seed: &engine.Seed{ + Seed: &Seed{ Engine: "aria2", ID: "gid", InfoHash: "infohash", Path: dir, - Snapshot: func(context.Context) (engine.SeedSnapshot, error) { - return engine.SeedSnapshot{}, nil + Snapshot: func(context.Context) (SeedSnapshot, error) { + return SeedSnapshot{}, nil }, Cleanup: func(context.Context) error { cleaned = true @@ -1440,8 +1382,8 @@ func TestRetainSeedKeepsDownloadedResult(t *testing.T) { if cleaned { t.Fatal("expected retained seed cleanup to be deferred") } - if len(w.retainedSeedSnapshot()) != 1 { - t.Fatalf("expected one retained seed, got %d", len(w.retainedSeedSnapshot())) + if len(w.seeds.retainedSeedSnapshot()) != 1 { + t.Fatalf("expected one retained seed, got %d", len(w.seeds.retainedSeedSnapshot())) } ledger, err := loadSeedLedger(stateDir) if err != nil { @@ -1458,21 +1400,22 @@ func TestRetainedSeedExpiresWhenLedgerPersistenceFails(t *testing.T) { t.Fatal(err) } cleaned := false - w := NewWithAPI(config.Config{SeedEnabled: true, SeedDuration: time.Hour, StateDir: stateFile}, &recordingAPI{}) - w.engine = &recordingEngine{} + w := NewTaskRunnerWithAPI(config.Config{SeedEnabled: true, SeedDuration: time.Hour, StateDir: stateFile}, &recordingAPI{}) + w.downloader = NewManagerWithDownloader(&recordingEngine{}) - retained := w.retainSeed( + retained := w.seeds.Retain( + context.Background(), clientTask("task-1"), - engine.Result{ + Result{ Path: filepath.Join(t.TempDir(), "result"), Size: 123, - Seed: &engine.Seed{ + Seed: &Seed{ Engine: "aria2", ID: "gid", InfoHash: "infohash", Path: t.TempDir(), - Snapshot: func(context.Context) (engine.SeedSnapshot, error) { - return engine.SeedSnapshot{}, nil + Snapshot: func(context.Context) (SeedSnapshot, error) { + return SeedSnapshot{}, nil }, Cleanup: func(context.Context) error { cleaned = true @@ -1486,31 +1429,31 @@ func TestRetainedSeedExpiresWhenLedgerPersistenceFails(t *testing.T) { if !retained { t.Fatal("expected seed to remain tracked in memory") } - if len(w.retainedSeedSnapshot()) != 1 { - t.Fatalf("expected retained seed despite ledger failure, got %d", len(w.retainedSeedSnapshot())) + if len(w.seeds.retainedSeedSnapshot()) != 1 { + t.Fatalf("expected retained seed despite ledger failure, got %d", len(w.seeds.retainedSeedSnapshot())) } - w.retainedSeeds[0].expiresAt = time.Now().Add(-time.Second) + w.seeds.retainedSeeds[0].expiresAt = time.Now().Add(-time.Second) - w.cleanupRetainedSeeds(context.Background()) + w.seeds.Cleanup(context.Background()) if !cleaned { t.Fatal("expected in-memory retained seed to expire and clean up") } - if len(w.retainedSeedSnapshot()) != 0 { - t.Fatalf("expected expired seed to be removed from memory, got %d", len(w.retainedSeedSnapshot())) + if len(w.seeds.retainedSeedSnapshot()) != 0 { + t.Fatalf("expected expired seed to be removed from memory, got %d", len(w.seeds.retainedSeedSnapshot())) } } func TestReportRetainedSeedsCleansMissingSeed(t *testing.T) { cleaned := false - w := NewWithAPI(config.Config{SeedEnabled: true}, &recordingAPI{}) - w.engine = &recordingEngine{} - w.retainedSeeds = []retainedSeed{{ + w := NewTaskRunnerWithAPI(config.Config{SeedEnabled: true}, &recordingAPI{}) + w.downloader = NewManagerWithDownloader(&recordingEngine{}) + w.seeds.retainedSeeds = []retainedSeed{{ taskID: "task-1", engine: "aria2", seedID: "missing-gid", - snapshot: func(context.Context) (engine.SeedSnapshot, error) { - return engine.SeedSnapshot{}, errors.New("GID missing-gid is not found") + snapshot: func(context.Context) (SeedSnapshot, error) { + return SeedSnapshot{}, errors.New("GID missing-gid is not found") }, cleanup: func(context.Context) error { cleaned = true @@ -1518,44 +1461,44 @@ func TestReportRetainedSeedsCleansMissingSeed(t *testing.T) { }, }} - w.reportRetainedSeeds(context.Background()) + w.seeds.Report(context.Background()) if !cleaned { t.Fatal("expected missing retained seed to be cleaned") } - if got := len(w.retainedSeedSnapshot()); got != 0 { + if got := len(w.seeds.retainedSeedSnapshot()); got != 0 { t.Fatalf("expected missing retained seed to be removed, got %d", got) } } func TestReportRetainedSeedsSendsCompleteSeedingSnapshot(t *testing.T) { api := &recordingAPI{} - w := NewWithAPI(config.Config{SeedEnabled: true}, api) + w := NewTaskRunnerWithAPI(config.Config{SeedEnabled: true}, api) total := int64(100) eta := int64(30) uploaded := int64(12) - w.retainedSeeds = []retainedSeed{{ + w.seeds.retainedSeeds = []retainedSeed{{ taskID: "task-1", engine: "aria2", seedID: "gid", size: total, downloaded: total, - snapshot: func(context.Context) (engine.SeedSnapshot, error) { - return engine.SeedSnapshot{ + snapshot: func(context.Context) (SeedSnapshot, error) { + return SeedSnapshot{ Downloaded: total, Total: &total, - Runtime: &client.DownloadTaskRuntime{ + Runtime: &TaskRuntime{ Engine: "aria2", Phase: "seeding", ETASeconds: &eta, - Seeding: &client.DownloadTaskSeedingRuntime{UploadedBytes: &uploaded}, + Seeding: &SeedingRuntime{UploadedBytes: &uploaded}, }, }, nil }, cleanup: func(context.Context) error { return nil }, }} - w.reportRetainedSeeds(context.Background()) + w.seeds.Report(context.Background()) patch := api.patches[len(api.patches)-1] if patch.Runtime == nil || patch.Runtime.ETASeconds != nil { @@ -1573,37 +1516,37 @@ func TestReportRetainedSeedsSendsCompleteSeedingSnapshot(t *testing.T) { func TestReportRetainedSeedsSkipsUnchangedSnapshot(t *testing.T) { api := &recordingAPI{} - w := NewWithAPI(config.Config{SeedEnabled: true}, api) + w := NewTaskRunnerWithAPI(config.Config{SeedEnabled: true}, api) total := int64(100) uploaded := int64(12) - w.retainedSeeds = []retainedSeed{{ + w.seeds.retainedSeeds = []retainedSeed{{ taskID: "task-1", engine: "aria2", seedID: "gid", size: total, downloaded: total, - snapshot: func(context.Context) (engine.SeedSnapshot, error) { - return engine.SeedSnapshot{ + snapshot: func(context.Context) (SeedSnapshot, error) { + return SeedSnapshot{ Downloaded: total, Total: &total, - Runtime: &client.DownloadTaskRuntime{ + Runtime: &TaskRuntime{ Engine: "aria2", Phase: "seeding", - Seeding: &client.DownloadTaskSeedingRuntime{UploadedBytes: &uploaded}, + Seeding: &SeedingRuntime{UploadedBytes: &uploaded}, }, }, nil }, cleanup: func(context.Context) error { return nil }, }} - w.reportRetainedSeeds(context.Background()) - w.reportRetainedSeeds(context.Background()) + w.seeds.Report(context.Background()) + w.seeds.Report(context.Background()) if len(api.patches) != 1 { t.Fatalf("expected unchanged seed snapshot to be reported once, got %d patches", len(api.patches)) } uploaded = 24 - w.reportRetainedSeeds(context.Background()) + w.seeds.Report(context.Background()) if len(api.patches) != 2 { t.Fatalf("expected changed seed snapshot to be reported again, got %d patches", len(api.patches)) } @@ -1611,31 +1554,31 @@ func TestReportRetainedSeedsSkipsUnchangedSnapshot(t *testing.T) { func TestReportRetainedSeedsStoppedClearsSeedingPhase(t *testing.T) { api := &recordingAPI{} - w := NewWithAPI(config.Config{SeedEnabled: true}, api) + w := NewTaskRunnerWithAPI(config.Config{SeedEnabled: true}, api) total := int64(100) uploaded := int64(40) active := true - w.retainedSeeds = []retainedSeed{{ + w.seeds.retainedSeeds = []retainedSeed{{ taskID: "task-1", engine: "aria2", seedID: "gid", size: total, downloaded: total, - snapshot: func(context.Context) (engine.SeedSnapshot, error) { - return engine.SeedSnapshot{ + snapshot: func(context.Context) (SeedSnapshot, error) { + return SeedSnapshot{ Downloaded: total, Total: &total, - Runtime: &client.DownloadTaskRuntime{ + Runtime: &TaskRuntime{ Engine: "aria2", Phase: "seeding", - Seeding: &client.DownloadTaskSeedingRuntime{Active: &active, UploadedBytes: &uploaded}, + Seeding: &SeedingRuntime{Active: &active, UploadedBytes: &uploaded}, }, }, nil }, cleanup: func(context.Context) error { return nil }, }} - w.reportRetainedSeedsStopped(context.Background()) + w.seeds.ReportStopped(context.Background()) patch := api.patches[len(api.patches)-1] if patch.Runtime == nil || patch.Runtime.Phase != "completed" { @@ -1670,24 +1613,24 @@ func TestRestoreRetainedSeedsLoadsLedger(t *testing.T) { }}}); err != nil { t.Fatal(err) } - eng := &recordingEngine{restoreSeed: &engine.Seed{ + eng := &recordingEngine{name: "aria2", restoreSeed: &Seed{ Engine: "aria2", ID: "new-gid", InfoHash: "abc123", Path: seedPath, - Snapshot: func(context.Context) (engine.SeedSnapshot, error) { - return engine.SeedSnapshot{}, nil + Snapshot: func(context.Context) (SeedSnapshot, error) { + return SeedSnapshot{}, nil }, Cleanup: func(context.Context) error { return nil }, }} - w := NewWithAPI(config.Config{SeedEnabled: true, StateDir: stateDir}, nil) - w.engine = eng + w := NewTaskRunnerWithAPI(config.Config{SeedEnabled: true, StateDir: stateDir}, nil) + w.downloader = NewManagerWithDownloader(eng) - w.restoreRetainedSeeds(context.Background()) + w.seeds.Restore(context.Background()) - seeds := w.retainedSeedSnapshot() + seeds := w.seeds.retainedSeedSnapshot() if len(seeds) != 1 { t.Fatalf("expected one restored seed, got %#v", seeds) } @@ -1714,25 +1657,25 @@ func TestRestoreRetainedSeedsDoesNotDuplicateAlreadyRestoredSeed(t *testing.T) { }}}); err != nil { t.Fatal(err) } - eng := &recordingEngine{restoreSeed: &engine.Seed{ + eng := &recordingEngine{name: "aria2", restoreSeed: &Seed{ Engine: "aria2", ID: "gid", InfoHash: "abc123", Path: seedPath, - Snapshot: func(context.Context) (engine.SeedSnapshot, error) { - return engine.SeedSnapshot{}, nil + Snapshot: func(context.Context) (SeedSnapshot, error) { + return SeedSnapshot{}, nil }, Cleanup: func(context.Context) error { return nil }, }} - w := NewWithAPI(config.Config{SeedEnabled: true, StateDir: stateDir}, nil) - w.engine = eng + w := NewTaskRunnerWithAPI(config.Config{SeedEnabled: true, StateDir: stateDir}, nil) + w.downloader = NewManagerWithDownloader(eng) - w.restoreRetainedSeeds(context.Background()) - w.restoreRetainedSeeds(context.Background()) + w.seeds.Restore(context.Background()) + w.seeds.Restore(context.Background()) - if got := len(w.retainedSeedSnapshot()); got != 1 { + if got := len(w.seeds.retainedSeedSnapshot()); got != 1 { t.Fatalf("expected one restored seed, got %d", got) } if eng.restoreCalls != 1 { @@ -1742,16 +1685,16 @@ func TestRestoreRetainedSeedsDoesNotDuplicateAlreadyRestoredSeed(t *testing.T) { func TestCleanupRetainedSeedsRemovesExpiredSeed(t *testing.T) { dir := t.TempDir() - w := NewWithAPI(config.Config{SeedEnabled: true, SeedDuration: time.Hour}, &recordingAPI{}) + w := NewTaskRunnerWithAPI(config.Config{SeedEnabled: true, SeedDuration: time.Hour}, &recordingAPI{}) cleaned := false - w.retainedSeeds = []retainedSeed{{ + w.seeds.retainedSeeds = []retainedSeed{{ taskID: "task-1", engine: "aria2", seedID: "gid", path: dir, expiresAt: time.Now().Add(-time.Second), - snapshot: func(context.Context) (engine.SeedSnapshot, error) { - return engine.SeedSnapshot{}, nil + snapshot: func(context.Context) (SeedSnapshot, error) { + return SeedSnapshot{}, nil }, cleanup: func(context.Context) error { cleaned = true @@ -1759,31 +1702,31 @@ func TestCleanupRetainedSeedsRemovesExpiredSeed(t *testing.T) { }, }} - w.cleanupRetainedSeeds(context.Background()) + w.seeds.Cleanup(context.Background()) if !cleaned { t.Fatal("expected expired seed to be cleaned") } - if len(w.retainedSeedSnapshot()) != 0 { - t.Fatalf("expected retained seed to be removed, got %d", len(w.retainedSeedSnapshot())) + if len(w.seeds.retainedSeedSnapshot()) != 0 { + t.Fatalf("expected retained seed to be removed, got %d", len(w.seeds.retainedSeedSnapshot())) } } func TestCleanupRetainedSeedsRemovesSeedAfterRatio(t *testing.T) { dir := t.TempDir() - w := NewWithAPI(config.Config{SeedEnabled: true, SeedRatio: 1.5}, &recordingAPI{}) + w := NewTaskRunnerWithAPI(config.Config{SeedEnabled: true, SeedRatio: 1.5}, &recordingAPI{}) cleaned := false uploaded := int64(151) - w.retainedSeeds = []retainedSeed{{ + w.seeds.retainedSeeds = []retainedSeed{{ taskID: "task-1", engine: "aria2", seedID: "gid", path: dir, downloaded: 100, - snapshot: func(context.Context) (engine.SeedSnapshot, error) { - return engine.SeedSnapshot{ - Runtime: &client.DownloadTaskRuntime{ - Seeding: &client.DownloadTaskSeedingRuntime{UploadedBytes: &uploaded}, + snapshot: func(context.Context) (SeedSnapshot, error) { + return SeedSnapshot{ + Runtime: &TaskRuntime{ + Seeding: &SeedingRuntime{UploadedBytes: &uploaded}, }, }, nil }, @@ -1793,13 +1736,13 @@ func TestCleanupRetainedSeedsRemovesSeedAfterRatio(t *testing.T) { }, }} - w.cleanupRetainedSeeds(context.Background()) + w.seeds.Cleanup(context.Background()) if !cleaned { t.Fatal("expected ratio seed to be cleaned") } - if len(w.retainedSeedSnapshot()) != 0 { - t.Fatalf("expected retained seed to be removed, got %d", len(w.retainedSeedSnapshot())) + if len(w.seeds.retainedSeedSnapshot()) != 0 { + t.Fatalf("expected retained seed to be removed, got %d", len(w.seeds.retainedSeedSnapshot())) } } @@ -1821,16 +1764,16 @@ func TestCleanupRetainedSeedsRemovesOldestWhenCacheLimitExceeded(t *testing.T) { } var cleaned []string - w := NewWithAPI(config.Config{SeedEnabled: true, SeedCacheLimit: 3}, &recordingAPI{}) - w.retainedSeeds = []retainedSeed{ + w := NewTaskRunnerWithAPI(config.Config{SeedEnabled: true, SeedCacheLimit: 3}, &recordingAPI{}) + w.seeds.retainedSeeds = []retainedSeed{ { taskID: "old", engine: "qbittorrent", seedID: "old-hash", path: oldDir, retainedAt: time.Now().Add(-time.Hour), - snapshot: func(context.Context) (engine.SeedSnapshot, error) { - return engine.SeedSnapshot{}, nil + snapshot: func(context.Context) (SeedSnapshot, error) { + return SeedSnapshot{}, nil }, cleanup: func(context.Context) error { cleaned = append(cleaned, "old") @@ -1843,8 +1786,8 @@ func TestCleanupRetainedSeedsRemovesOldestWhenCacheLimitExceeded(t *testing.T) { seedID: "new-hash", path: newDir, retainedAt: time.Now(), - snapshot: func(context.Context) (engine.SeedSnapshot, error) { - return engine.SeedSnapshot{}, nil + snapshot: func(context.Context) (SeedSnapshot, error) { + return SeedSnapshot{}, nil }, cleanup: func(context.Context) error { cleaned = append(cleaned, "new") @@ -1853,12 +1796,12 @@ func TestCleanupRetainedSeedsRemovesOldestWhenCacheLimitExceeded(t *testing.T) { }, } - w.cleanupRetainedSeeds(context.Background()) + w.seeds.Cleanup(context.Background()) if len(cleaned) != 1 || cleaned[0] != "old" { t.Fatalf("expected oldest seed to be cleaned first, got %v", cleaned) } - seeds := w.retainedSeedSnapshot() + seeds := w.seeds.retainedSeedSnapshot() if len(seeds) != 1 || seeds[0].taskID != "new" { t.Fatalf("expected newest seed to remain, got %+v", seeds) } @@ -1947,17 +1890,20 @@ func findPatchWithStatus(patches []client.TaskPatch, status string) (client.Task } type recordingEngine struct { - downloadResult engine.Result + name string + downloadResult Result downloadErr error - downloadFunc func(context.Context, client.DownloadTask, engine.Progress) (engine.Result, error) + downloadFunc func(context.Context, DownloadTask, ProgressReporter) (Result, error) resetErr error - resetTaskFn func(context.Context, client.DownloadTask) error - taskSnapshot engine.TaskSnapshot + resetTaskFn func(context.Context, DownloadTask) error + taskSnapshot TaskSnapshot inspectErr error inspectPanic any taskFound bool - restoreSeed *engine.Seed - listSeeds []engine.Seed + restoreSeed *Seed + restoreErr error + listSeeds []Seed + listSeedsErr error downloadCalls int resetCalls int inspectCalls int @@ -1965,40 +1911,135 @@ type recordingEngine struct { listSeedsCalls int } +type preservingHTTPDownloadEngine struct { + dir string +} + +func (e preservingHTTPDownloadEngine) Name() string { + return "recording-http" +} + +func (e preservingHTTPDownloadEngine) Capabilities() Capabilities { + return Capabilities{SourceTypes: []string{"http"}} +} + +func (e preservingHTTPDownloadEngine) Start(context.Context) error { + return nil +} + +func (e preservingHTTPDownloadEngine) Stop(context.Context) error { + return nil +} + +func (e preservingHTTPDownloadEngine) Check(context.Context) error { + return nil +} + +func (e preservingHTTPDownloadEngine) InspectTask(_ context.Context, task DownloadTask) (TaskSnapshot, bool, error) { + path := filepath.Join(e.dir, task.ID, task.Name()) + info, err := os.Stat(path) + if err != nil { + if os.IsNotExist(err) { + return TaskSnapshot{}, false, nil + } + return TaskSnapshot{}, false, err + } + if info.IsDir() { + return TaskSnapshot{}, false, nil + } + size := info.Size() + result := Result{Path: path, Name: task.Name(), Size: size} + return TaskSnapshot{ + State: TaskStateCompleted, + Downloaded: size, + Total: &size, + Runtime: &TaskRuntime{Engine: "recording-http", Phase: "completed"}, + Result: &result, + }, true, nil +} + +func (e preservingHTTPDownloadEngine) Download(ctx context.Context, task DownloadTask, progress ProgressReporter) (Result, error) { + res, err := http.Get(task.SourceURI()) + if err != nil { + return Result{}, err + } + defer res.Body.Close() + if res.StatusCode < 200 || res.StatusCode >= 300 { + return Result{}, errors.New(res.Status) + } + dir := filepath.Join(e.dir, task.ID) + if err := os.MkdirAll(dir, 0o755); err != nil { + return Result{}, err + } + path := filepath.Join(dir, task.Name()) + file, err := os.Create(path) + if err != nil { + return Result{}, err + } + size, copyErr := io.Copy(file, res.Body) + closeErr := file.Close() + if copyErr != nil { + return Result{}, copyErr + } + if closeErr != nil { + return Result{}, closeErr + } + total := size + if err := progress(ProgressUpdate{ + Downloaded: size, + Total: &total, + Runtime: &TaskRuntime{Engine: "recording-http", Phase: "completed"}, + }); err != nil { + return Result{}, err + } + return Result{Path: path, Name: task.Name(), Size: size}, nil +} + func (e *recordingEngine) Name() string { + if e.name != "" { + return e.name + } return "recording" } -func (e *recordingEngine) Capabilities() []string { - return []string{"http", "magnet", "torrent"} +func (e *recordingEngine) Capabilities() Capabilities { + return Capabilities{SourceTypes: []string{"http", "magnet", "torrent", "torrent_url"}} +} + +func (e *recordingEngine) Start(context.Context) error { + return nil +} + +func (e *recordingEngine) Stop(context.Context) error { + return nil } func (e *recordingEngine) Check(context.Context) error { return nil } -func (e *recordingEngine) InspectTask(context.Context, client.DownloadTask) (engine.TaskSnapshot, bool, error) { +func (e *recordingEngine) InspectTask(context.Context, DownloadTask) (TaskSnapshot, bool, error) { e.inspectCalls++ if e.inspectPanic != nil { panic(e.inspectPanic) } if e.inspectErr != nil { - return engine.TaskSnapshot{}, false, e.inspectErr + return TaskSnapshot{}, false, e.inspectErr } return e.taskSnapshot, e.taskFound, nil } -func (e *recordingEngine) RestoreSeed(context.Context, engine.SeedRef) (*engine.Seed, error) { +func (e *recordingEngine) RestoreSeed(context.Context, SeedRef) (*Seed, error) { e.restoreCalls++ - return e.restoreSeed, nil + return e.restoreSeed, e.restoreErr } -func (e *recordingEngine) ListSeeds(context.Context) ([]engine.Seed, error) { +func (e *recordingEngine) ListSeeds(context.Context) ([]Seed, error) { e.listSeedsCalls++ - return e.listSeeds, nil + return e.listSeeds, e.listSeedsErr } -func (e *recordingEngine) ResetTask(ctx context.Context, task client.DownloadTask) error { +func (e *recordingEngine) ResetTask(ctx context.Context, task DownloadTask) error { e.resetCalls++ if e.resetTaskFn != nil { return e.resetTaskFn(ctx, task) @@ -2006,7 +2047,7 @@ func (e *recordingEngine) ResetTask(ctx context.Context, task client.DownloadTas return e.resetErr } -func (e *recordingEngine) Download(ctx context.Context, task client.DownloadTask, progress engine.Progress) (engine.Result, error) { +func (e *recordingEngine) Download(ctx context.Context, task DownloadTask, progress ProgressReporter) (Result, error) { e.downloadCalls++ if e.downloadFunc != nil { return e.downloadFunc(ctx, task, progress) @@ -2021,16 +2062,24 @@ type recordingAPI struct { controlTasks []client.DownloadTask assignedTasks []client.DownloadTask localResultTasks []client.DownloadTask + heartbeatErr error + updateErr error + localResultErr error assignedTasksCalls int localResultTasksCalls int nextPollAfterSeconds int suspendDownloading bool createFolderErr error + createFolderDrafts []client.ObjectDraft + deletedObjects []string createObjectDraft client.ObjectDraft completeErrs []error } func (a *recordingAPI) Heartbeat(context.Context, client.Heartbeat) (client.HeartbeatResult, error) { + if a.heartbeatErr != nil { + return client.HeartbeatResult{}, a.heartbeatErr + } nextPoll := a.nextPollAfterSeconds if nextPoll == 0 { nextPoll = 5 @@ -2045,6 +2094,9 @@ func (a *recordingAPI) AssignedTasks(context.Context) ([]client.DownloadTask, er func (a *recordingAPI) LocalResultTasks(context.Context) ([]client.DownloadTask, error) { a.localResultTasksCalls++ + if a.localResultErr != nil { + return nil, a.localResultErr + } return a.localResultTasks, nil } @@ -2053,6 +2105,9 @@ func (a *recordingAPI) SeedingTasks(context.Context) ([]client.DownloadTask, err } func (a *recordingAPI) UpdateTask(_ context.Context, id string, patch client.TaskPatch) (client.DownloadTask, error) { + if a.updateErr != nil { + return client.DownloadTask{}, a.updateErr + } a.patches = append(a.patches, patch) a.patchedIDs = append(a.patchedIDs, id) state := patch.State() @@ -2097,7 +2152,15 @@ func applyTaskPatch(task client.DownloadTask, patch client.TaskPatch) client.Dow } func (a *recordingAPI) CreateFolder(context.Context, string, string, string) (client.ObjectDraft, error) { - return client.ObjectDraft{}, a.createFolderErr + if a.createFolderErr != nil { + return client.ObjectDraft{}, a.createFolderErr + } + if len(a.createFolderDrafts) > 0 { + draft := a.createFolderDrafts[0] + a.createFolderDrafts = a.createFolderDrafts[1:] + return draft, nil + } + return client.ObjectDraft{ID: "folder-1", Name: "folder"}, nil } func (a *recordingAPI) CreateObject(context.Context, string, string, int64, string) (client.ObjectDraft, error) { @@ -2116,3 +2179,8 @@ func (a *recordingAPI) CompleteObjectUpload(context.Context, string, string, str func (a *recordingAPI) AbortObjectUploadSession(context.Context, string, string, string) error { return nil } + +func (a *recordingAPI) DeleteObject(_ context.Context, _ string, id string) error { + a.deletedObjects = append(a.deletedObjects, id) + return nil +} diff --git a/cmd/internal/worker/uploader.go b/cmd/internal/downloader/uploader.go similarity index 55% rename from cmd/internal/worker/uploader.go rename to cmd/internal/downloader/uploader.go index 0eab3525..7c85dc79 100644 --- a/cmd/internal/worker/uploader.go +++ b/cmd/internal/downloader/uploader.go @@ -1,4 +1,4 @@ -package worker +package downloader import ( "context" @@ -8,11 +8,13 @@ import ( "net/http" "os" "path" + "path/filepath" + "sort" "strings" "time" "github.com/saltbo/zpan/internal/client" - "github.com/saltbo/zpan/internal/engine" + "github.com/saltbo/zpan/pkg/system" ) type uploadProgress struct { @@ -22,23 +24,43 @@ type uploadProgress struct { lastBytes int64 } -func (w *Worker) uploadResult( +type Uploader struct { + api apiClient + reportSpeed func(taskID string, speeds transferSpeeds) +} + +func NewUploader(api apiClient, reportSpeed func(taskID string, speeds transferSpeeds)) *Uploader { + return &Uploader{api: api, reportSpeed: reportSpeed} +} + +func (u *Uploader) Upload( ctx context.Context, log *slog.Logger, task client.DownloadTask, - result engine.Result, + result Result, ) (string, error) { if !result.IsDir { progress := &uploadProgress{totalBytes: result.Size, lastAt: time.Now()} - return w.uploadSingleFile(ctx, log, task, result.Path, result.Name, result.Size, task.TargetFolder(), progress) + return u.uploadSingleFile(ctx, log, task, result.Path, result.Name, result.Size, task.TargetFolder(), progress) } progress := &uploadProgress{totalBytes: result.Size, lastAt: time.Now()} log.Info("creating remote folder", "name", result.Name, "size", result.Size, "target_folder", task.TargetFolder()) - root, err := w.createFolder(ctx, task.UploadToken(), result.Name, task.TargetFolder()) + root, err := u.createFolder(ctx, task.UploadToken(), result.Name, task.TargetFolder()) if err != nil { return "", fmt.Errorf("create remote folder: %w", err) } + completed := false + defer func() { + if completed { + return + } + cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second) + defer cancel() + if deleteErr := u.deleteObject(cleanupCtx, task.UploadToken(), root.ID); deleteErr != nil { + log.Warn("failed to delete remote folder after directory upload failure", "object_id", root.ID, "name", root.Name, "error", deleteErr) + } + }() rootPath := joinObjectPath(task.TargetFolder(), root.Name) entries, err := collectDirectoryEntries(result.Path) if err != nil { @@ -48,19 +70,20 @@ func (w *Worker) uploadResult( parent := joinObjectPath(rootPath, path.Dir(entry.relativePath)) if entry.isDir { log.Debug("creating remote subfolder", "name", entry.name, "parent", parent) - if _, err := w.createFolder(ctx, task.UploadToken(), entry.name, parent); err != nil { + if _, err := u.createFolder(ctx, task.UploadToken(), entry.name, parent); err != nil { return "", fmt.Errorf("create remote subfolder %s: %w", entry.relativePath, err) } continue } - if _, err := w.uploadSingleFile(ctx, log, task, entry.path, entry.name, entry.size, parent, progress); err != nil { + if _, err := u.uploadSingleFile(ctx, log, task, entry.path, entry.name, entry.size, parent, progress); err != nil { return "", err } } + completed = true return root.ID, nil } -func (w *Worker) uploadSingleFile( +func (u *Uploader) uploadSingleFile( ctx context.Context, log *slog.Logger, task client.DownloadTask, @@ -71,7 +94,7 @@ func (w *Worker) uploadSingleFile( progress *uploadProgress, ) (string, error) { log.Info("creating remote object", "name", name, "size", size, "target_folder", parent) - draft, err := w.createObject(ctx, task.UploadToken(), name, size, parent) + draft, err := u.createObject(ctx, task.UploadToken(), name, size, parent) if err != nil { return "", fmt.Errorf("create remote object: %w", err) } @@ -79,7 +102,7 @@ func (w *Worker) uploadSingleFile( return "", fmt.Errorf("create remote object %s: missing upload instructions", draft.ID) } log.Info("uploading file to object storage", "object_id", draft.ID, "path", path, "parts", len(draft.Upload.URLs)) - if err := w.uploadObjectSlices(ctx, log, task, draft, path, size, progress); err != nil { + if err := u.uploadObjectSlices(ctx, log, task, draft, path, size, progress); err != nil { return "", fmt.Errorf("upload object %s: %w", draft.ID, err) } return draft.ID, nil @@ -88,7 +111,7 @@ func (w *Worker) uploadSingleFile( // uploadObjectSlices runs the uniform upload: PUT each presigned slice (1 URL = // single PutObject, N URLs = multipart), read each ETag, then finalize. On any // failure it aborts the session, which also discards the draft. -func (w *Worker) uploadObjectSlices( +func (u *Uploader) uploadObjectSlices( ctx context.Context, log *slog.Logger, task client.DownloadTask, @@ -105,7 +128,7 @@ func (w *Worker) uploadObjectSlices( } abortCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second) defer cancel() - if abortErr := w.abortObjectUploadSession(abortCtx, task.UploadToken(), draft.ID, upload.SessionID); abortErr != nil { + if abortErr := u.abortObjectUploadSession(abortCtx, task.UploadToken(), draft.ID, upload.SessionID); abortErr != nil { log.Warn("failed to abort upload session", "object_id", draft.ID, "upload_session_id", upload.SessionID, "error", abortErr) } }() @@ -125,7 +148,7 @@ func (w *Worker) uploadObjectSlices( length = remaining } etag, err := uploadFilePart(ctx, url, file, offset, length, func(written int64) error { - return w.reportUploadProgress(ctx, log, task, progress, written) + return u.reportUploadProgress(ctx, log, task, progress, written) }) if err != nil { return fmt.Errorf("upload part %d: %w", partNumber, err) @@ -135,14 +158,14 @@ func (w *Worker) uploadObjectSlices( } parts = append(parts, client.CompletedObjectUploadPart{PartNumber: partNumber, ETag: etag}) } - if err := w.completeObjectUpload(ctx, task.UploadToken(), draft.ID, upload.SessionID, parts); err != nil { + if err := u.completeObjectUpload(ctx, task.UploadToken(), draft.ID, upload.SessionID, parts); err != nil { return fmt.Errorf("complete upload: %w", err) } completed = true return nil } -func (w *Worker) reportUploadProgress( +func (u *Uploader) reportUploadProgress( ctx context.Context, log *slog.Logger, task client.DownloadTask, @@ -159,7 +182,9 @@ func (w *Worker) reportUploadProgress( if elapsed > 0 { bps = int64(float64(progress.uploaded-progress.lastBytes) / elapsed) } - w.setTaskTransferSpeed(task.ID, transferSpeeds{uploadBps: bps}) + if u.reportSpeed != nil { + u.reportSpeed(task.ID, transferSpeeds{uploadBps: bps}) + } detail := task.Runtime() if detail == nil { detail = &client.DownloadTaskRuntime{} @@ -171,7 +196,7 @@ func (w *Worker) reportUploadProgress( Upload: *transferProgress(progress.uploaded, &progress.totalBytes, bps), } detail.Seeding = nil - _, err := w.updateTask(ctx, task.ID, client.TaskPatch{ + _, err := u.updateTask(ctx, task.ID, client.TaskPatch{ Status: "uploading", Progress: uploadProgressPatch(progress.uploaded, progress.totalBytes, bps), Runtime: detail, @@ -186,6 +211,121 @@ func (w *Worker) reportUploadProgress( return nil } +func (u *Uploader) updateTask(ctx context.Context, id string, patch client.TaskPatch) (client.DownloadTask, error) { + var task client.DownloadTask + err := callAPI(ctx, nil, "update task", func(ctx context.Context) error { + var err error + task, err = u.api.UpdateTask(ctx, id, patch) + return err + }) + return task, err +} + +func (u *Uploader) createFolder(ctx context.Context, token string, name string, parent string) (client.ObjectDraft, error) { + var draft client.ObjectDraft + err := callAPI(ctx, nil, "create folder", func(ctx context.Context) error { + var err error + draft, err = u.api.CreateFolder(ctx, token, name, parent) + return err + }) + return draft, err +} + +func (u *Uploader) createObject(ctx context.Context, token string, name string, size int64, parent string) (client.ObjectDraft, error) { + var draft client.ObjectDraft + err := callAPI(ctx, nil, "create object", func(ctx context.Context) error { + var err error + draft, err = u.api.CreateObject(ctx, token, name, size, parent) + return err + }) + return draft, err +} + +func (u *Uploader) completeObjectUpload(ctx context.Context, token string, id string, sessionID string, parts []client.CompletedObjectUploadPart) error { + return callAPI(ctx, nil, "complete object upload", func(ctx context.Context) error { + return u.api.CompleteObjectUpload(ctx, token, id, sessionID, parts) + }) +} + +func (u *Uploader) abortObjectUploadSession(ctx context.Context, token string, id string, sessionID string) error { + return callAPI(ctx, nil, "abort object upload", func(ctx context.Context) error { + return u.api.AbortObjectUploadSession(ctx, token, id, sessionID) + }) +} + +func (u *Uploader) deleteObject(ctx context.Context, token string, id string) error { + return callAPI(ctx, nil, "delete object", func(ctx context.Context) error { + return u.api.DeleteObject(ctx, token, id) + }) +} + +type directoryEntry struct { + path string + relativePath string + name string + size int64 + isDir bool +} + +func collectDirectoryEntries(root string) ([]directoryEntry, error) { + var entries []directoryEntry + err := filepath.WalkDir(root, func(path string, entry os.DirEntry, err error) error { + if err != nil { + return err + } + if path == root { + return nil + } + if strings.HasPrefix(entry.Name(), ".") { + if entry.IsDir() { + return filepath.SkipDir + } + return nil + } + relativePath, err := filepath.Rel(root, path) + if err != nil { + return err + } + if !entry.IsDir() && system.IsDownloadSidecarPath(relativePath) { + return nil + } + item := directoryEntry{ + path: path, + relativePath: filepath.ToSlash(relativePath), + name: entry.Name(), + isDir: entry.IsDir(), + } + if !entry.IsDir() { + info, err := entry.Info() + if err != nil { + return err + } + item.size = info.Size() + } + entries = append(entries, item) + return nil + }) + sort.Slice(entries, func(i, j int) bool { + if entries[i].isDir != entries[j].isDir { + return entries[i].isDir + } + return entries[i].relativePath < entries[j].relativePath + }) + return entries, err +} + +func joinObjectPath(parent string, name string) string { + name = strings.Trim(filepath.ToSlash(name), "/") + if name == "" || name == "." { + return strings.Trim(parent, "/") + } + parent = strings.Trim(parent, "/") + if parent == "" { + return name + } + return parent + "/" + name +} + func uploadFilePart(ctx context.Context, url string, file *os.File, offset int64, length int64, progress func(written int64) error) (string, error) { reader := io.NewSectionReader(file, offset, length) var body io.Reader = reader diff --git a/cmd/internal/engine/engine.go b/cmd/internal/engine/engine.go deleted file mode 100644 index e21a7300..00000000 --- a/cmd/internal/engine/engine.go +++ /dev/null @@ -1,369 +0,0 @@ -package engine - -import ( - "context" - "net/url" - "os" - "path/filepath" - "strings" - "time" - - "github.com/saltbo/zpan/internal/client" -) - -type Result struct { - Path string - Name string - Size int64 - IsDir bool - Seed *Seed -} - -type Seed struct { - Engine string - ID string - InfoHash string - Path string - Snapshot func(context.Context) (SeedSnapshot, error) - Cleanup func(context.Context) error -} - -type SeedRef struct { - TaskID string - Engine string - ID string - InfoHash string - Path string -} - -type SeedSnapshot struct { - Downloaded int64 - Total *int64 - Bps int64 - Runtime *client.DownloadTaskRuntime -} - -type Progress func(downloaded int64, total *int64, bps int64, detail *client.DownloadTaskRuntime) error - -type TaskState string - -const ( - TaskStateDownloading TaskState = "downloading" - TaskStateCompleted TaskState = "completed" - TaskStateFailed TaskState = "failed" -) - -type TaskSnapshot struct { - State TaskState - Downloaded int64 - Total *int64 - Bps int64 - Runtime *client.DownloadTaskRuntime - Result *Result - Error string -} - -type Engine interface { - Name() string - Capabilities() []string - Check(ctx context.Context) error - InspectTask(ctx context.Context, task client.DownloadTask) (TaskSnapshot, bool, error) - Download(ctx context.Context, task client.DownloadTask, progress Progress) (Result, error) -} - -type TaskResetter interface { - ResetTask(ctx context.Context, task client.DownloadTask) error -} - -type SeedRestorer interface { - RestoreSeed(ctx context.Context, ref SeedRef) (*Seed, error) -} - -// SeedLister enumerates every torrent the engine is currently seeding, -// including ones the worker is no longer tracking. The worker uses this to -// reconcile orphaned seeds so they cannot occupy runtime slots forever. -type SeedLister interface { - ListSeeds(ctx context.Context) ([]Seed, error) -} - -type SessionSaver interface { - SaveSession(ctx context.Context) error -} - -type progressWriter struct { - progress Progress - total *int64 - downloaded int64 - lastBytes int64 - lastAt time.Time -} - -func (p *progressWriter) Write(data []byte) (int, error) { - n := len(data) - p.downloaded += int64(n) - now := time.Now() - if now.Sub(p.lastAt) >= time.Second { - bps := int64(float64(p.downloaded-p.lastBytes) / now.Sub(p.lastAt).Seconds()) - if err := p.progress(p.downloaded, p.total, bps, &client.DownloadTaskRuntime{Engine: "builtin", Phase: "downloading"}); err != nil { - return n, err - } - p.lastBytes = p.downloaded - p.lastAt = now - } - return n, nil -} - -func resultFromPath(task client.DownloadTask, path string, fallbackName string) (Result, error) { - info, err := os.Stat(path) - if err != nil { - candidate := filepath.Join(path, fallbackName) - if fallbackName != "" { - if _, statErr := os.Stat(candidate); statErr == nil { - return resultFromPath(task, candidate, fallbackName) - } - } - return Result{}, err - } - if !info.IsDir() { - return resultFromFile(task, path) - } - entries, err := os.ReadDir(path) - if err != nil { - return Result{}, err - } - visible := make([]os.DirEntry, 0, len(entries)) - for _, entry := range entries { - if strings.HasPrefix(entry.Name(), ".") { - continue - } - if !entry.IsDir() && isDownloadSidecarPath(entry.Name()) { - continue - } - visible = append(visible, entry) - } - if len(visible) == 1 && !visible[0].IsDir() { - return resultFromFile(task, filepath.Join(path, visible[0].Name())) - } - if len(visible) == 1 && visible[0].IsDir() { - return resultFromPath(task, filepath.Join(path, visible[0].Name()), visible[0].Name()) - } - size, err := directorySize(path) - if err != nil { - return Result{}, err - } - return Result{Path: path, Name: outputName(task, fallbackName), Size: size, IsDir: true}, nil -} - -type downloadedFile struct { - path string - relativePath string -} - -func resultFromDownloadedFiles(task client.DownloadTask, taskDir string, fallbackName string, files []downloadedFile) (Result, error) { - if len(files) == 1 && !hasPathSeparator(files[0].relativePath) { - if task.SourceType() != "http" { - size, err := directorySize(taskDir) - if err != nil { - return Result{}, err - } - return Result{Path: taskDir, Name: singleFileTorrentFolderName(task, files[0].relativePath, fallbackName), Size: size, IsDir: true}, nil - } - return resultFromFile(task, files[0].path) - } - root, ok := singleTopLevelDirectory(files) - if ok { - path := filepath.Join(taskDir, root) - size, err := directorySize(path) - if err != nil { - return Result{}, err - } - return Result{Path: path, Name: outputName(task, root), Size: size, IsDir: true}, nil - } - size, err := directorySize(taskDir) - if err != nil { - return Result{}, err - } - return Result{Path: taskDir, Name: outputName(task, fallbackName), Size: size, IsDir: true}, nil -} - -func singleFileTorrentFolderName(task client.DownloadTask, filePath string, fallbackName string) string { - if name := requestedOutputName(task); name != "" { - return name - } - if name := payloadFallbackName(fallbackName); name != "" { - return name - } - base := filepath.Base(filePath) - if base != "" && base != "." && base != string(filepath.Separator) { - ext := filepath.Ext(base) - if ext != "" { - base = strings.TrimSuffix(base, ext) - } - if base != "" { - return base - } - } - return outputName(task, fallbackName) -} - -func payloadFallbackName(fallbackName string) string { - name := strings.TrimSpace(fallbackName) - if name == "" || isDownloadSidecarPath(name) { - return "" - } - name = filepath.Base(name) - if name == "." || name == string(filepath.Separator) { - return "" - } - return name -} - -func singleTopLevelDirectory(files []downloadedFile) (string, bool) { - var root string - for _, file := range files { - segments := splitRelativePath(file.relativePath) - if len(segments) < 2 { - return "", false - } - if root == "" { - root = segments[0] - continue - } - if segments[0] != root { - return "", false - } - } - return root, root != "" -} - -func splitRelativePath(path string) []string { - normalized := filepath.ToSlash(filepath.Clean(path)) - if normalized == "." || normalized == "/" { - return nil - } - parts := strings.Split(strings.Trim(normalized, "/"), "/") - out := parts[:0] - for _, part := range parts { - if part != "" && part != "." { - out = append(out, part) - } - } - return out -} - -func stripTorrentRoot(path string, torrentName string) string { - parts := splitRelativePath(path) - if len(parts) < 2 { - return filepath.ToSlash(filepath.Clean(path)) - } - if torrentName == "" || parts[0] != torrentName { - return filepath.ToSlash(filepath.Clean(path)) - } - return strings.Join(parts[1:], "/") -} - -func hasPathSeparator(path string) bool { - return len(splitRelativePath(path)) > 1 -} - -func isAria2MetadataPath(path string) bool { - return strings.HasPrefix(path, "[MEMORY]") || strings.HasPrefix(path, "[METADATA]") -} - -func isDownloadSidecarPath(path string) bool { - base := filepath.Base(path) - ext := filepath.Ext(base) - return isAria2MetadataPath(path) || - isAria2MetadataPath(base) || - strings.EqualFold(ext, ".torrent") || - strings.EqualFold(ext, ".aria2") -} - -func resultFromFile(task client.DownloadTask, path string) (Result, error) { - info, err := os.Stat(path) - if err != nil { - return Result{}, err - } - return Result{Path: path, Name: outputName(task, filepath.Base(path)), Size: info.Size()}, nil -} - -func directorySize(path string) (int64, error) { - var total int64 - err := filepath.WalkDir(path, func(entryPath string, entry os.DirEntry, err error) error { - if err != nil { - return err - } - if entryPath == path { - return nil - } - if strings.HasPrefix(entry.Name(), ".") { - if entry.IsDir() { - return filepath.SkipDir - } - return nil - } - if entry.IsDir() { - return nil - } - if isDownloadSidecarPath(entry.Name()) { - return nil - } - info, err := entry.Info() - if err != nil { - return err - } - total += info.Size() - return nil - }) - return total, err -} - -func downloadedPath(baseDir string, path string) (string, string) { - if filepath.IsAbs(path) { - abs := filepath.Clean(path) - rel, err := filepath.Rel(baseDir, abs) - if err != nil || isUnsafeRelativePath(rel) { - return abs, filepath.Base(abs) - } - return abs, rel - } - rel := filepath.Clean(path) - if isUnsafeRelativePath(rel) { - return filepath.Join(baseDir, filepath.Base(rel)), filepath.Base(rel) - } - return filepath.Join(baseDir, rel), rel -} - -func isUnsafeRelativePath(path string) bool { - return path == ".." || strings.HasPrefix(path, ".."+string(filepath.Separator)) || filepath.IsAbs(path) -} - -func outputName(task client.DownloadTask, fallback string) string { - name := requestedOutputName(task) - if name == "" { - name = strings.TrimSpace(fallback) - } - if name == "" || name == "." || name == string(filepath.Separator) { - name = task.ID - } - return filepath.Base(name) -} - -func requestedOutputName(task client.DownloadTask) string { - name := strings.TrimSpace(task.Name()) - if name == "" { - return "" - } - if task.SourceType() != "http" && isDownloadSidecarPath(name) { - return "" - } - return filepath.Base(name) -} - -func filenameFromURL(parsed *url.URL) string { - name := filepath.Base(parsed.Path) - if name == "." || name == "/" { - return "" - } - return name -} diff --git a/cmd/internal/engine/engine_test.go b/cmd/internal/engine/engine_test.go deleted file mode 100644 index 84493f4e..00000000 --- a/cmd/internal/engine/engine_test.go +++ /dev/null @@ -1,1041 +0,0 @@ -package engine - -import ( - "context" - "errors" - "io" - "net/http" - "net/http/httptest" - "os" - "path/filepath" - "strings" - "testing" - "time" - - "github.com/Braurbeki/arigo" - qbittorrent "github.com/autobrr/go-qbittorrent" - "github.com/cenkalti/rpc2" - "github.com/saltbo/zpan/internal/client" -) - -func downloadTask(id, sourceType, sourceURI string) client.DownloadTask { - return client.DownloadTask{ - ID: id, - Spec: client.DownloadTaskSpec{ - Source: client.DownloadTaskSource{Type: sourceType, URI: sourceURI}, - Destination: client.DownloadTaskDestination{}, - Labels: client.DownloadTaskLabels{Tags: []string{}}, - }, - Status: client.DownloadTaskStatus{}, - } -} - -func downloadTaskWithName(id, sourceType, sourceURI, name string) client.DownloadTask { - task := downloadTask(id, sourceType, sourceURI) - task.Spec.Destination.Name = name - return task -} - -func completedDownloadTask(id, sourceType, sourceURI string, size int64) client.DownloadTask { - task := downloadTask(id, sourceType, sourceURI) - task.Status.Progress.Download = client.DownloadTaskTransferProgress{Bytes: size, TotalBytes: &size} - return task -} - -func TestHTTPDownload(t *testing.T) { - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Content-Length", "11") - _, _ = w.Write([]byte("hello world")) - })) - defer server.Close() - - dir := t.TempDir() - progressCalls := 0 - var lastDetail *client.DownloadTaskRuntime - result, err := (HTTP{Dir: dir}).Download( - context.Background(), - downloadTask("task-1", "http", server.URL+"/file.txt"), - func(downloaded int64, total *int64, bps int64, detail *client.DownloadTaskRuntime) error { - progressCalls++ - lastDetail = detail - if downloaded < 0 { - t.Fatalf("downloaded bytes must not be negative") - } - return nil - }, - ) - if err != nil { - t.Fatal(err) - } - if result.Name != "file.txt" { - t.Fatalf("expected file.txt, got %s", result.Name) - } - if result.Size != 11 { - t.Fatalf("expected size 11, got %d", result.Size) - } - data, err := os.ReadFile(result.Path) - if err != nil { - t.Fatal(err) - } - if string(data) != "hello world" { - t.Fatalf("unexpected file content: %q", string(data)) - } - if progressCalls == 0 { - t.Fatal("expected progress callback") - } - if lastDetail == nil || lastDetail.Engine != "builtin" { - t.Fatalf("expected builtin progress detail, got %#v", lastDetail) - } -} - -func TestHTTPRejectsMagnet(t *testing.T) { - _, err := (HTTP{Dir: t.TempDir()}).Download( - context.Background(), - downloadTask("task-1", "magnet", "magnet:?xt=urn:btih:abc"), - func(downloaded int64, total *int64, bps int64, detail *client.DownloadTaskRuntime) error { return nil }, - ) - if err == nil { - t.Fatal("expected magnet to be rejected by HTTP engine") - } -} - -func TestAria2DelegatesHTTPToBuiltin(t *testing.T) { - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Content-Length", "11") - _, _ = w.Write([]byte("hello world")) - })) - defer server.Close() - - result, err := (Aria2{ - URL: "ws://127.0.0.1:1/jsonrpc", - Dir: t.TempDir(), - }).Download( - context.Background(), - downloadTask("task-1", "http", server.URL+"/file.txt"), - func(downloaded int64, total *int64, bps int64, detail *client.DownloadTaskRuntime) error { - if detail != nil && detail.Engine != "builtin" { - t.Fatalf("expected builtin HTTP runtime detail, got %#v", detail) - } - return nil - }, - ) - - if err != nil { - t.Fatal(err) - } - data, err := os.ReadFile(result.Path) - if err != nil { - t.Fatal(err) - } - if string(data) != "hello world" { - t.Fatalf("unexpected file content: %q", string(data)) - } -} - -func TestAria2StartArgsSaveSessionWithoutAutoRestore(t *testing.T) { - stateDir := t.TempDir() - args, err := (Aria2{Dir: t.TempDir(), StateDir: stateDir, ListenPort: 51413}).startArgs("6800") - if err != nil { - t.Fatal(err) - } - joined := strings.Join(args, "\n") - if strings.Contains(joined, "--input-file=") { - t.Fatalf("aria2 must not auto-restore old session tasks, got %v", args) - } - for _, expected := range []string{ - "--save-session=" + filepath.Join(stateDir, "aria2.session"), - "--save-session-interval=30", - "--force-save=true", - "--listen-port=51413", - } { - if !strings.Contains(joined, expected) { - t.Fatalf("expected aria2 args to contain %q, got %v", expected, args) - } - } -} - -func TestAria2StartArgsEnablesPeerDiscovery(t *testing.T) { - stateDir := t.TempDir() - args, err := (Aria2{Dir: t.TempDir(), StateDir: stateDir}).startArgs("6800") - if err != nil { - t.Fatal(err) - } - joined := strings.Join(args, "\n") - for _, expected := range []string{ - "--enable-dht=true", - "--enable-peer-exchange=true", - "--bt-tracker=", - "udp://tracker.opentrackr.org:1337/announce", - "--dht-file-path=" + filepath.Join(stateDir, "dht.dat"), - } { - if !strings.Contains(joined, expected) { - t.Fatalf("expected aria2 args to contain %q for magnet peer discovery, got %v", expected, args) - } - } - - // A provided (live-fetched) tracker list is used verbatim instead of the fallback. - custom, err := (Aria2{Dir: t.TempDir(), BtTrackers: "udp://custom.example:1337/announce"}).startArgs("6800") - if err != nil { - t.Fatal(err) - } - if !strings.Contains(strings.Join(custom, "\n"), "--bt-tracker=udp://custom.example:1337/announce") { - t.Fatalf("expected provided BtTrackers to be used, got %v", custom) - } -} - -func TestAria2StartArgsSetsMaxConcurrentDownloads(t *testing.T) { - args, err := (Aria2{Dir: t.TempDir(), MaxConcurrentDownloads: 25}).startArgs("6800") - if err != nil { - t.Fatal(err) - } - if !strings.Contains(strings.Join(args, "\n"), "--max-concurrent-downloads=25") { - t.Fatalf("expected aria2 args to set max-concurrent-downloads, got %v", args) - } - - zeroArgs, err := (Aria2{Dir: t.TempDir()}).startArgs("6800") - if err != nil { - t.Fatal(err) - } - if strings.Contains(strings.Join(zeroArgs, "\n"), "--max-concurrent-downloads") { - t.Fatalf("expected no max-concurrent-downloads flag when unset, got %v", zeroArgs) - } -} - -func TestAria2StatusKeysCoverReportedFields(t *testing.T) { - // aria2's tellStatus only returns the keys it's asked for. aria2Detail reads - // these fields, so they must be requested or the runtime reports zeros - // (e.g. seeding upload speed/total showing 0 while peers upload). - required := []string{ - "status", "totalLength", "completedLength", "downloadSpeed", - "uploadLength", "uploadSpeed", "connections", "numSeeders", - } - have := map[string]bool{} - for _, key := range aria2StatusKeys { - have[key] = true - } - for _, key := range required { - if !have[key] { - t.Fatalf("aria2StatusKeys missing %q — tellStatus won't return it, so aria2Detail reports 0", key) - } - } -} - -func TestAria2SeedTimeMinutes(t *testing.T) { - if got := aria2SeedTimeMinutes(time.Hour); got != 60 { - t.Fatalf("expected 1h to map to 60 minutes, got %d", got) - } - if got := aria2SeedTimeMinutes(0); got != aria2SeedForeverMinutes { - t.Fatalf("expected zero duration to seed indefinitely, got %d", got) - } - if got := aria2SeedTimeMinutes(30 * time.Second); got != 1 { - t.Fatalf("expected sub-minute duration to round up to 1, got %d", got) - } -} - -func TestIsAria2RPCDisconnectedNilSafe(t *testing.T) { - // findSeed/findTask call this with a nil error when tellStatus succeeded but - // the status wasn't what we wanted; it must not panic on err.Error(). - if isAria2RPCDisconnected(nil) { - t.Fatal("nil error must not be treated as disconnected") - } -} - -func TestShouldAttachExistingAria2Task(t *testing.T) { - // 'interrupted' (how restart resumes a task) must attach to an existing - // engine task when one is still present rather than re-add it. - for _, state := range []string{"downloading", "uploading", "interrupted"} { - if !shouldAttachExistingAria2Task(client.DownloadTask{Status: client.DownloadTaskStatus{State: state}}) { - t.Fatalf("expected to attach for state %q", state) - } - } - for _, state := range []string{"queued", "assigned", "paused", "completed", "canceling"} { - if shouldAttachExistingAria2Task(client.DownloadTask{Status: client.DownloadTaskStatus{State: state}}) { - t.Fatalf("did not expect to attach for state %q", state) - } - } -} - -func TestAria2PeerProgress(t *testing.T) { - if p := aria2PeerProgress(arigo.Peer{Seeder: true}); p == nil || *p != 1.0 { - t.Fatalf("expected seeder progress 1.0, got %v", p) - } - if p := aria2PeerProgress(arigo.Peer{BitField: "ff"}); p == nil || *p != 1.0 { - t.Fatalf("expected full bitfield 1.0, got %v", p) - } - if p := aria2PeerProgress(arigo.Peer{BitField: "f0"}); p == nil || *p != 0.5 { - t.Fatalf("expected half bitfield 0.5, got %v", p) - } - if p := aria2PeerProgress(arigo.Peer{BitField: ""}); p != nil { - t.Fatalf("expected nil progress for empty bitfield, got %v", p) - } -} - -func TestQBittorrentStartArgsWritesManagedListenPort(t *testing.T) { - stateDir := t.TempDir() - downloadDir := t.TempDir() - args, err := (QBittorrent{ - URL: "http://127.0.0.1:8080", - Dir: downloadDir, - StateDir: stateDir, - ListenPort: 51413, - }).startArgs("/usr/bin/qbittorrent-nox") - if err != nil { - t.Fatal(err) - } - joined := strings.Join(args, " ") - if !strings.Contains(joined, "--profile="+filepath.Join(stateDir, "qbittorrent")) { - t.Fatalf("expected qBittorrent args to contain managed profile, got %v", args) - } - if !strings.Contains(joined, "--webui-port=8080") { - t.Fatalf("expected qBittorrent args to contain webui port, got %v", args) - } - content, err := os.ReadFile(filepath.Join(stateDir, "qbittorrent", "qBittorrent", "config", "qBittorrent.conf")) - if err != nil { - t.Fatal(err) - } - text := string(content) - if !strings.Contains(text, `Connection\PortRangeMin=51413`) { - t.Fatalf("expected custom listen port, got:\n%s", text) - } - if !strings.Contains(text, `Downloads\SavePath=`+filepath.ToSlash(downloadDir)+`/`) { - t.Fatalf("expected custom download dir, got:\n%s", text) - } -} - -func TestAria2ResetOperations(t *testing.T) { - tests := []struct { - name string - status arigo.DownloadStatus - wantRemoveActive bool - wantRemoveResult bool - }{ - {name: "active", status: arigo.StatusActive, wantRemoveActive: true, wantRemoveResult: true}, - {name: "waiting", status: arigo.StatusWaiting, wantRemoveActive: true, wantRemoveResult: true}, - {name: "paused", status: arigo.StatusPaused, wantRemoveActive: true, wantRemoveResult: true}, - {name: "completed", status: arigo.StatusCompleted, wantRemoveActive: false, wantRemoveResult: true}, - {name: "error", status: arigo.StatusError, wantRemoveActive: false, wantRemoveResult: true}, - {name: "removed", status: arigo.StatusRemoved, wantRemoveActive: false, wantRemoveResult: true}, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - removeActive, removeResult := aria2ResetOperations(arigo.Status{Status: tt.status}) - if removeActive != tt.wantRemoveActive || removeResult != tt.wantRemoveResult { - t.Fatalf( - "expected removeActive=%v removeResult=%v, got removeActive=%v removeResult=%v", - tt.wantRemoveActive, - tt.wantRemoveResult, - removeActive, - removeResult, - ) - } - }) - } -} - -func TestIsAria2DownloadNotFound(t *testing.T) { - if !isAria2DownloadNotFound(errors.New("Active Download not found for GID#b384ccaa7eae88da")) { - t.Fatal("expected aria2 active download not found to be ignored during reset") - } - if !isAria2DownloadNotFound(errors.New("Download result not found for GID#b384ccaa7eae88da")) { - t.Fatal("expected aria2 download result not found to be ignored during reset") - } - if isAria2DownloadNotFound(errors.New("aria2 download ended with status error")) { - t.Fatal("expected ordinary aria2 download errors to stay visible") - } -} - -func TestIsAria2GIDNotFound(t *testing.T) { - if !isAria2GIDNotFound(errors.New("GID 8bddd19e07ad6dc3 is not found")) { - t.Fatal("expected tellStatus GID-not-found to trigger re-discovery") - } - if !isAria2GIDNotFound(errors.New("Active Download not found for GID#b384ccaa7eae88da")) { - t.Fatal("expected aria2 active download not found to trigger re-discovery") - } - if isAria2GIDNotFound(errors.New("aria2 download ended with status error")) { - t.Fatal("expected ordinary aria2 download errors to stay visible") - } -} - -func TestHTTPDownloadResumesExistingFile(t *testing.T) { - var rangeHeader string - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - rangeHeader = r.Header.Get("Range") - if rangeHeader != "bytes=5-" { - t.Fatalf("expected resume range bytes=5-, got %q", rangeHeader) - } - w.Header().Set("Content-Length", "6") - w.WriteHeader(http.StatusPartialContent) - _, _ = w.Write([]byte(" world")) - })) - defer server.Close() - - dir := t.TempDir() - taskDir := filepath.Join(dir, "task-1") - if err := os.MkdirAll(taskDir, 0o755); err != nil { - t.Fatal(err) - } - path := filepath.Join(taskDir, "file.txt") - if err := os.WriteFile(path, []byte("hello"), 0o644); err != nil { - t.Fatal(err) - } - - result, err := (HTTP{Dir: dir}).Download( - context.Background(), - downloadTask("task-1", "http", server.URL+"/file.txt"), - func(downloaded int64, total *int64, bps int64, detail *client.DownloadTaskRuntime) error { return nil }, - ) - if err != nil { - t.Fatal(err) - } - data, err := os.ReadFile(result.Path) - if err != nil { - t.Fatal(err) - } - if string(data) != "hello world" { - t.Fatalf("expected resumed content, got %q", string(data)) - } - if result.Size != 11 { - t.Fatalf("expected size 11, got %d", result.Size) - } -} - -func TestHTTPInspectTaskUsesCompletedCheckpoint(t *testing.T) { - dir := t.TempDir() - taskDir := filepath.Join(dir, "task-1") - if err := os.MkdirAll(taskDir, 0o755); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(filepath.Join(taskDir, "payload.bin"), []byte("payload"), 0o644); err != nil { - t.Fatal(err) - } - total := int64(7) - - snapshot, found, err := (HTTP{Dir: dir}).InspectTask( - context.Background(), - completedDownloadTask("task-1", "http", "https://example.com/payload.bin", total), - ) - - if err != nil { - t.Fatal(err) - } - if !found { - t.Fatal("expected completed http checkpoint to be found") - } - if snapshot.State != TaskStateCompleted { - t.Fatalf("expected completed state, got %#v", snapshot) - } - if snapshot.Result == nil || snapshot.Result.Path != filepath.Join(taskDir, "payload.bin") || snapshot.Result.Size != 7 { - t.Fatalf("unexpected completed result: %#v", snapshot.Result) - } -} - -func TestHTTPInspectTaskDoesNotTrustLocalFileWithoutCompletedCheckpoint(t *testing.T) { - dir := t.TempDir() - taskDir := filepath.Join(dir, "task-1") - if err := os.MkdirAll(taskDir, 0o755); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(filepath.Join(taskDir, "payload.bin"), []byte("payload"), 0o644); err != nil { - t.Fatal(err) - } - - snapshot, found, err := (HTTP{Dir: dir}).InspectTask( - context.Background(), - downloadTask("task-1", "http", "https://example.com/payload.bin"), - ) - - if err != nil { - t.Fatal(err) - } - if found { - t.Fatalf("expected local file without checkpoint not to be trusted, got %#v", snapshot) - } -} - -func TestHTTPInspectTaskRejectsSizeMismatch(t *testing.T) { - dir := t.TempDir() - taskDir := filepath.Join(dir, "task-1") - if err := os.MkdirAll(taskDir, 0o755); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(filepath.Join(taskDir, "payload.bin"), []byte("payload"), 0o644); err != nil { - t.Fatal(err) - } - total := int64(8) - - _, _, err := (HTTP{Dir: dir}).InspectTask( - context.Background(), - completedDownloadTask("task-1", "http", "https://example.com/payload.bin", total), - ) - - if err == nil || !strings.Contains(err.Error(), "size mismatch") { - t.Fatalf("expected size mismatch error, got %v", err) - } -} - -func TestAria2StatusKeysRequestBittorrentPayload(t *testing.T) { - keys := strings.Join(aria2StatusKeys, ",") - if !strings.Contains(keys, "bittorrent") { - t.Fatalf("expected aria2 status keys to request bittorrent payload, got %v", aria2StatusKeys) - } - if strings.Contains(keys, "bitTorrent") { - t.Fatalf("aria2 status key is case-sensitive; use bittorrent, got %v", aria2StatusKeys) - } -} - -func TestQBittorrentCheckUsesWebAPIVersion(t *testing.T) { - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.URL.Path != "/api/v2/app/version" { - t.Fatalf("unexpected path: %s", r.URL.Path) - } - _, _ = w.Write([]byte("5.0.0")) - })) - defer server.Close() - - if err := (QBittorrent{URL: server.URL, Dir: t.TempDir()}).Check(context.Background()); err != nil { - t.Fatal(err) - } -} - -func TestQBittorrentHTTPDelegatesToBuiltin(t *testing.T) { - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Content-Length", "11") - _, _ = w.Write([]byte("hello qbit!")) - })) - defer server.Close() - - result, err := (QBittorrent{URL: "http://127.0.0.1:1", Dir: t.TempDir()}).Download( - context.Background(), - downloadTask("task-1", "http", server.URL+"/file.txt"), - func(downloaded int64, total *int64, bps int64, detail *client.DownloadTaskRuntime) error { return nil }, - ) - if err != nil { - t.Fatal(err) - } - data, err := os.ReadFile(result.Path) - if err != nil { - t.Fatal(err) - } - if string(data) != "hello qbit!" { - t.Fatalf("unexpected file content: %q", string(data)) - } -} - -func TestQBittorrentAddOptionsPassThroughTaskClassification(t *testing.T) { - options := qbittorrentAddOptions( - func() client.DownloadTask { - task := downloadTaskWithName("task-1", "magnet", "magnet:?xt=urn:btih:abc", "fixture") - task.Spec.Labels.Category = "movies" - task.Spec.Labels.Tags = []string{"4k", "private"} - return task - }(), - "/tmp/zpan/task-1", - qbittorrentTrackingTag("task-1"), - ) - - if options["category"] != "movies" { - t.Fatalf("expected task category, got %q", options["category"]) - } - if options["tags"] != "ztid=task-1,4k,private" { - t.Fatalf("expected tracking and task tags, got %q", options["tags"]) - } - if options["rename"] != "fixture" { - t.Fatalf("expected rename option, got %q", options["rename"]) - } -} - -func TestQBittorrentAddOptionsDefaultsCategory(t *testing.T) { - options := qbittorrentAddOptions( - client.DownloadTask{ID: "task-1"}, - "/tmp/zpan/task-1", - qbittorrentTrackingTag("task-1"), - ) - - if options["category"] != "zpan" { - t.Fatalf("expected default category, got %q", options["category"]) - } - if options["tags"] != "ztid=task-1" { - t.Fatalf("expected tracking tag, got %q", options["tags"]) - } -} - -func TestQBittorrentAddOptionsIgnoresTorrentTaskName(t *testing.T) { - options := qbittorrentAddOptions( - downloadTaskWithName("task-1", "magnet", "magnet:?xt=urn:btih:abc", "movie.torrent"), - "/tmp/zpan/task-1", - qbittorrentTrackingTag("task-1"), - ) - - if _, ok := options["rename"]; ok { - t.Fatalf("expected torrent task name to be ignored, got rename=%q", options["rename"]) - } -} - -func TestIsAria2RPCDisconnected(t *testing.T) { - for _, err := range []error{ - rpc2.ErrShutdown, - io.ErrClosedPipe, - errors.New("connection is shut down"), - } { - if !isAria2RPCDisconnected(err) { - t.Fatalf("expected %v to be treated as aria2 rpc disconnect", err) - } - } - if isAria2RPCDisconnected(errors.New("aria2 download ended with status error")) { - t.Fatal("expected ordinary aria2 error to stay non-transient") - } -} - -func TestAria2TaskInfoHash(t *testing.T) { - const infoHash = "0546769f209ec059284b47f68659791a6f75ca8e" - - if got := aria2TaskInfoHash(downloadTask("task-1", "magnet", "magnet:?xt=urn:btih:"+infoHash+"&dn=fixture")); got != infoHash { - t.Fatalf("expected magnet infohash %s, got %s", infoHash, got) - } - taskWithRuntime := downloadTask("task-1", "magnet", "magnet:?xt=urn:btih:abc") - taskWithRuntime.Status.Runtime = &client.DownloadTaskRuntime{ - Torrent: &client.DownloadTaskTorrentRuntime{InfoHash: "0546769F209EC059284B47F68659791A6F75CA8E"}, - } - if got := aria2TaskInfoHash(taskWithRuntime); got != infoHash { - t.Fatalf("expected detail infohash %s, got %s", infoHash, got) - } - if got := aria2TaskInfoHash(downloadTask("task-1", "http", "https://example.com/file")); got != "" { - t.Fatalf("expected no infohash for http task, got %s", got) - } -} - -func TestAria2StatusMatchesTaskByInfoHash(t *testing.T) { - const infoHash = "0546769f209ec059284b47f68659791a6f75ca8e" - - if !aria2StatusMatchesTask(arigo.Status{InfoHash: strings.ToUpper(infoHash)}, "/tmp/zpan/task-1", "taskgid", infoHash) { - t.Fatal("expected status to match by infohash") - } - if aria2StatusMatchesTask(arigo.Status{InfoHash: infoHash}, "/tmp/zpan/task-1", "taskgid", "") { - t.Fatal("expected empty requested infohash not to match") - } -} - -func TestIsAria2DownloadCompleteTreatsActiveFullTorrentAsComplete(t *testing.T) { - status := arigo.Status{ - Status: arigo.StatusActive, - TotalLength: 100, - CompletedLength: 100, - Files: []arigo.File{ - {Path: "/tmp/zpan/task-1/movie.mkv", Length: 100, CompletedLength: 100}, - }, - } - - if !isAria2DownloadComplete(status) { - t.Fatal("expected active full torrent to be treated as completed") - } -} - -func TestSelectAria2SeedStatusPrefersCompletedPayloadOverMetadata(t *testing.T) { - infoHash := "f8f8044d5dfeef2719dcda6ce42dba1c4eb9ea22" - taskDir := filepath.Join(t.TempDir(), "task-1") - metadata := arigo.Status{ - GID: "metadata-gid", - Status: arigo.StatusCompleted, - InfoHash: infoHash, - Dir: taskDir, - TotalLength: 17596, - CompletedLength: 17596, - Files: []arigo.File{{ - Path: "[METADATA]", - Length: 17596, - CompletedLength: 17596, - Selected: true, - }}, - } - payload := arigo.Status{ - GID: "payload-gid", - Status: arigo.StatusActive, - InfoHash: infoHash, - Dir: taskDir, - TotalLength: 100, - CompletedLength: 100, - Files: []arigo.File{{ - Path: filepath.Join(taskDir, "album", "track.flac"), - Length: 100, - CompletedLength: 100, - Selected: true, - }}, - } - - got, ok := selectAria2SeedStatus([]arigo.Status{metadata, payload}, SeedRef{InfoHash: infoHash}, taskDir) - if !ok { - t.Fatal("expected seed status") - } - if got.GID != "payload-gid" { - t.Fatalf("expected payload seed, got %s", got.GID) - } -} - -func TestAria2TaskState(t *testing.T) { - if got := aria2TaskState(arigo.Status{ - Status: arigo.StatusActive, - TotalLength: 100, - CompletedLength: 100, - Files: []arigo.File{{Path: "/tmp/zpan/task-1/file.bin", Length: 100, CompletedLength: 100}}, - }); got != TaskStateCompleted { - t.Fatalf("expected active full torrent to be completed, got %s", got) - } - if got := aria2TaskState(arigo.Status{Status: arigo.StatusActive, TotalLength: 100, CompletedLength: 10}); got != TaskStateDownloading { - t.Fatalf("expected active partial torrent to be downloading, got %s", got) - } - if got := aria2TaskState(arigo.Status{Status: arigo.StatusError}); got != TaskStateFailed { - t.Fatalf("expected error torrent to be failed, got %s", got) - } -} - -func TestQBittorrentTaskState(t *testing.T) { - if got := qbittorrentTaskState(qbittorrent.Torrent{ - State: qbittorrent.TorrentState("stalledUP"), - Progress: 1, - AmountLeft: 0, - TotalSize: 100, - }); got != TaskStateCompleted { - t.Fatalf("expected seeding torrent to be completed, got %s", got) - } - if got := qbittorrentTaskState(qbittorrent.Torrent{ - State: qbittorrent.TorrentState("downloading"), - Progress: 0.5, - AmountLeft: 50, - TotalSize: 100, - }); got != TaskStateDownloading { - t.Fatalf("expected partial torrent to be downloading, got %s", got) - } - if got := qbittorrentTaskState(qbittorrent.Torrent{ - State: qbittorrent.TorrentState("missingFiles"), - Progress: 0.5, - AmountLeft: 50, - TotalSize: 100, - }); got != TaskStateFailed { - t.Fatalf("expected missing files torrent to be failed, got %s", got) - } -} - -func TestQBittorrentDetailOmitsSeedingETA(t *testing.T) { - detail := qbittorrentDetail(context.Background(), nil, qbittorrent.Torrent{ - State: qbittorrent.TorrentState("stalledUP"), - ETA: 3600, - Progress: 1, - AmountLeft: 0, - TotalSize: 100, - }, nil) - - if detail.Phase != "seeding" { - t.Fatalf("expected seeding phase, got %s", detail.Phase) - } - if detail.ETASeconds != nil { - t.Fatalf("expected seeding detail without ETA, got %#v", detail.ETASeconds) - } -} - -func TestIsAria2InfoHashAlreadyRegistered(t *testing.T) { - err := errors.New("InfoHash 0546769f209ec059284b47f68659791a6f75ca8e is already registered.") - if !isAria2InfoHashAlreadyRegistered(err) { - t.Fatal("expected aria2 infohash conflict to be attachable") - } - if isAria2InfoHashAlreadyRegistered(errors.New("aria2 download ended with status error")) { - t.Fatal("expected ordinary aria2 error not to be treated as an infohash conflict") - } -} - -func TestAria2FilesReportsRelativeTorrentPaths(t *testing.T) { - taskDir := filepath.Join(t.TempDir(), "task-1") - files := aria2Files(taskDir, "album", []arigo.File{ - { - Path: filepath.Join(taskDir, "album", "disc-1", "track.flac"), - Length: 100, - CompletedLength: 50, - Selected: true, - }, - { - Path: filepath.Join(t.TempDir(), "outside.flac"), - Length: 10, - CompletedLength: 10, - Selected: true, - }, - { - Path: "[METADATA]info", - Length: 1, - }, - }) - - if len(files) != 2 { - t.Fatalf("expected two visible files, got %#v", files) - } - if files[0].Path != "disc-1/track.flac" { - t.Fatalf("expected torrent root to be stripped, got %s", files[0].Path) - } - if files[1].Path != "outside.flac" { - t.Fatalf("expected outside path to fall back to basename, got %s", files[1].Path) - } -} - -func TestStripTorrentRoot(t *testing.T) { - cases := []struct { - name string - path string - torrentName string - want string - }{ - {name: "nested torrent root", path: "Album/Disc 1/track.flac", torrentName: "Album", want: "Disc 1/track.flac"}, - {name: "single file named like torrent", path: "Album", torrentName: "Album", want: "Album"}, - {name: "different root", path: "Other/track.flac", torrentName: "Album", want: "Other/track.flac"}, - {name: "empty torrent name", path: "Album/track.flac", torrentName: "", want: "Album/track.flac"}, - } - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - if got := stripTorrentRoot(tc.path, tc.torrentName); got != tc.want { - t.Fatalf("expected %q, got %q", tc.want, got) - } - }) - } -} - -func TestAria2DetailIncludesPeers(t *testing.T) { - detail := aria2Detail( - arigo.Status{ - TotalLength: 1000, - CompletedLength: 250, - DownloadSpeed: 200, - Connections: 44, - NumSeeders: 4, - BitTorrent: arigo.BitTorrentStatus{ - AnnounceList: [][]string{{"udp://tracker.example:1337/announce"}}, - Info: arigo.BitTorrentStatusInfo{Name: "fixture.torrent"}, - }, - }, - []arigo.Peer{ - {IP: "192.0.2.10", Port: 6881, DownloadSpeed: 1024, UploadSpeed: 256, Seeder: true}, - {IP: "192.0.2.11", Port: 6882, DownloadSpeed: 2048, UploadSpeed: 512, Seeder: false}, - }, - nil, - ) - - if detail.Torrent == nil || detail.Torrent.Peers == nil || *detail.Torrent.Peers != 2 { - t.Fatalf("expected peer count 2, got %#v", detail.Torrent) - } - if detail.Torrent.Leechers == nil || *detail.Torrent.Leechers != 1 { - t.Fatalf("expected leecher count 1, got %#v", detail.Torrent.Leechers) - } - if len(detail.Peers) != 2 { - t.Fatalf("expected peer samples, got %#v", detail.Peers) - } - if detail.Peers[0].Address != "192.0.2.10:6881" { - t.Fatalf("unexpected peer address: %s", detail.Peers[0].Address) - } - if len(detail.Trackers) != 1 || detail.Trackers[0].Status != "announce" || detail.Trackers[0].Message == "" { - t.Fatalf("expected aria2 tracker limitation marker, got %#v", detail.Trackers) - } - if detail.ETASeconds == nil || *detail.ETASeconds != 4 { - t.Fatalf("expected ETA seconds 4, got %#v", detail.ETASeconds) - } -} - -func TestAria2DetailOmitsETAWithoutUsableSpeed(t *testing.T) { - detail := aria2Detail( - arigo.Status{ - TotalLength: 1000, - CompletedLength: 250, - DownloadSpeed: 0, - }, - nil, - nil, - ) - - if detail.ETASeconds != nil { - t.Fatalf("expected empty ETA without download speed, got %#v", detail.ETASeconds) - } -} - -type fakeGeoIPResolver struct{} - -func (fakeGeoIPResolver) LookupPeerRegion(ip string) (string, string) { - if ip == "203.0.113.10" { - return "us", "ca" - } - return "", "" -} - -func TestApplyPeerRegionUsesGeoIPAndFallbackCountry(t *testing.T) { - peer := client.DownloadTaskPeer{} - applyPeerRegion(&peer, "203.0.113.10", "", fakeGeoIPResolver{}) - if peer.CountryCode != "US" || peer.RegionCode != "CA" { - t.Fatalf("expected US/CA from geoip, got %#v", peer) - } - - fallback := client.DownloadTaskPeer{} - applyPeerRegion(&fallback, "198.51.100.10", "jp", fakeGeoIPResolver{}) - if fallback.CountryCode != "JP" || fallback.RegionCode != "" { - t.Fatalf("expected JP fallback country, got %#v", fallback) - } -} - -func TestResultFromPathReturnsDirectory(t *testing.T) { - dir := t.TempDir() - taskDir := filepath.Join(dir, "task-1") - if err := os.MkdirAll(filepath.Join(taskDir, "folder"), 0o755); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(filepath.Join(taskDir, "folder", "a.txt"), []byte("a"), 0o644); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(filepath.Join(taskDir, "folder", "b.txt"), []byte("b"), 0o644); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(filepath.Join(taskDir, "fixture.torrent"), []byte("torrent"), 0o644); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(filepath.Join(taskDir, "folder.aria2"), []byte("control"), 0o644); err != nil { - t.Fatal(err) - } - - result, err := resultFromPath(downloadTaskWithName("task-1", "http", "https://example.com/bundle", "bundle"), taskDir, "bundle") - if err != nil { - t.Fatal(err) - } - if !result.IsDir { - t.Fatal("expected directory result") - } - if result.Name != "bundle" { - t.Fatalf("expected bundle, got %s", result.Name) - } - if result.Size != 2 { - t.Fatalf("expected directory size 2, got %d", result.Size) - } - if result.Path != filepath.Join(taskDir, "folder") { - t.Fatalf("expected content dir path, got %s", result.Path) - } - if _, err := os.Stat(filepath.Join(result.Path, "a.txt")); err != nil { - t.Fatal(err) - } -} - -func TestResultFromAria2FilesUsesSingleTopLevelDirectory(t *testing.T) { - dir := t.TempDir() - taskDir := filepath.Join(dir, "task-1") - if err := os.MkdirAll(filepath.Join(taskDir, "payload"), 0o755); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(filepath.Join(taskDir, "payload", "a.txt"), []byte("a"), 0o644); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(filepath.Join(taskDir, "payload", "b.txt"), []byte("b"), 0o644); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(filepath.Join(taskDir, "payload.aria2"), []byte("control"), 0o644); err != nil { - t.Fatal(err) - } - - result, err := resultFromAria2Files( - client.DownloadTask{ID: "task-1"}, - taskDir, - "payload", - []arigo.File{ - {Path: filepath.Join(taskDir, "payload", "a.txt"), Length: 1, Selected: true}, - {Path: filepath.Join(taskDir, "payload", "b.txt"), Length: 1, Selected: true}, - }, - ) - if err != nil { - t.Fatal(err) - } - if result.Path != filepath.Join(taskDir, "payload") { - t.Fatalf("expected payload dir path, got %s", result.Path) - } - if result.Name != "payload" { - t.Fatalf("expected payload dir name, got %s", result.Name) - } - if result.Size != 2 { - t.Fatalf("expected payload size 2, got %d", result.Size) - } -} - -func TestResultFromAria2FilesWrapsSingleFileBTTask(t *testing.T) { - dir := t.TempDir() - taskDir := filepath.Join(dir, "task-1") - if err := os.MkdirAll(taskDir, 0o755); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(filepath.Join(taskDir, "movie.mkv"), []byte("movie"), 0o644); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(filepath.Join(taskDir, "movie.mkv.aria2"), []byte("control"), 0o644); err != nil { - t.Fatal(err) - } - - result, err := resultFromAria2Files( - downloadTaskWithName("task-1", "magnet", "magnet:?xt=urn:btih:abc", "movie.torrent"), - taskDir, - "Iron.Lung.2026.1080p.WEBRip.10Bit.DDP.5.1.x265-NeoNoir", - []arigo.File{{Path: filepath.Join(taskDir, "movie.mkv"), Length: 5, Selected: true}}, - ) - if err != nil { - t.Fatal(err) - } - if !result.IsDir { - t.Fatal("expected directory result") - } - if result.Path != taskDir { - t.Fatalf("expected task dir path, got %s", result.Path) - } - if result.Name != "Iron.Lung.2026.1080p.WEBRip.10Bit.DDP.5.1.x265-NeoNoir" { - t.Fatalf("expected torrent name wrapper dir, got %s", result.Name) - } - if result.Size != 5 { - t.Fatalf("expected payload-only size 5, got %d", result.Size) - } -} - -func TestResultFromDownloadedFilesWrapsMultipleTopLevelEntries(t *testing.T) { - dir := t.TempDir() - taskDir := filepath.Join(dir, "task-1") - if err := os.MkdirAll(filepath.Join(taskDir, "folder"), 0o755); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(filepath.Join(taskDir, "folder", "a.txt"), []byte("a"), 0o644); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(filepath.Join(taskDir, "root.txt"), []byte("b"), 0o644); err != nil { - t.Fatal(err) - } - - result, err := resultFromDownloadedFiles(downloadTaskWithName("task-1", "http", "https://example.com/bundle", "bundle"), taskDir, "fallback", []downloadedFile{ - {path: filepath.Join(taskDir, "folder", "a.txt"), relativePath: filepath.Join("folder", "a.txt")}, - {path: filepath.Join(taskDir, "root.txt"), relativePath: "root.txt"}, - }) - if err != nil { - t.Fatal(err) - } - if result.Path != taskDir { - t.Fatalf("expected task dir wrapper path, got %s", result.Path) - } - if result.Name != "bundle" { - t.Fatalf("expected bundle wrapper name, got %s", result.Name) - } -} - -func TestOutputNameIgnoresTorrentTaskNameForBT(t *testing.T) { - name := outputName( - downloadTaskWithName("task-1", "magnet", "magnet:?xt=urn:btih:abc", "movie.torrent"), - "movie.mkv", - ) - - if name != "movie.mkv" { - t.Fatalf("expected payload fallback name, got %s", name) - } -} - -func TestOutputNameAllowsHTTPDownloadName(t *testing.T) { - name := outputName( - downloadTaskWithName("task-1", "http", "https://example.com/movie.torrent", "movie.torrent"), - "download", - ) - - if name != "movie.torrent" { - t.Fatalf("expected HTTP task name to be preserved, got %s", name) - } -} diff --git a/cmd/internal/engine/http.go b/cmd/internal/engine/http.go deleted file mode 100644 index 25cb8210..00000000 --- a/cmd/internal/engine/http.go +++ /dev/null @@ -1,196 +0,0 @@ -package engine - -import ( - "context" - "errors" - "fmt" - "io" - "net/http" - "net/url" - "os" - "path/filepath" - "strconv" - "time" - - "github.com/saltbo/zpan/internal/client" -) - -type HTTP struct { - Dir string -} - -func (h HTTP) Name() string { - return "builtin" -} - -func (h HTTP) Capabilities() []string { - return []string{"http"} -} - -func (h HTTP) Check(ctx context.Context) error { - if err := os.MkdirAll(h.Dir, 0o755); err != nil { - return err - } - file, err := os.CreateTemp(h.Dir, ".zpan-check-*") - if err != nil { - return err - } - path := file.Name() - if err := file.Close(); err != nil { - return err - } - return os.Remove(path) -} - -func (h HTTP) ResetTask(ctx context.Context, task client.DownloadTask) error { - if task.SourceType() != "http" { - return nil - } - return os.RemoveAll(filepath.Join(h.Dir, task.ID)) -} - -func (h HTTP) InspectTask(ctx context.Context, task client.DownloadTask) (TaskSnapshot, bool, error) { - if task.SourceType() != "http" { - return TaskSnapshot{}, false, nil - } - size, ok := completedHTTPCheckpoint(task) - if !ok { - return TaskSnapshot{}, false, nil - } - path, name, err := h.outputPath(task) - if err != nil { - return TaskSnapshot{}, false, err - } - info, err := os.Stat(path) - if err != nil { - return TaskSnapshot{}, false, fmt.Errorf("inspect http completed file: %w", err) - } - if info.IsDir() { - return TaskSnapshot{}, false, fmt.Errorf("inspect http completed file: %s is a directory", path) - } - if info.Size() != size { - return TaskSnapshot{}, false, fmt.Errorf("inspect http completed file: size mismatch path=%s expected=%d actual=%d", path, size, info.Size()) - } - result := Result{Path: path, Name: name, Size: size} - return TaskSnapshot{ - State: TaskStateCompleted, - Downloaded: size, - Total: &size, - Runtime: &client.DownloadTaskRuntime{Engine: "builtin", Phase: "completed"}, - Result: &result, - }, true, nil -} - -func (h HTTP) Download(ctx context.Context, task client.DownloadTask, progress Progress) (Result, error) { - if task.SourceType() != "http" { - return Result{}, errors.New("http engine only supports http sources") - } - taskDir := filepath.Join(h.Dir, task.ID) - if err := os.MkdirAll(taskDir, 0o755); err != nil { - return Result{}, err - } - - path, name, err := h.outputPath(task) - if err != nil { - return Result{}, err - } - existingSize, err := existingFileSize(path) - if err != nil { - return Result{}, err - } - req, err := http.NewRequestWithContext(ctx, http.MethodGet, task.SourceURI(), nil) - if err != nil { - return Result{}, err - } - if existingSize > 0 { - req.Header.Set("Range", "bytes="+strconv.FormatInt(existingSize, 10)+"-") - } - - res, err := http.DefaultClient.Do(req) - if err != nil { - return Result{}, err - } - defer res.Body.Close() - if res.StatusCode == http.StatusRequestedRangeNotSatisfiable && existingSize > 0 { - return resultFromFile(task, path) - } - if res.StatusCode < 200 || res.StatusCode >= 300 { - return Result{}, errors.New(res.Status) - } - - appendExisting := existingSize > 0 && res.StatusCode == http.StatusPartialContent - if existingSize > 0 && !appendExisting { - existingSize = 0 - } - file, err := openOutputFile(path, appendExisting) - if err != nil { - return Result{}, err - } - defer file.Close() - - var total *int64 - if res.ContentLength > 0 { - value := res.ContentLength + existingSize - total = &value - } - counter := &progressWriter{progress: progress, total: total, downloaded: existingSize, lastBytes: existingSize, lastAt: time.Now()} - if _, err := io.Copy(file, io.TeeReader(res.Body, counter)); err != nil { - return Result{}, err - } - if err := progress(counter.downloaded, total, 0, &client.DownloadTaskRuntime{Engine: "builtin", Phase: "completed"}); err != nil { - return Result{}, err - } - return Result{Path: path, Name: name, Size: counter.downloaded}, nil -} - -func (h HTTP) outputPath(task client.DownloadTask) (string, string, error) { - parsed, err := httpURL(task.SourceURI()) - if err != nil { - return "", "", err - } - name := outputName(task, filenameFromURL(parsed)) - return filepath.Join(h.Dir, task.ID, name), name, nil -} - -func completedHTTPCheckpoint(task client.DownloadTask) (int64, bool) { - total := task.Status.Progress.Download.TotalBytes - if total == nil || *total <= 0 { - return 0, false - } - if task.Status.Progress.Download.Bytes != *total { - return 0, false - } - return *total, true -} - -func httpURL(raw string) (*url.URL, error) { - parsed, err := url.Parse(raw) - if err != nil { - return nil, err - } - if parsed.Scheme != "http" && parsed.Scheme != "https" { - return nil, fmt.Errorf("unsupported http source url: %s", raw) - } - return parsed, nil -} - -func existingFileSize(path string) (int64, error) { - info, err := os.Stat(path) - if err == nil { - if info.IsDir() { - return 0, nil - } - return info.Size(), nil - } - if os.IsNotExist(err) { - return 0, nil - } - return 0, err -} - -func openOutputFile(path string, appendExisting bool) (*os.File, error) { - if appendExisting { - return os.OpenFile(path, os.O_WRONLY|os.O_APPEND, 0o644) - } - return os.Create(path) -} diff --git a/cmd/internal/engine/process_windows.go b/cmd/internal/engine/process_windows.go deleted file mode 100644 index e24ef785..00000000 --- a/cmd/internal/engine/process_windows.go +++ /dev/null @@ -1,7 +0,0 @@ -//go:build windows - -package engine - -import "os/exec" - -func configureEngineProcess(cmd *exec.Cmd) {} diff --git a/cmd/internal/openapi/client.gen.go b/cmd/internal/openapi/client.gen.go index a67575ec..72cc9e6d 100644 --- a/cmd/internal/openapi/client.gen.go +++ b/cmd/internal/openapi/client.gen.go @@ -63,7 +63,7 @@ func (e DownloadTaskStatusBillingState) Valid() bool { // Defines values for DownloadTaskStatusRuntimeEngine. const ( DownloadTaskStatusRuntimeEngineAria2 DownloadTaskStatusRuntimeEngine = "aria2" - DownloadTaskStatusRuntimeEngineBuiltin DownloadTaskStatusRuntimeEngine = "builtin" + DownloadTaskStatusRuntimeEngineHttp DownloadTaskStatusRuntimeEngine = "http" DownloadTaskStatusRuntimeEngineQbittorrent DownloadTaskStatusRuntimeEngine = "qbittorrent" ) @@ -72,7 +72,7 @@ func (e DownloadTaskStatusRuntimeEngine) Valid() bool { switch e { case DownloadTaskStatusRuntimeEngineAria2: return true - case DownloadTaskStatusRuntimeEngineBuiltin: + case DownloadTaskStatusRuntimeEngineHttp: return true case DownloadTaskStatusRuntimeEngineQbittorrent: return true @@ -162,7 +162,7 @@ func (e DownloadTaskStatusState) Valid() bool { // Defines values for DownloaderEngine. const ( DownloaderEngineAria2 DownloaderEngine = "aria2" - DownloaderEngineBuiltin DownloaderEngine = "builtin" + DownloaderEngineHttp DownloaderEngine = "http" DownloaderEngineQbittorrent DownloaderEngine = "qbittorrent" ) @@ -171,7 +171,7 @@ func (e DownloaderEngine) Valid() bool { switch e { case DownloaderEngineAria2: return true - case DownloaderEngineBuiltin: + case DownloaderEngineHttp: return true case DownloaderEngineQbittorrent: return true @@ -204,7 +204,7 @@ func (e DownloaderStatus) Valid() bool { // Defines values for DownloaderHeartbeatResultEngine. const ( DownloaderHeartbeatResultEngineAria2 DownloaderHeartbeatResultEngine = "aria2" - DownloaderHeartbeatResultEngineBuiltin DownloaderHeartbeatResultEngine = "builtin" + DownloaderHeartbeatResultEngineHttp DownloaderHeartbeatResultEngine = "http" DownloaderHeartbeatResultEngineQbittorrent DownloaderHeartbeatResultEngine = "qbittorrent" ) @@ -213,7 +213,7 @@ func (e DownloaderHeartbeatResultEngine) Valid() bool { switch e { case DownloaderHeartbeatResultEngineAria2: return true - case DownloaderHeartbeatResultEngineBuiltin: + case DownloaderHeartbeatResultEngineHttp: return true case DownloaderHeartbeatResultEngineQbittorrent: return true @@ -504,7 +504,7 @@ func (e CancelBackgroundJobJSONBodyStatus) Valid() bool { // Defines values for CreateDownloaderJSONBodyHeartbeatEngine. const ( CreateDownloaderJSONBodyHeartbeatEngineAria2 CreateDownloaderJSONBodyHeartbeatEngine = "aria2" - CreateDownloaderJSONBodyHeartbeatEngineBuiltin CreateDownloaderJSONBodyHeartbeatEngine = "builtin" + CreateDownloaderJSONBodyHeartbeatEngineHttp CreateDownloaderJSONBodyHeartbeatEngine = "http" CreateDownloaderJSONBodyHeartbeatEngineQbittorrent CreateDownloaderJSONBodyHeartbeatEngine = "qbittorrent" ) @@ -513,7 +513,7 @@ func (e CreateDownloaderJSONBodyHeartbeatEngine) Valid() bool { switch e { case CreateDownloaderJSONBodyHeartbeatEngineAria2: return true - case CreateDownloaderJSONBodyHeartbeatEngineBuiltin: + case CreateDownloaderJSONBodyHeartbeatEngineHttp: return true case CreateDownloaderJSONBodyHeartbeatEngineQbittorrent: return true @@ -525,7 +525,7 @@ func (e CreateDownloaderJSONBodyHeartbeatEngine) Valid() bool { // Defines values for RecordDownloaderHeartbeatJSONBodyEngine. const ( RecordDownloaderHeartbeatJSONBodyEngineAria2 RecordDownloaderHeartbeatJSONBodyEngine = "aria2" - RecordDownloaderHeartbeatJSONBodyEngineBuiltin RecordDownloaderHeartbeatJSONBodyEngine = "builtin" + RecordDownloaderHeartbeatJSONBodyEngineHttp RecordDownloaderHeartbeatJSONBodyEngine = "http" RecordDownloaderHeartbeatJSONBodyEngineQbittorrent RecordDownloaderHeartbeatJSONBodyEngine = "qbittorrent" ) @@ -534,7 +534,7 @@ func (e RecordDownloaderHeartbeatJSONBodyEngine) Valid() bool { switch e { case RecordDownloaderHeartbeatJSONBodyEngineAria2: return true - case RecordDownloaderHeartbeatJSONBodyEngineBuiltin: + case RecordDownloaderHeartbeatJSONBodyEngineHttp: return true case RecordDownloaderHeartbeatJSONBodyEngineQbittorrent: return true @@ -633,7 +633,7 @@ func (e CreateDownloadTaskJSONBodySourceType) Valid() bool { // Defines values for UpdateDownloadTaskJSONBodyRuntimeEngine. const ( UpdateDownloadTaskJSONBodyRuntimeEngineAria2 UpdateDownloadTaskJSONBodyRuntimeEngine = "aria2" - UpdateDownloadTaskJSONBodyRuntimeEngineBuiltin UpdateDownloadTaskJSONBodyRuntimeEngine = "builtin" + UpdateDownloadTaskJSONBodyRuntimeEngineHttp UpdateDownloadTaskJSONBodyRuntimeEngine = "http" UpdateDownloadTaskJSONBodyRuntimeEngineQbittorrent UpdateDownloadTaskJSONBodyRuntimeEngine = "qbittorrent" ) @@ -642,7 +642,7 @@ func (e UpdateDownloadTaskJSONBodyRuntimeEngine) Valid() bool { switch e { case UpdateDownloadTaskJSONBodyRuntimeEngineAria2: return true - case UpdateDownloadTaskJSONBodyRuntimeEngineBuiltin: + case UpdateDownloadTaskJSONBodyRuntimeEngineHttp: return true case UpdateDownloadTaskJSONBodyRuntimeEngineQbittorrent: return true diff --git a/cmd/internal/worker/api.go b/cmd/internal/worker/api.go deleted file mode 100644 index db343ae1..00000000 --- a/cmd/internal/worker/api.go +++ /dev/null @@ -1,117 +0,0 @@ -package worker - -import ( - "context" - "fmt" - "time" - - "github.com/saltbo/zpan/internal/client" -) - -const apiRetryAttempts = 3 - -func (w *Worker) updateTask(ctx context.Context, id string, patch client.TaskPatch) (client.DownloadTask, error) { - var task client.DownloadTask - err := w.callAPI(ctx, "update task", func(ctx context.Context) error { - var err error - task, err = w.api.UpdateTask(ctx, id, patch) - return err - }) - return task, err -} - -func (w *Worker) createFolder(ctx context.Context, token string, name string, parent string) (client.ObjectDraft, error) { - var draft client.ObjectDraft - err := w.callAPI(ctx, "create folder", func(ctx context.Context) error { - var err error - draft, err = w.api.CreateFolder(ctx, token, name, parent) - return err - }) - return draft, err -} - -func (w *Worker) createObject(ctx context.Context, token string, name string, size int64, parent string) (client.ObjectDraft, error) { - var draft client.ObjectDraft - err := w.callAPI(ctx, "create object", func(ctx context.Context) error { - var err error - draft, err = w.api.CreateObject(ctx, token, name, size, parent) - return err - }) - return draft, err -} - -func (w *Worker) completeObjectUpload(ctx context.Context, token string, id string, sessionID string, parts []client.CompletedObjectUploadPart) error { - return w.callAPI(ctx, "complete upload", func(ctx context.Context) error { - return w.api.CompleteObjectUpload(ctx, token, id, sessionID, parts) - }) -} - -func (w *Worker) abortObjectUploadSession(ctx context.Context, token string, id string, sessionID string) error { - return w.callAPI(ctx, "abort multipart upload session", func(ctx context.Context) error { - return w.api.AbortObjectUploadSession(ctx, token, id, sessionID) - }) -} - -func (w *Worker) callAPI(ctx context.Context, operation string, call func(context.Context) error) error { - var last error - for attempt := 1; attempt <= apiRetryAttempts; attempt++ { - if err := ctx.Err(); err != nil { - return err - } - if err := call(ctx); err != nil { - last = err - if attempt == apiRetryAttempts || !isRetryableAPIError(err) { - return err - } - delay := time.Duration(attempt) * 500 * time.Millisecond - w.logger.Warn("retrying downloader api call", "operation", operation, "attempt", attempt, "delay", delay.String(), "error", err) - select { - case <-ctx.Done(): - return ctx.Err() - case <-time.After(delay): - } - continue - } - return nil - } - return fmt.Errorf("%s failed: %w", operation, last) -} - -func isRetryableAPIError(err error) bool { - if err == nil { - return false - } - message := err.Error() - return containsAny(message, - "connection refused", - "connection reset", - "connection is shut down", - "timeout", - "temporary failure", - "Too Many Requests", - "Bad Gateway", - "Service Unavailable", - "Gateway Timeout", - ) -} - -func containsAny(value string, needles ...string) bool { - for _, needle := range needles { - if needle != "" && contains(value, needle) { - return true - } - } - return false -} - -func contains(value string, needle string) bool { - if len(needle) > len(value) { - return false - } - for i := 0; i <= len(value)-len(needle); i++ { - if value[i:i+len(needle)] == needle { - return true - } - } - return false -} diff --git a/cmd/internal/worker/disk_unix_test.go b/cmd/internal/worker/disk_unix_test.go deleted file mode 100644 index 9b649b17..00000000 --- a/cmd/internal/worker/disk_unix_test.go +++ /dev/null @@ -1,25 +0,0 @@ -//go:build !windows - -package worker - -import ( - "testing" - - "github.com/saltbo/zpan/internal/config" - "golang.org/x/sys/unix" -) - -func TestHeartbeatReportsDownloadDirFreeDiskExactly(t *testing.T) { - downloadDir := t.TempDir() - var stat unix.Statfs_t - if err := unix.Statfs(downloadDir, &stat); err != nil { - t.Fatalf("statfs %s: %v", downloadDir, err) - } - want := int64(stat.Bavail) * int64(stat.Bsize) - - w := NewWithAPI(config.Config{DownloadDir: downloadDir}, &recordingAPI{}) - - if got := w.heartbeat().FreeDiskBytes; got != want { - t.Fatalf("expected heartbeat free disk %d, got %d", want, got) - } -} diff --git a/cmd/internal/worker/engines.go b/cmd/internal/worker/engines.go deleted file mode 100644 index 6a7a1d21..00000000 --- a/cmd/internal/worker/engines.go +++ /dev/null @@ -1,227 +0,0 @@ -package worker - -import ( - "context" - "fmt" - "os/exec" - "strings" - "time" - - "github.com/saltbo/zpan/internal/config" - "github.com/saltbo/zpan/internal/engine" -) - -func (w *Worker) resolveEngine(ctx context.Context) error { - if w.cfg.Engine == "" || w.cfg.Engine == "auto" { - if downloader, ok, err := explicitlyConfiguredExternalEngine(w.cfg, w.geoIP); err != nil { - return err - } else if ok { - return w.useConfiguredExternalEngine(ctx, downloader) - } - return w.resolveAutoEngine(ctx) - } - downloader, err := configuredEngine(w.cfg, w.geoIP) - if err != nil { - return err - } - if downloader.Name() == "builtin" { - w.useEngine(downloader, "configured built-in engine") - return nil - } - return w.useConfiguredExternalEngine(ctx, downloader) -} - -func (w *Worker) useConfiguredExternalEngine(ctx context.Context, downloader engine.Engine) error { - w.logger.Info("checking configured downloader engine", "engine", downloader.Name()) - if err := w.checkEngine(ctx, downloader); err != nil { - return fmt.Errorf("configured downloader engine %q is not available: %w", downloader.Name(), err) - } - w.useEngine(downloader, "configured external engine") - return nil -} - -func (w *Worker) resolveAutoEngine(ctx context.Context) error { - candidates := externalEngines(w.cfg, w.geoIP) - w.logger.Info("auto selecting downloader runtime", "priority", engineNames(candidates)) - for _, downloader := range candidates { - w.logger.Info("checking downloader runtime binary", "engine", downloader.Name()) - if err := w.startEngine(ctx, downloader); err != nil { - w.logger.Info("downloader runtime is not available for managed start", "engine", downloader.Name(), "error", err) - continue - } - w.useEngine(downloader, "managed runtime binary found and started") - return nil - } - downloader := engine.HTTP{Dir: w.cfg.DownloadDir} - w.useEngine(downloader, "no external downloader runtime binary is installed") - return nil -} - -func (w *Worker) useEngine(downloader engine.Engine, reason string) { - w.cfg.Engine = downloader.Name() - w.engine = downloader - w.logger.Info("selected downloader engine", "engine", downloader.Name(), "reason", reason) -} - -func (w *Worker) checkEngine(ctx context.Context, downloader engine.Engine) error { - checkCtx, cancel := context.WithTimeout(ctx, 2*time.Second) - defer cancel() - return downloader.Check(checkCtx) -} - -func (w *Worker) startEngine(ctx context.Context, downloader engine.Engine) error { - starter, ok := downloader.(engine.Starter) - if !ok { - return fmt.Errorf("%s cannot be auto started", downloader.Name()) - } - w.logger.Info("starting managed downloader runtime", "engine", downloader.Name()) - cmd, err := starter.Start(ctx) - if err != nil { - return err - } - if cmd.Process != nil { - w.logger.Info("downloader engine process started", "engine", downloader.Name(), "pid", cmd.Process.Pid) - } - w.started = append(w.started, cmd) - if err := waitForEngine(ctx, downloader); err != nil { - if cmd.Process != nil { - _ = cmd.Process.Kill() - } - _ = cmd.Wait() - return err - } - go w.watchEngineProcess(downloader.Name(), cmd) - return nil -} - -// watchEngineProcess waits on a managed engine subprocess for the worker's -// lifetime. The engine is expected to outlive every task, so any exit we did -// not initiate is fatal: log it and cancel the run context, which makes Run -// return errEngineExited and the process exit non-zero so the supervisor -// restarts the whole downloader. -func (w *Worker) watchEngineProcess(name string, cmd *exec.Cmd) { - err := cmd.Wait() - if w.isStopping() { - return - } - pid := 0 - if cmd.Process != nil { - pid = cmd.Process.Pid - } - w.logger.Error("managed downloader engine exited unexpectedly", "engine", name, "pid", pid, "error", err) - if w.cancelRun != nil { - w.cancelRun(fmt.Errorf("%w: %s (pid %d): %v", errEngineExited, name, pid, err)) - } -} - -func (w *Worker) stopStartedEngines() { - w.markStopping() - if len(w.started) > 0 { - if saver, ok := w.engine.(engine.SessionSaver); ok { - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - if err := saver.SaveSession(ctx); err != nil { - w.logger.Warn("failed to save downloader engine session", "engine", w.engine.Name(), "error", err) - } - cancel() - } - } - for _, cmd := range w.started { - if cmd.Process == nil { - continue - } - w.logger.Info("stopping auto-started downloader engine", "pid", cmd.Process.Pid) - _ = cmd.Process.Kill() - } -} - -func configuredEngine(cfg config.Config, geoIP engine.PeerGeoIPResolver) (engine.Engine, error) { - for _, downloader := range append(externalEngines(cfg, geoIP), engine.HTTP{Dir: cfg.DownloadDir}) { - if downloader.Name() == cfg.Engine { - return downloader, nil - } - } - return nil, fmt.Errorf("unsupported downloader engine %q; expected auto, builtin, aria2, or qbittorrent", cfg.Engine) -} - -func explicitlyConfiguredExternalEngine(cfg config.Config, geoIP engine.PeerGeoIPResolver) (engine.Engine, bool, error) { - configured := make([]engine.Engine, 0, 2) - for _, downloader := range externalEngines(cfg, geoIP) { - switch downloader.Name() { - case "aria2": - if cfg.Aria2Configured { - configured = append(configured, downloader) - } - case "qbittorrent": - if cfg.QBittorrentConfigured { - configured = append(configured, downloader) - } - } - } - if len(configured) == 0 { - return nil, false, nil - } - if len(configured) > 1 { - return nil, false, fmt.Errorf("multiple external downloader engines are configured; set engine to aria2 or qbittorrent") - } - return configured[0], true, nil -} - -func externalEngines(cfg config.Config, geoIP engine.PeerGeoIPResolver) []engine.Engine { - return []engine.Engine{ - engine.Aria2{ - URL: cfg.Aria2URL, - Secret: cfg.Aria2Secret, - Dir: cfg.DownloadDir, - StateDir: cfg.StateDir, - ListenPort: cfg.BTListenPort, - MaxConcurrentDownloads: aria2MaxConcurrentDownloads(cfg), - RetainSeed: cfg.SeedEnabled, - SeedDuration: cfg.SeedDuration, - SeedRatio: cfg.SeedRatio, - BtTrackers: engine.FetchBtTrackers(), - GeoIP: geoIP, - }, - engine.QBittorrent{URL: cfg.QBittorrentURL, Username: cfg.QBittorrentUser, Password: cfg.QBittorrentPass, Dir: cfg.DownloadDir, StateDir: cfg.StateDir, ListenPort: cfg.BTListenPort, RetainSeed: cfg.SeedEnabled, GeoIP: geoIP}, - } -} - -// aria2MaxConcurrentDownloads keeps download and seeding concurrency separate. -// max_concurrent_tasks is the worker's download budget; retained seeds (which -// aria2 counts as active downloads) get their own budget on top so they never -// consume a download slot. -func aria2MaxConcurrentDownloads(cfg config.Config) int { - limit := cfg.MaxConcurrentTasks - if cfg.SeedEnabled { - limit += cfg.SeedMaxConcurrent - } - return limit -} - -func waitForEngine(ctx context.Context, downloader engine.Engine) error { - deadline := time.Now().Add(8 * time.Second) - var lastErr error - for time.Now().Before(deadline) { - checkCtx, cancel := context.WithTimeout(ctx, time.Second) - err := downloader.Check(checkCtx) - cancel() - if err == nil { - return nil - } - lastErr = err - select { - case <-ctx.Done(): - return ctx.Err() - case <-time.After(500 * time.Millisecond): - } - } - return lastErr -} - -func engineNames(engines []engine.Engine) string { - names := make([]string, 0, len(engines)+1) - for _, downloader := range engines { - names = append(names, downloader.Name()) - } - names = append(names, "builtin") - return strings.Join(names, ",") -} diff --git a/cmd/main.go b/cmd/main.go index a3a66877..c95dfe5e 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -13,8 +13,11 @@ import ( "github.com/saltbo/zpan/internal/client" "github.com/saltbo/zpan/internal/config" - "github.com/saltbo/zpan/internal/host" - "github.com/saltbo/zpan/internal/worker" + "github.com/saltbo/zpan/internal/downloader" + _ "github.com/saltbo/zpan/pkg/downloaders/aria2" + _ "github.com/saltbo/zpan/pkg/downloaders/httpdl" + _ "github.com/saltbo/zpan/pkg/downloaders/qbittorrent" + "github.com/saltbo/zpan/pkg/system" "github.com/spf13/cobra" "github.com/spf13/viper" ) @@ -107,11 +110,11 @@ func upCommand(v *viper.Viper, cfgFile *string) *cobra.Command { } cfg.Token = registered.Token } - downloader, err := worker.New(cfg) + runner, err := downloader.NewTaskRunner(cfg) if err != nil { return err } - return downloader.Run(ctx) + return runner.Run(ctx) }, } } @@ -180,7 +183,7 @@ func saveRegisteredDownloaderConfig(cfg config.Config, cfgFile string, token str } func downloaderName() string { - if hostname := host.DownloaderHostname(); hostname != "" { + if hostname := system.DownloaderHostname(); hostname != "" { return hostname } return "zpan" @@ -219,8 +222,8 @@ func isPendingDeviceAuthError(err error) bool { func registrationHeartbeat(cfg config.Config) client.Heartbeat { engine := normalizeRegistrationEngine(cfg) return client.Heartbeat{ - Version: worker.Version, - Hostname: host.DownloaderHostname(), + Version: downloader.Version, + Hostname: system.DownloaderHostname(), Platform: runtime.GOOS, Arch: runtime.GOARCH, Engine: engine, @@ -235,7 +238,7 @@ func registrationHeartbeat(cfg config.Config) client.Heartbeat { func normalizeRegistrationEngine(cfg config.Config) string { switch strings.ToLower(strings.TrimSpace(cfg.Engine)) { - case "aria2", "qbittorrent", "builtin": + case "aria2", "qbittorrent", "http": return strings.ToLower(strings.TrimSpace(cfg.Engine)) } if cfg.Aria2Configured { @@ -244,22 +247,22 @@ func normalizeRegistrationEngine(cfg config.Config) string { if cfg.QBittorrentConfigured { return "qbittorrent" } - return "builtin" + return "aria2" } func validateConfiguredEngine(engine string) error { switch strings.ToLower(strings.TrimSpace(engine)) { - case "", "auto", "builtin", "aria2", "qbittorrent": + case "", "auto", "http", "aria2", "qbittorrent": return nil default: - return fmt.Errorf("unsupported downloader engine %q; expected auto, builtin, aria2, or qbittorrent", engine) + return fmt.Errorf("unsupported downloader engine %q; expected auto, http, aria2, or qbittorrent", engine) } } func runtimeCapabilities(name string) []string { switch name { case "aria2", "qbittorrent": - return []string{"http", "magnet", "torrent"} + return []string{"http", "magnet", "torrent", "torrent_url"} default: return []string{"http"} } diff --git a/cmd/main_test.go b/cmd/main_test.go index 7453ab6d..33e0c299 100644 --- a/cmd/main_test.go +++ b/cmd/main_test.go @@ -1,18 +1,28 @@ package main import ( + "bytes" + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" "testing" + "github.com/saltbo/zpan/internal/client" "github.com/saltbo/zpan/internal/config" + "github.com/spf13/cobra" ) func TestRegistrationHeartbeatNormalizesAutoEngine(t *testing.T) { heartbeat := registrationHeartbeat(config.Config{Engine: "auto", MaxConcurrentTasks: 3}) - if heartbeat.Engine != "builtin" { - t.Fatalf("expected builtin engine, got %q", heartbeat.Engine) + if heartbeat.Engine != "aria2" { + t.Fatalf("expected default BT engine, got %q", heartbeat.Engine) } - if len(heartbeat.Capabilities) != 1 || heartbeat.Capabilities[0] != "http" { - t.Fatalf("expected http-only capabilities, got %#v", heartbeat.Capabilities) + if len(heartbeat.Capabilities) != 4 || heartbeat.Capabilities[0] != "http" || heartbeat.Capabilities[1] != "magnet" || heartbeat.Capabilities[2] != "torrent" || heartbeat.Capabilities[3] != "torrent_url" { + t.Fatalf("expected BT plus HTTP capabilities, got %#v", heartbeat.Capabilities) } } @@ -23,10 +33,67 @@ func TestRegistrationHeartbeatUsesConfiguredExternalEngine(t *testing.T) { } } +func TestNormalizeRegistrationEngineBranches(t *testing.T) { + tests := []struct { + cfg config.Config + want string + }{ + {cfg: config.Config{Engine: " HTTP "}, want: "http"}, + {cfg: config.Config{Engine: "qbittorrent"}, want: "qbittorrent"}, + {cfg: config.Config{Engine: "auto", QBittorrentConfigured: true}, want: "qbittorrent"}, + {cfg: config.Config{Engine: "unknown"}, want: "aria2"}, + } + for _, tt := range tests { + if got := normalizeRegistrationEngine(tt.cfg); got != tt.want { + t.Fatalf("expected %q, got %q", tt.want, got) + } + } +} + func TestValidateConfiguredEngineRejectsUnknownEngine(t *testing.T) { if err := validateConfiguredEngine("bad-engine"); err == nil { t.Fatal("expected unsupported engine error") } + for _, engine := range []string{"", "auto", "http", "aria2", "qbittorrent", " warning "} { + if engine == " warning " { + continue + } + if err := validateConfiguredEngine(engine); err != nil { + t.Fatalf("expected valid engine %q: %v", engine, err) + } + } +} + +func TestLogLevelPendingAuthNameAndCapabilitiesHelpers(t *testing.T) { + for _, level := range []string{"debug", "info", "", "warn", "warning", "error", "bad"} { + setLogLevel(level) + } + if !isPendingDeviceAuthError(assertError("authorization_pending")) { + t.Fatal("authorization_pending should be pending") + } + if !isPendingDeviceAuthError(assertError("slow_down")) { + t.Fatal("slow_down should be pending") + } + if isPendingDeviceAuthError(assertError("access_denied")) { + t.Fatal("access_denied should not be pending") + } + if len(runtimeCapabilities("http")) != 1 || runtimeCapabilities("http")[0] != "http" { + t.Fatalf("unexpected http capabilities") + } + if downloaderName() == "" { + t.Fatal("downloader name should never be empty") + } +} + +func TestPollDeviceTokenStopsOnContextAndExpiry(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if _, err := pollDeviceToken(ctx, nil, client.DeviceCode{DeviceCode: "device-1", ExpiresIn: 60, Interval: 1}); err == nil { + t.Fatal("expected canceled context error") + } + if _, err := pollDeviceToken(context.Background(), nil, client.DeviceCode{DeviceCode: "device-1", ExpiresIn: -1, Interval: 1}); err == nil { + t.Fatal("expected expired device login") + } } func TestRootCommandExposesDownloaderSubcommands(t *testing.T) { @@ -63,3 +130,109 @@ func TestRootCommandExposesDownloaderSubcommands(t *testing.T) { t.Fatal("downloader config command should not be exposed") } } + +func TestConfigInitCommandWritesDefaultConfig(t *testing.T) { + path := filepath.Join(t.TempDir(), "config.yaml") + root := rootCommand() + root.SetArgs([]string{"--config", path, "config", "init", "--server-url", "https://zpan.test"}) + var out bytes.Buffer + root.SetOut(&out) + root.SetErr(&out) + if err := root.Execute(); err != nil { + t.Fatal(err) + } + data := readFile(t, path) + if !strings.Contains(data, `server_url: "https://zpan.test"`) { + t.Fatalf("unexpected config file:\n%s", data) + } +} + +func TestRegisterDownloaderWithDeviceLoginHappyPath(t *testing.T) { + var tokenPolls int + var createBody map[string]any + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + switch { + case r.Method == http.MethodPost && r.URL.Path == "/api/auth/device/code": + _ = json.NewEncoder(w).Encode(map[string]any{ + "device_code": "device-1", + "user_code": "ABCD-EFGH", + "verification_uri": serverURL(r) + "/device", + "verification_uri_complete": serverURL(r) + "/device?user_code=ABCD-EFGH", + "expires_in": 5, + "interval": 1, + }) + case r.Method == http.MethodPost && r.URL.Path == "/api/auth/device/token": + tokenPolls++ + _ = json.NewEncoder(w).Encode(map[string]any{ + "access_token": "access-token", + "token_type": "Bearer", + "expires_in": 3600, + }) + case r.Method == http.MethodPost && r.URL.Path == "/api/downloads/downloaders": + if r.Header.Get("Authorization") != "Bearer access-token" { + t.Fatalf("unexpected auth header: %q", r.Header.Get("Authorization")) + } + if err := json.NewDecoder(r.Body).Decode(&createBody); err != nil { + t.Fatal(err) + } + w.WriteHeader(http.StatusCreated) + _ = json.NewEncoder(w).Encode(map[string]any{ + "downloader": map[string]any{"id": "downloader-1"}, + "token": "downloader-token", + }) + default: + t.Fatalf("unexpected request: %s %s", r.Method, r.URL.Path) + } + })) + defer server.Close() + + cfgFile := filepath.Join(t.TempDir(), "config.yaml") + cmd := &cobra.Command{} + var out bytes.Buffer + cmd.SetOut(&out) + registered, err := registerDownloaderWithDeviceLogin(context.Background(), cmd, config.Config{ + ServerURL: server.URL, + Engine: "http", + MaxConcurrentTasks: 2, + }, cfgFile) + if err != nil { + t.Fatal(err) + } + if registered.Token != "downloader-token" || registered.Downloader.ID != "downloader-1" { + t.Fatalf("unexpected registration response: %#v", registered) + } + if tokenPolls != 1 { + t.Fatalf("expected one token poll, got %d", tokenPolls) + } + heartbeat, ok := createBody["heartbeat"].(map[string]any) + if !ok || heartbeat["engine"] != "http" || heartbeat["maxConcurrentTasks"] != float64(2) { + t.Fatalf("unexpected create downloader body: %#v", createBody) + } + if !strings.Contains(out.String(), "Downloader registered: downloader-1") { + t.Fatalf("unexpected output: %s", out.String()) + } + data := readFile(t, cfgFile) + if !strings.Contains(data, `token: "downloader-token"`) { + t.Fatalf("registered token was not saved:\n%s", data) + } +} + +type assertError string + +func (e assertError) Error() string { + return string(e) +} + +func readFile(t *testing.T, path string) string { + t.Helper() + data, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + return string(data) +} + +func serverURL(r *http.Request) string { + return "http://" + r.Host +} diff --git a/cmd/internal/engine/aria2.go b/cmd/pkg/downloaders/aria2/aria2.go similarity index 78% rename from cmd/internal/engine/aria2.go rename to cmd/pkg/downloaders/aria2/aria2.go index 286fc9c9..ab79ae5c 100644 --- a/cmd/internal/engine/aria2.go +++ b/cmd/pkg/downloaders/aria2/aria2.go @@ -1,4 +1,4 @@ -package engine +package aria2 import ( "context" @@ -20,9 +20,38 @@ import ( "github.com/Braurbeki/arigo" "github.com/cenkalti/rpc2" - "github.com/saltbo/zpan/internal/client" + "github.com/saltbo/zpan/internal/downloader" + "github.com/saltbo/zpan/pkg/downloaders/core" + "github.com/saltbo/zpan/pkg/downloaders/httpdl" + "github.com/saltbo/zpan/pkg/geoip" + "github.com/saltbo/zpan/pkg/system" ) +func init() { + downloader.Register("aria2", false, configured, New) +} + +func configured(cfg downloader.Config) bool { + return cfg.Aria2.Configured +} + +func New(cfg downloader.Config) (downloader.Downloader, error) { + return &Aria2{ + URL: cfg.Aria2.URL, + Secret: cfg.Aria2.Secret, + Dir: cfg.DownloadDir, + StateDir: cfg.StateDir, + ListenPort: cfg.BTListenPort, + MaxConcurrentDownloads: cfg.MaxConcurrentDownloads, + RetainSeed: cfg.SeedEnabled, + SeedDuration: cfg.SeedDuration, + SeedRatio: cfg.SeedRatio, + BtTrackers: cfg.Aria2.BtTrackers, + Managed: !cfg.Aria2.Configured && (cfg.Engine == "" || cfg.Engine == "auto" || cfg.Engine == "aria2"), + GeoIP: cfg.GeoIP, + }, nil +} + type Aria2 struct { URL string Secret string @@ -33,10 +62,12 @@ type Aria2 struct { RetainSeed bool SeedDuration time.Duration SeedRatio float64 - // BtTrackers is the comma-separated --bt-tracker list (aria2 format). When - // empty, startArgs falls back to the bundled defaultBtTrackers snapshot. + // BtTrackers is the comma-separated --bt-tracker list. When empty, startup + // uses the shared downloader tracker list with a logged fallback. BtTrackers string - GeoIP PeerGeoIPResolver + Managed bool + GeoIP geoip.Resolver + cmd *exec.Cmd } // aria2 measures --seed-time in minutes. A zero seed duration means "seed @@ -79,84 +110,61 @@ func (a Aria2) Name() string { return "aria2" } -func (a Aria2) Capabilities() []string { - return []string{"http", "magnet", "torrent"} +func (a Aria2) Capabilities() downloader.Capabilities { + return downloader.Capabilities{SourceTypes: []string{"magnet", "torrent", "torrent_url"}} } -func (a Aria2) Start(ctx context.Context) (*exec.Cmd, error) { +func (a *Aria2) Start(ctx context.Context) error { + if !a.Managed { + return nil + } path, err := exec.LookPath("aria2c") if err != nil { - return nil, err + return err } - rpcURL, err := parseLocalEngineURL(a.URL, "6800") + rpcURL, err := system.ParseLocalEngineURL(a.URL, "6800") if err != nil { - return nil, err + return err } - args, err := a.startArgs(rpcURL.port) + args, err := a.startArgs(rpcURL.Port) if err != nil { - return nil, err + return err } cmd := exec.Command(path, args...) - configureEngineProcess(cmd) + system.ConfigureProcess(cmd) if err := cmd.Start(); err != nil { - return nil, err + return err } - return cmd, nil + a.cmd = cmd + return nil } -// btTrackerListURL is the maintained XIU2 TrackersListCollection "best" list in -// aria2 (comma-separated) format. Fetched at startup so dead magnet trackers are -// supplemented with currently-healthy ones. -const btTrackerListURL = "https://raw.githubusercontent.com/XIU2/TrackersListCollection/master/best_aria2.txt" - -// defaultBtTrackers is the bundled fallback (a snapshot of the XIU2 best list) -// used only when the live list can't be fetched. Combined with DHT + PEX this is -// what lets stale magnets — whose own announce list has rotted — still resolve. -var defaultBtTrackers = []string{ - "udp://tracker.opentrackr.org:1337/announce", - "udp://open.demonii.com:1337/announce", - "udp://open.stealth.si:80/announce", - "udp://tracker.torrent.eu.org:451/announce", - "udp://exodus.desync.com:6969/announce", - "udp://tracker.openbittorrent.com:6969/announce", - "udp://opentracker.i2p.rocks:6969/announce", - "udp://tracker.dler.org:6969/announce", - "http://tracker.openbittorrent.com:80/announce", - "udp://tracker.moeking.me:6969/announce", -} - -// FetchBtTrackers pulls the maintained XIU2 best-tracker list, falling back to -// the bundled snapshot if the network fetch fails or returns empty. -func FetchBtTrackers() string { - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - req, err := http.NewRequestWithContext(ctx, http.MethodGet, btTrackerListURL, nil) - if err != nil { - return strings.Join(defaultBtTrackers, ",") +func (a *Aria2) Stop(ctx context.Context) error { + if a.cmd == nil { + return nil } - resp, err := http.DefaultClient.Do(req) - if err != nil { - return strings.Join(defaultBtTrackers, ",") + var errs []error + if err := a.SaveSession(ctx); err != nil { + errs = append(errs, fmt.Errorf("save aria2 session: %w", err)) } - defer resp.Body.Close() - if resp.StatusCode != http.StatusOK { - return strings.Join(defaultBtTrackers, ",") + if a.cmd.Process != nil { + if err := a.cmd.Process.Kill(); err != nil { + errs = append(errs, fmt.Errorf("kill aria2 process: %w", err)) + } } - body, err := io.ReadAll(io.LimitReader(resp.Body, 1<<20)) - if err != nil { - return strings.Join(defaultBtTrackers, ",") + done := make(chan error, 1) + go func() { done <- a.cmd.Wait() }() + select { + case <-done: + case <-ctx.Done(): + errs = append(errs, ctx.Err()) } - if list := strings.TrimSpace(string(body)); list != "" { - return list - } - return strings.Join(defaultBtTrackers, ",") + a.cmd = nil + return errors.Join(errs...) } func (a Aria2) startArgs(rpcPort string) ([]string, error) { - trackers := a.BtTrackers - if trackers == "" { - trackers = strings.Join(defaultBtTrackers, ",") - } + trackers := core.BTTrackers(a.BtTrackers) args := []string{ "--enable-rpc=true", "--rpc-listen-all=false", @@ -227,9 +235,9 @@ func (a Aria2) Check(ctx context.Context) error { return nil } -func (a Aria2) ResetTask(ctx context.Context, task client.DownloadTask) error { +func (a Aria2) ResetTask(ctx context.Context, task downloader.DownloadTask) error { if task.SourceType() == "http" { - return HTTP{Dir: a.Dir}.ResetTask(ctx, task) + return httpdl.HTTP{Dir: a.Dir}.ResetTask(ctx, task) } aria, err := a.client(ctx) if err != nil { @@ -308,7 +316,7 @@ func (a Aria2) SaveSession(ctx context.Context) error { return client.SaveSession() } -func (a Aria2) RestoreSeed(ctx context.Context, ref SeedRef) (*Seed, error) { +func (a Aria2) RestoreSeed(ctx context.Context, ref downloader.SeedRef) (*downloader.Seed, error) { aria, err := a.client(ctx) if err != nil { return nil, err @@ -329,7 +337,7 @@ func (a Aria2) RestoreSeed(ctx context.Context, ref SeedRef) (*Seed, error) { if path == "" { path = aria2StatusTaskDir(status, filepath.Join(a.Dir, ref.TaskID)) } - return &Seed{ + return &downloader.Seed{ Engine: "aria2", ID: status.GID, InfoHash: strings.ToLower(status.InfoHash), @@ -339,7 +347,7 @@ func (a Aria2) RestoreSeed(ctx context.Context, ref SeedRef) (*Seed, error) { }, nil } -func (a Aria2) ListSeeds(ctx context.Context) ([]Seed, error) { +func (a Aria2) ListSeeds(ctx context.Context) ([]downloader.Seed, error) { aria, err := a.client(ctx) if err != nil { return nil, err @@ -349,7 +357,7 @@ func (a Aria2) ListSeeds(ctx context.Context) ([]Seed, error) { if err != nil { return nil, err } - var seeds []Seed + var seeds []downloader.Seed for _, status := range statuses { if !isAria2SeedStatus(status) { continue @@ -358,7 +366,7 @@ func (a Aria2) ListSeeds(ctx context.Context) ([]Seed, error) { if path == "" { continue } - seeds = append(seeds, Seed{ + seeds = append(seeds, downloader.Seed{ Engine: "aria2", ID: status.GID, InfoHash: strings.ToLower(status.InfoHash), @@ -370,41 +378,41 @@ func (a Aria2) ListSeeds(ctx context.Context) ([]Seed, error) { return seeds, nil } -func (a Aria2) InspectTask(ctx context.Context, task client.DownloadTask) (TaskSnapshot, bool, error) { +func (a Aria2) InspectTask(ctx context.Context, task downloader.DownloadTask) (downloader.TaskSnapshot, bool, error) { if task.SourceType() == "http" { - return HTTP{Dir: a.Dir}.InspectTask(ctx, task) + return httpdl.HTTP{Dir: a.Dir}.InspectTask(ctx, task) } aria, err := a.client(ctx) if err != nil { - return TaskSnapshot{}, false, err + return downloader.TaskSnapshot{}, false, err } defer aria.Close() status, ok, err := a.findTask(ctx, &aria, task) if err != nil || !ok { - return TaskSnapshot{}, ok, err + return downloader.TaskSnapshot{}, ok, err } return a.snapshotTask(ctx, &aria, task, status) } -func (a Aria2) Download(ctx context.Context, task client.DownloadTask, progress Progress) (Result, error) { +func (a Aria2) Download(ctx context.Context, task downloader.DownloadTask, progress downloader.ProgressReporter) (downloader.Result, error) { if task.SourceType() == "http" { - return HTTP{Dir: a.Dir}.Download(ctx, task, progress) + return httpdl.HTTP{Dir: a.Dir}.Download(ctx, task, progress) } taskDir := filepath.Join(a.Dir, task.ID) if err := os.MkdirAll(taskDir, 0o755); err != nil { - return Result{}, err + return downloader.Result{}, err } aria, err := a.client(ctx) if err != nil { - return Result{}, err + return downloader.Result{}, err } defer aria.Close() if shouldAttachExistingAria2Task(task) { status, ok, err := a.findTask(ctx, &aria, task) if err != nil { - return Result{}, fmt.Errorf("find aria2 task: %w", err) + return downloader.Result{}, fmt.Errorf("find aria2 task: %w", err) } if ok { if string(status.Status) == string(arigo.StatusPaused) { @@ -438,14 +446,14 @@ func (a Aria2) Download(ctx context.Context, task client.DownloadTask, progress gid, err := addAria2Task(ctx, aria, task, options) if err != nil { if !isAria2InfoHashAlreadyRegistered(err) { - return Result{}, fmt.Errorf("add aria2 task: %w", err) + return downloader.Result{}, fmt.Errorf("add aria2 task: %w", err) } status, ok, findErr := a.findTask(ctx, &aria, task) if findErr != nil { - return Result{}, fmt.Errorf("find aria2 task after add failed: %w", findErr) + return downloader.Result{}, fmt.Errorf("find aria2 task after add failed: %w", findErr) } if !ok { - return Result{}, fmt.Errorf("add aria2 uri: %w", err) + return downloader.Result{}, fmt.Errorf("add aria2 uri: %w", err) } gid.GID = status.GID taskDir = aria2StatusTaskDir(status, taskDir) @@ -455,18 +463,18 @@ func (a Aria2) Download(ctx context.Context, task client.DownloadTask, progress if isAria2InfoHashAlreadyRegistered(err) { status, ok, findErr := a.findTask(ctx, &aria, task) if findErr != nil { - return Result{}, fmt.Errorf("find aria2 task after infohash conflict: %w", findErr) + return downloader.Result{}, fmt.Errorf("find aria2 task after infohash conflict: %w", findErr) } if ok { return a.waitResult(ctx, &aria, task, aria2StatusTaskDir(status, taskDir), status.GID, progress) } } - return Result{}, fmt.Errorf("wait aria2 result: %w", err) + return downloader.Result{}, fmt.Errorf("wait aria2 result: %w", err) } return result, nil } -func shouldAttachExistingAria2Task(task client.DownloadTask) bool { +func shouldAttachExistingAria2Task(task downloader.DownloadTask) bool { // 'interrupted' is how a task comes back after a downloader restart. aria2 // reloads the same download from its saved session, so we must attach to it // — re-adding would create a duplicate that errors (infohash already @@ -482,9 +490,9 @@ func shouldAttachExistingAria2Task(task client.DownloadTask) bool { func (a Aria2) snapshotTask( ctx context.Context, aria **arigo.Client, - task client.DownloadTask, + task downloader.DownloadTask, status arigo.Status, -) (TaskSnapshot, bool, error) { +) (downloader.TaskSnapshot, bool, error) { total := int64(status.TotalLength) completed := int64(status.CompletedLength) bps := int64(status.DownloadSpeed) @@ -493,7 +501,7 @@ func (a Aria2) snapshotTask( totalPtr = &total } peers := a.getAria2Peers(ctx, aria, status.GID) - snapshot := TaskSnapshot{ + snapshot := downloader.TaskSnapshot{ State: aria2TaskState(status), Downloaded: completed, Total: totalPtr, @@ -501,17 +509,17 @@ func (a Aria2) snapshotTask( Runtime: aria2Detail(status, peers, a.GeoIP), Error: status.ErrorMessage, } - if snapshot.State != TaskStateCompleted { + if snapshot.State != downloader.TaskStateCompleted { return snapshot, true, nil } files, err := a.getAria2Files(ctx, aria, status.GID) if err != nil { - return TaskSnapshot{}, false, err + return downloader.TaskSnapshot{}, false, err } taskDir := aria2StatusTaskDir(status, filepath.Join(a.Dir, task.ID)) result, err := resultFromAria2Files(task, taskDir, status.BitTorrent.Info.Name, files) if err != nil { - return TaskSnapshot{}, false, err + return downloader.TaskSnapshot{}, false, err } if a.RetainSeed && task.SourceType() != "http" { result.Seed = a.seedFromStatus(status, taskDir) @@ -520,19 +528,19 @@ func (a Aria2) snapshotTask( return snapshot, true, nil } -func aria2TaskState(status arigo.Status) TaskState { +func aria2TaskState(status arigo.Status) downloader.TaskState { if isAria2DownloadComplete(status) { - return TaskStateCompleted + return downloader.TaskStateCompleted } switch string(status.Status) { case string(arigo.StatusError), string(arigo.StatusRemoved): - return TaskStateFailed + return downloader.TaskStateFailed default: - return TaskStateDownloading + return downloader.TaskStateDownloading } } -func addAria2Task(ctx context.Context, aria *arigo.Client, task client.DownloadTask, options *arigo.Options) (arigo.GID, error) { +func addAria2Task(ctx context.Context, aria *arigo.Client, task downloader.DownloadTask, options *arigo.Options) (arigo.GID, error) { if task.SourceType() != "torrent_url" { return aria.AddURI(arigo.URIs(task.SourceURI()), options) } @@ -555,16 +563,16 @@ func addAria2Task(ctx context.Context, aria *arigo.Client, task client.DownloadT return aria.AddTorrent(data, []string{}, options) } -func (a Aria2) waitResult(ctx context.Context, aria **arigo.Client, task client.DownloadTask, taskDir string, gid string, progress Progress) (Result, error) { +func (a Aria2) waitResult(ctx context.Context, aria **arigo.Client, task downloader.DownloadTask, taskDir string, gid string, progress downloader.ProgressReporter) (downloader.Result, error) { initialProgress := progress if task.SourceType() != "http" { - initialProgress = func(downloaded int64, total *int64, bps int64, detail *client.DownloadTaskRuntime) error { return nil } + initialProgress = func(downloader.ProgressUpdate) error { return nil } } primaryGID := gid status, err := a.waitAria2(ctx, aria, task, primaryGID, initialProgress) if err != nil { _ = (*aria).Remove(primaryGID) - return Result{}, fmt.Errorf("wait primary gid %s: %w", primaryGID, err) + return downloader.Result{}, fmt.Errorf("wait primary gid %s: %w", primaryGID, err) } resultGID := status.GID if len(status.FollowedBy) > 0 { @@ -572,17 +580,17 @@ func (a Aria2) waitResult(ctx context.Context, aria **arigo.Client, task client. status, err = a.waitAria2(ctx, aria, task, childGID, progress) if err != nil { _ = (*aria).Remove(childGID) - return Result{}, fmt.Errorf("wait followed gid %s: %w", childGID, err) + return downloader.Result{}, fmt.Errorf("wait followed gid %s: %w", childGID, err) } resultGID = status.GID } files, err := a.getAria2Files(ctx, aria, status.GID) if err != nil { - return Result{}, fmt.Errorf("get files for gid %s: %w", status.GID, err) + return downloader.Result{}, fmt.Errorf("get files for gid %s: %w", status.GID, err) } result, err := resultFromAria2Files(task, taskDir, status.BitTorrent.Info.Name, files) if err != nil { - return Result{}, fmt.Errorf("build result from aria2 files: %w", err) + return downloader.Result{}, fmt.Errorf("build result from aria2 files: %w", err) } if a.RetainSeed && task.SourceType() != "http" { result.Seed = a.seedFromStatus(status, taskDir) @@ -597,8 +605,8 @@ func (a Aria2) waitResult(ctx context.Context, aria **arigo.Client, task client. return result, nil } -func (a Aria2) seedFromStatus(status arigo.Status, taskDir string) *Seed { - return &Seed{ +func (a Aria2) seedFromStatus(status arigo.Status, taskDir string) *downloader.Seed { + return &downloader.Seed{ Engine: "aria2", ID: status.GID, InfoHash: strings.ToLower(status.InfoHash), @@ -617,7 +625,7 @@ func isAria2DownloadComplete(status arigo.Status) bool { return string(status.Status) == string(arigo.StatusCompleted) } -func (a Aria2) findSeed(ctx context.Context, aria **arigo.Client, ref SeedRef) (arigo.Status, bool, error) { +func (a Aria2) findSeed(ctx context.Context, aria **arigo.Client, ref downloader.SeedRef) (arigo.Status, bool, error) { taskDir := aria2SeedTaskDir(a.Dir, ref) if ref.ID != "" { status, err := tellAria2Status(*aria, ref.ID) @@ -649,7 +657,7 @@ func (a Aria2) findSeed(ctx context.Context, aria **arigo.Client, ref SeedRef) ( return arigo.Status{}, false, nil } -func selectAria2SeedStatus(statuses []arigo.Status, ref SeedRef, taskDir string) (arigo.Status, bool) { +func selectAria2SeedStatus(statuses []arigo.Status, ref downloader.SeedRef, taskDir string) (arigo.Status, bool) { infoHash := strings.ToLower(ref.InfoHash) for _, status := range statuses { if !isAria2SeedStatus(status) { @@ -665,7 +673,7 @@ func selectAria2SeedStatus(statuses []arigo.Status, ref SeedRef, taskDir string) return arigo.Status{}, false } -func aria2SeedTaskDir(root string, ref SeedRef) string { +func aria2SeedTaskDir(root string, ref downloader.SeedRef) string { taskDir := filepath.Clean(ref.Path) if taskDir == "." || taskDir == string(filepath.Separator) { taskDir = filepath.Clean(filepath.Join(root, ref.TaskID)) @@ -682,7 +690,7 @@ func (a Aria2) client(ctx context.Context) (*arigo.Client, error) { return arigo.DialContext(ctx, a.URL, a.Secret) } -func (a Aria2) findTask(ctx context.Context, aria **arigo.Client, task client.DownloadTask) (arigo.Status, bool, error) { +func (a Aria2) findTask(ctx context.Context, aria **arigo.Client, task downloader.DownloadTask) (arigo.Status, bool, error) { gid := aria2TaskGID(task.ID) status, err := tellAria2Status(*aria, gid) if err == nil { @@ -751,7 +759,7 @@ func aria2TaskGID(taskID string) string { return hex.EncodeToString(sum[:])[:16] } -func aria2TaskInfoHash(task client.DownloadTask) string { +func aria2TaskInfoHash(task downloader.DownloadTask) string { runtime := task.Runtime() if runtime != nil && runtime.Torrent != nil && runtime.Torrent.InfoHash != "" { return strings.ToLower(runtime.Torrent.InfoHash) @@ -808,7 +816,7 @@ func aria2StatusMatchesTask(status arigo.Status, taskDir string, gid string, inf if file.Path == "" { continue } - abs, _ := downloadedPath(taskDir, file.Path) + abs, _ := core.DownloadedPath(taskDir, file.Path) if strings.HasPrefix(filepath.Clean(abs), taskDir+string(filepath.Separator)) { return true } @@ -827,7 +835,7 @@ func aria2StatusBelongsToTask(status arigo.Status, taskDir string, gid string) b if file.Path == "" { continue } - abs, _ := downloadedPath(taskDir, file.Path) + abs, _ := core.DownloadedPath(taskDir, file.Path) if strings.HasPrefix(filepath.Clean(abs), taskDir+string(filepath.Separator)) { return true } @@ -850,16 +858,16 @@ func isAria2InfoHashAlreadyRegistered(err error) bool { return strings.Contains(msg, "InfoHash") && strings.Contains(msg, "already registered") } -func (a Aria2) seedSnapshot(gid string) func(context.Context) (SeedSnapshot, error) { - return func(ctx context.Context) (SeedSnapshot, error) { +func (a Aria2) seedSnapshot(gid string) func(context.Context) (downloader.SeedSnapshot, error) { + return func(ctx context.Context) (downloader.SeedSnapshot, error) { aria, err := a.client(ctx) if err != nil { - return SeedSnapshot{}, err + return downloader.SeedSnapshot{}, err } defer aria.Close() status, err := tellAria2Status(aria, gid) if err != nil { - return SeedSnapshot{}, err + return downloader.SeedSnapshot{}, err } peers := a.getAria2Peers(ctx, &aria, gid) total := int64(status.TotalLength) @@ -869,7 +877,7 @@ func (a Aria2) seedSnapshot(gid string) func(context.Context) (SeedSnapshot, err } detail := aria2Detail(status, peers, a.GeoIP) detail.Phase = "seeding" - return SeedSnapshot{ + return downloader.SeedSnapshot{ Downloaded: int64(status.CompletedLength), Total: totalPtr, Bps: int64(status.DownloadSpeed), @@ -896,7 +904,7 @@ func (a Aria2) cleanupSeed(gid string, localPath string) func(context.Context) e } } -func (a Aria2) waitAria2(ctx context.Context, aria **arigo.Client, task client.DownloadTask, gid string, progress Progress) (arigo.Status, error) { +func (a Aria2) waitAria2(ctx context.Context, aria **arigo.Client, task downloader.DownloadTask, gid string, progress downloader.ProgressReporter) (arigo.Status, error) { ticker := time.NewTicker(time.Second) defer ticker.Stop() for { @@ -933,7 +941,12 @@ func (a Aria2) waitAria2(ctx context.Context, aria **arigo.Client, task client.D totalPtr = &total } peers := a.getAria2Peers(ctx, aria, gid) - if err := progress(completed, totalPtr, bps, aria2Detail(status, peers, a.GeoIP)); err != nil { + if err := progress(downloader.ProgressUpdate{ + Downloaded: completed, + Total: totalPtr, + Bps: bps, + Runtime: aria2Detail(status, peers, a.GeoIP), + }); err != nil { _ = (*aria).ForcePause(gid) return arigo.Status{}, err } @@ -1006,27 +1019,27 @@ func isAria2RPCDisconnected(err error) bool { return errors.Is(err, rpc2.ErrShutdown) || errors.Is(err, io.ErrClosedPipe) || strings.Contains(err.Error(), "connection is shut down") } -func aria2Detail(status arigo.Status, peers []arigo.Peer, geoIP PeerGeoIPResolver) *client.DownloadTaskRuntime { +func aria2Detail(status arigo.Status, peers []arigo.Peer, geoIP geoip.Resolver) *downloader.TaskRuntime { connections := int64(status.Connections) seeders := int64(status.NumSeeders) peerCount := int64(len(peers)) leechers := aria2Leechers(peers) uploaded := int64(status.UploadLength) uploadBps := int64(status.UploadSpeed) - detail := &client.DownloadTaskRuntime{ + detail := &downloader.TaskRuntime{ Engine: "aria2", Phase: aria2Phase(string(status.Status), status.FollowedBy), State: string(status.Status), ETASeconds: aria2ETA(status), Connections: &connections, - Torrent: &client.DownloadTaskTorrentRuntime{ + Torrent: &downloader.TorrentRuntime{ InfoHash: status.InfoHash, Name: status.BitTorrent.Info.Name, Seeders: &seeders, Leechers: leechers, Peers: &peerCount, }, - Seeding: &client.DownloadTaskSeedingRuntime{ + Seeding: &downloader.SeedingRuntime{ UploadedBytes: &uploaded, UploadBytesPerSecond: &uploadBps, }, @@ -1070,8 +1083,8 @@ func aria2Phase(state string, followedBy []string) string { } } -func aria2Trackers(announceList [][]string) []client.DownloadTaskTracker { - trackers := make([]client.DownloadTaskTracker, 0, 20) +func aria2Trackers(announceList [][]string) []downloader.Tracker { + trackers := make([]downloader.Tracker, 0, 20) seen := map[string]struct{}{} for _, tier := range announceList { for _, url := range tier { @@ -1082,7 +1095,7 @@ func aria2Trackers(announceList [][]string) []client.DownloadTaskTracker { continue } seen[url] = struct{}{} - trackers = append(trackers, client.DownloadTaskTracker{ + trackers = append(trackers, downloader.Tracker{ URL: url, Status: "announce", Message: "aria2 exposes announce URLs only", @@ -1136,21 +1149,21 @@ func aria2PeerProgress(peer arigo.Peer) *float64 { return &progress } -func aria2Peers(peers []arigo.Peer, geoIP PeerGeoIPResolver) []client.DownloadTaskPeer { - out := make([]client.DownloadTaskPeer, 0, min(len(peers), 20)) +func aria2Peers(peers []arigo.Peer, geoIP geoip.Resolver) []downloader.Peer { + out := make([]downloader.Peer, 0, min(len(peers), 20)) for _, peer := range peers { if peer.IP == "" { continue } down := int64(peer.DownloadSpeed) up := int64(peer.UploadSpeed) - item := client.DownloadTaskPeer{ + item := downloader.Peer{ Address: fmt.Sprintf("%s:%d", peer.IP, peer.Port), Progress: aria2PeerProgress(peer), DownloadBps: &down, UploadBps: &up, } - applyPeerRegion(&item, peer.IP, "", geoIP) + core.ApplyPeerRegion(&item, peer.IP, "", geoIP) out = append(out, item) if len(out) >= 20 { break @@ -1159,18 +1172,18 @@ func aria2Peers(peers []arigo.Peer, geoIP PeerGeoIPResolver) []client.DownloadTa return out } -func aria2Files(baseDir string, torrentName string, files []arigo.File) []client.DownloadTaskFile { - out := make([]client.DownloadTaskFile, 0, min(len(files), 50)) +func aria2Files(baseDir string, torrentName string, files []arigo.File) []downloader.File { + out := make([]downloader.File, 0, min(len(files), 50)) for _, file := range files { - if file.Path == "" || isAria2MetadataPath(file.Path) { + if file.Path == "" || system.IsAria2MetadataPath(file.Path) { continue } - _, rel := downloadedPath(baseDir, file.Path) - rel = stripTorrentRoot(rel, torrentName) + _, rel := core.DownloadedPath(baseDir, file.Path) + rel = core.StripTorrentRoot(rel, torrentName) size := int64(file.Length) completed := int64(file.CompletedLength) selected := file.Selected - out = append(out, client.DownloadTaskFile{ + out = append(out, downloader.File{ Path: filepath.ToSlash(rel), Size: size, CompletedBytes: &completed, @@ -1185,23 +1198,23 @@ func aria2Files(baseDir string, torrentName string, files []arigo.File) []client func hasAria2LocalFile(files []arigo.File) bool { for _, file := range files { - if file.Path != "" && !isAria2MetadataPath(file.Path) { + if file.Path != "" && !system.IsAria2MetadataPath(file.Path) { return true } } return false } -func resultFromAria2Files(task client.DownloadTask, taskDir string, fallbackName string, files []arigo.File) (Result, error) { - downloaded := make([]downloadedFile, 0, len(files)) +func resultFromAria2Files(task downloader.DownloadTask, taskDir string, fallbackName string, files []arigo.File) (downloader.Result, error) { + downloaded := make([]core.DownloadedFile, 0, len(files)) for _, file := range files { - if file.Selected && file.Length > 0 && !isDownloadSidecarPath(file.Path) { - abs, rel := downloadedPath(taskDir, file.Path) - downloaded = append(downloaded, downloadedFile{path: abs, relativePath: rel}) + if file.Selected && file.Length > 0 && !system.IsDownloadSidecarPath(file.Path) { + abs, rel := core.DownloadedPath(taskDir, file.Path) + downloaded = append(downloaded, core.DownloadedFile{Path: abs, RelativePath: rel}) } } if len(downloaded) == 0 { - return resultFromPath(task, taskDir, fallbackName) + return core.ResultFromPath(task, taskDir, fallbackName) } - return resultFromDownloadedFiles(task, taskDir, fallbackName, downloaded) + return core.ResultFromDownloadedFiles(task, taskDir, fallbackName, downloaded) } diff --git a/cmd/pkg/downloaders/aria2/aria2_test.go b/cmd/pkg/downloaders/aria2/aria2_test.go new file mode 100644 index 00000000..15206214 --- /dev/null +++ b/cmd/pkg/downloaders/aria2/aria2_test.go @@ -0,0 +1,570 @@ +package aria2 + +import ( + "context" + "errors" + "fmt" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/Braurbeki/arigo" + "github.com/cenkalti/rpc2" + "github.com/saltbo/zpan/internal/downloader" +) + +func downloadTask(id, sourceType, sourceURI string) downloader.DownloadTask { + return downloader.DownloadTask{ID: id, Source: downloader.Source{Type: sourceType, URI: sourceURI}, Labels: downloader.Labels{Tags: []string{}}} +} + +func downloadTaskWithName(id, sourceType, sourceURI, name string) downloader.DownloadTask { + task := downloadTask(id, sourceType, sourceURI) + task.Destination.Name = name + return task +} + +func TestAria2DelegatesHTTPToBuiltin(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Length", "11") + _, _ = w.Write([]byte("hello world")) + })) + defer server.Close() + + result, err := (Aria2{URL: "ws://127.0.0.1:1/jsonrpc", Dir: t.TempDir()}).Download( + context.Background(), + downloadTask("task-1", "http", server.URL+"/file.txt"), + func(update downloader.ProgressUpdate) error { + if update.Runtime != nil && update.Runtime.Engine != "http" { + t.Fatalf("expected http HTTP runtime detail, got %#v", update.Runtime) + } + return nil + }, + ) + if err != nil { + t.Fatal(err) + } + data, err := os.ReadFile(result.Path) + if err != nil { + t.Fatal(err) + } + if string(data) != "hello world" { + t.Fatalf("unexpected file content: %q", string(data)) + } +} + +func TestAria2StartArgs(t *testing.T) { + stateDir := t.TempDir() + args, err := (Aria2{ + Dir: t.TempDir(), + StateDir: stateDir, + ListenPort: 51413, + MaxConcurrentDownloads: 25, + BtTrackers: "udp://custom.example:1337/announce", + }).startArgs("6800") + if err != nil { + t.Fatal(err) + } + joined := strings.Join(args, "\n") + for _, expected := range []string{ + "--save-session=" + filepath.Join(stateDir, "aria2.session"), + "--save-session-interval=30", + "--force-save=true", + "--listen-port=51413", + "--enable-dht=true", + "--enable-peer-exchange=true", + "--bt-tracker=udp://custom.example:1337/announce", + "--max-concurrent-downloads=25", + "--dht-file-path=" + filepath.Join(stateDir, "dht.dat"), + } { + if !strings.Contains(joined, expected) { + t.Fatalf("expected aria2 args to contain %q, got %v", expected, args) + } + } + if strings.Contains(joined, "--input-file=") { + t.Fatalf("aria2 must not auto-restore old session tasks, got %v", args) + } +} + +func TestAria2ConstructorAndMetadata(t *testing.T) { + cfg := downloader.Config{ + Engine: "aria2", + DownloadDir: t.TempDir(), + StateDir: t.TempDir(), + BTListenPort: 51413, + MaxConcurrentDownloads: 12, + SeedEnabled: true, + SeedDuration: time.Hour, + SeedRatio: 1.5, + Aria2: downloader.Aria2Config{ + URL: "ws://127.0.0.1:6800/jsonrpc", + Secret: "secret", + BtTrackers: "udp://tracker.example:1337/announce", + }, + } + d, err := New(cfg) + if err != nil { + t.Fatal(err) + } + a := d.(*Aria2) + if a.Name() != "aria2" { + t.Fatalf("unexpected name %q", a.Name()) + } + if got := a.Capabilities().SourceTypes; len(got) != 3 || got[0] != "magnet" { + t.Fatalf("unexpected capabilities %#v", got) + } + if !a.Managed || !a.RetainSeed || a.Secret != "secret" || a.MaxConcurrentDownloads != 12 || a.SeedRatio != 1.5 { + t.Fatalf("unexpected constructed downloader: %#v", a) + } + a.Managed = false + if err := a.Start(context.Background()); err != nil { + t.Fatal(err) + } + if err := a.Stop(context.Background()); err != nil { + t.Fatal(err) + } + if !configured(downloader.Config{Aria2: downloader.Aria2Config{Configured: true}}) { + t.Fatal("expected configured aria2 config") + } +} + +func TestAria2StartErrors(t *testing.T) { + engine := &Aria2{Managed: true, URL: "ws://example.com:6800/jsonrpc"} + if err := engine.Start(context.Background()); err == nil { + _ = engine.Stop(context.Background()) + t.Fatal("expected remote managed URL error") + } + if _, err := (Aria2{StateDir: string([]byte{0})}).startArgs("6800"); err == nil { + t.Fatal("expected invalid state dir error") + } +} + +func TestAria2SeedTimeMinutes(t *testing.T) { + if got := aria2SeedTimeMinutes(time.Hour); got != 60 { + t.Fatalf("expected 1h to map to 60 minutes, got %d", got) + } + if got := aria2SeedTimeMinutes(0); got != aria2SeedForeverMinutes { + t.Fatalf("expected zero duration to seed indefinitely, got %d", got) + } + if got := aria2SeedTimeMinutes(30 * time.Second); got != 1 { + t.Fatalf("expected sub-minute duration to round up to 1, got %d", got) + } +} + +func TestShouldAttachExistingAria2Task(t *testing.T) { + for _, state := range []string{"downloading", "uploading", "interrupted"} { + if !shouldAttachExistingAria2Task(downloader.DownloadTask{Status: downloader.Status{State: state}}) { + t.Fatalf("expected to attach for state %q", state) + } + } + for _, state := range []string{"queued", "assigned", "paused", "completed", "canceling"} { + if shouldAttachExistingAria2Task(downloader.DownloadTask{Status: downloader.Status{State: state}}) { + t.Fatalf("did not expect to attach for state %q", state) + } + } +} + +func TestAria2StatusKeysCoverReportedFields(t *testing.T) { + required := []string{ + "bittorrent", "status", "totalLength", "completedLength", + "downloadSpeed", "uploadLength", "uploadSpeed", "connections", "numSeeders", + } + have := map[string]bool{} + for _, key := range aria2StatusKeys { + have[key] = true + } + for _, key := range required { + if !have[key] { + t.Fatalf("aria2StatusKeys missing %q", key) + } + } + if have["bitTorrent"] { + t.Fatal("aria2 status key is case-sensitive; use bittorrent") + } +} + +func TestAria2ResetOperations(t *testing.T) { + tests := []struct { + name string + status arigo.DownloadStatus + wantRemoveActive bool + wantRemoveResult bool + }{ + {name: "active", status: arigo.StatusActive, wantRemoveActive: true, wantRemoveResult: true}, + {name: "waiting", status: arigo.StatusWaiting, wantRemoveActive: true, wantRemoveResult: true}, + {name: "paused", status: arigo.StatusPaused, wantRemoveActive: true, wantRemoveResult: true}, + {name: "completed", status: arigo.StatusCompleted, wantRemoveActive: false, wantRemoveResult: true}, + {name: "error", status: arigo.StatusError, wantRemoveActive: false, wantRemoveResult: true}, + {name: "removed", status: arigo.StatusRemoved, wantRemoveActive: false, wantRemoveResult: true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + removeActive, removeResult := aria2ResetOperations(arigo.Status{Status: tt.status}) + if removeActive != tt.wantRemoveActive || removeResult != tt.wantRemoveResult { + t.Fatalf("expected removeActive=%v removeResult=%v, got %v %v", tt.wantRemoveActive, tt.wantRemoveResult, removeActive, removeResult) + } + }) + } +} + +func TestAria2ErrorClassification(t *testing.T) { + if !isAria2DownloadNotFound(errors.New("Active Download not found for GID#b384ccaa7eae88da")) { + t.Fatal("expected aria2 active download not found") + } + if !isAria2GIDNotFound(errors.New("GID 8bddd19e07ad6dc3 is not found")) { + t.Fatal("expected tellStatus GID-not-found") + } + for _, err := range []error{rpc2.ErrShutdown, io.ErrClosedPipe, errors.New("connection is shut down")} { + if !isAria2RPCDisconnected(err) { + t.Fatalf("expected %v to be treated as aria2 rpc disconnect", err) + } + } + if isAria2RPCDisconnected(nil) { + t.Fatal("nil error must not be treated as disconnected") + } + if !isAria2InfoHashAlreadyRegistered(errors.New("InfoHash 0546769f209ec059284b47f68659791a6f75ca8e is already registered.")) { + t.Fatal("expected aria2 infohash conflict to be attachable") + } +} + +func TestAria2TaskInfoHash(t *testing.T) { + const infoHash = "0546769f209ec059284b47f68659791a6f75ca8e" + if got := aria2TaskInfoHash(downloadTask("task-1", "magnet", "magnet:?xt=urn:btih:"+infoHash+"&dn=fixture")); got != infoHash { + t.Fatalf("expected magnet infohash %s, got %s", infoHash, got) + } + if got := aria2TaskInfoHash(downloadTask("task-1", "magnet", "magnet:?xt=urn:btih:ARLHNHRAT3AFSKELI7TIMGMRDJXXLSP2")); len(got) != 40 { + t.Fatalf("expected decoded base32 infohash, got %s", got) + } + taskWithRuntime := downloadTask("task-1", "magnet", "magnet:?xt=urn:btih:abc") + taskWithRuntime.Status.Runtime = &downloader.TaskRuntime{Torrent: &downloader.TorrentRuntime{InfoHash: strings.ToUpper(infoHash)}} + if got := aria2TaskInfoHash(taskWithRuntime); got != infoHash { + t.Fatalf("expected detail infohash %s, got %s", infoHash, got) + } + for _, task := range []downloader.DownloadTask{ + downloadTask("task-1", "http", "https://example.com/file.bin"), + downloadTask("task-1", "magnet", "%zz"), + downloadTask("task-1", "magnet", "magnet:?xt=urn:notbtih:abc"), + downloadTask("task-1", "magnet", "magnet:?xt=urn:btih:not-hex"), + } { + if got := aria2TaskInfoHash(task); got != "" { + t.Fatalf("expected empty infohash for %#v, got %q", task, got) + } + } + if got := aria2TaskGID("task-1"); got != "7afaa346b4bf92bf" { + t.Fatalf("unexpected deterministic gid: %s", got) + } +} + +func TestAria2TaskState(t *testing.T) { + if got := aria2TaskState(arigo.Status{ + Status: arigo.StatusActive, + TotalLength: 100, + CompletedLength: 100, + Files: []arigo.File{{Path: "/tmp/zpan/task-1/file.bin", Length: 100, CompletedLength: 100}}, + }); got != downloader.TaskStateCompleted { + t.Fatalf("expected active full torrent to be completed, got %s", got) + } + if got := aria2TaskState(arigo.Status{Status: arigo.StatusActive, TotalLength: 100, CompletedLength: 10}); got != downloader.TaskStateDownloading { + t.Fatalf("expected active partial torrent to be downloading, got %s", got) + } + if got := aria2TaskState(arigo.Status{Status: arigo.StatusError}); got != downloader.TaskStateFailed { + t.Fatalf("expected error torrent to be failed, got %s", got) + } +} + +func TestAria2SeedAndStatusHelpers(t *testing.T) { + taskDir := filepath.Join(t.TempDir(), "task-1") + statuses := []arigo.Status{ + {GID: "not-seed", TotalLength: 100, CompletedLength: 50, Dir: taskDir, Files: []arigo.File{{Path: filepath.Join(taskDir, "partial.bin")}}}, + {GID: "seed-by-hash", InfoHash: "ABCDEF", TotalLength: 100, CompletedLength: 100, Dir: taskDir, Files: []arigo.File{{Path: filepath.Join(taskDir, "done.bin")}}}, + {GID: "seed-by-dir", TotalLength: 100, CompletedLength: 100, Dir: taskDir, Files: []arigo.File{{Path: filepath.Join(taskDir, "done.bin")}}}, + } + if got, ok := selectAria2SeedStatus(statuses, downloader.SeedRef{InfoHash: "abcdef"}, ""); !ok || got.GID != "seed-by-hash" { + t.Fatalf("expected seed by hash, got %#v ok=%v", got, ok) + } + if got, ok := selectAria2SeedStatus(statuses, downloader.SeedRef{}, taskDir); !ok || got.GID != "seed-by-hash" { + t.Fatalf("expected seed by task dir, got %#v ok=%v", got, ok) + } + if _, ok := selectAria2SeedStatus(statuses, downloader.SeedRef{InfoHash: "missing"}, filepath.Join(taskDir, "missing")); ok { + t.Fatal("did not expect missing seed match") + } + if got := aria2SeedTaskDir("/downloads", downloader.SeedRef{TaskID: "task-2"}); got != filepath.Clean("/downloads/task-2") { + t.Fatalf("unexpected fallback seed dir: %s", got) + } + if got := aria2SeedTaskDir("/downloads", downloader.SeedRef{TaskID: "task-2", Path: taskDir}); got != filepath.Clean(taskDir) { + t.Fatalf("unexpected explicit seed dir: %s", got) + } + if got := aria2StatusTaskDir(arigo.Status{}, taskDir); got != taskDir { + t.Fatalf("unexpected fallback status dir: %s", got) + } + if got := aria2StatusTaskDir(arigo.Status{Dir: taskDir}, "fallback"); got != filepath.Clean(taskDir) { + t.Fatalf("unexpected status dir: %s", got) + } + seed := (Aria2{}).seedFromStatus(arigo.Status{GID: "gid-1", InfoHash: "ABCDEF"}, taskDir) + if seed.Engine != "aria2" || seed.ID != "gid-1" || seed.InfoHash != "abcdef" || seed.Path != taskDir || seed.Snapshot == nil || seed.Cleanup == nil { + t.Fatalf("unexpected seed: %#v", seed) + } +} + +func TestAria2StatusMatchingHelpers(t *testing.T) { + taskDir := filepath.Join(t.TempDir(), "task-1") + if !aria2StatusMatchesTask(arigo.Status{GID: "gid-1"}, taskDir, "gid-1", "") { + t.Fatal("expected gid match") + } + if !aria2StatusMatchesTask(arigo.Status{Following: "gid-1"}, taskDir, "gid-1", "") { + t.Fatal("expected following match") + } + if !aria2StatusMatchesTask(arigo.Status{BelongsTo: "gid-1"}, taskDir, "gid-1", "") { + t.Fatal("expected belongsTo match") + } + if !aria2StatusMatchesTask(arigo.Status{InfoHash: "ABCDEF"}, taskDir, "gid-1", "abcdef") { + t.Fatal("expected infohash match") + } + if !aria2StatusMatchesTask(arigo.Status{Dir: taskDir}, taskDir, "gid-1", "") { + t.Fatal("expected directory match") + } + if !aria2StatusMatchesTask(arigo.Status{Files: []arigo.File{{Path: filepath.Join(taskDir, "file.bin")}}}, taskDir, "gid-1", "") { + t.Fatal("expected file path match") + } + if aria2StatusMatchesTask(arigo.Status{Files: []arigo.File{{Path: filepath.Join(filepath.Dir(taskDir), "other", "file.bin")}}}, taskDir, "gid-1", "") { + t.Fatal("did not expect unrelated file path match") + } + if !aria2StatusBelongsToTask(arigo.Status{Dir: taskDir}, taskDir, "gid-1") { + t.Fatal("expected belongs-to directory match") + } + if aria2StatusBelongsToTask(arigo.Status{Dir: filepath.Dir(taskDir)}, taskDir, "gid-1") { + t.Fatal("did not expect parent directory match") + } +} + +func TestAria2RuntimeConversionHelpers(t *testing.T) { + if got := aria2Phase(string(arigo.StatusWaiting), []string{"child"}); got != "metadata" { + t.Fatalf("expected metadata phase, got %s", got) + } + if got := aria2Phase(string(arigo.StatusWaiting), nil); got != "downloading" { + t.Fatalf("expected waiting download phase, got %s", got) + } + if got := aria2Phase(string(arigo.StatusCompleted), nil); got != "completed" { + t.Fatalf("expected completed phase, got %s", got) + } + if got := aria2Phase(string(arigo.StatusRemoved), nil); got != "error" { + t.Fatalf("expected removed phase error, got %s", got) + } + if aria2PeerProgress(arigo.Peer{}) != nil { + t.Fatal("expected nil progress without bitfield") + } + if aria2PeerProgress(arigo.Peer{BitField: "xyz"}) != nil { + t.Fatal("expected nil progress for invalid bitfield") + } + if progress := aria2PeerProgress(arigo.Peer{Seeder: true}); progress == nil || *progress != 1 { + t.Fatalf("expected seeder full progress, got %#v", progress) + } + if progress := aria2PeerProgress(arigo.Peer{BitField: "f0"}); progress == nil || *progress != 0.5 { + t.Fatalf("expected half progress, got %#v", progress) + } + + trackers := make([][]string, 1) + for i := 0; i < 25; i++ { + trackers[0] = append(trackers[0], fmt.Sprintf("udp://tracker-%02d/announce", i)) + } + trackers[0] = append(trackers[0], "udp://tracker-00/announce", "") + if got := aria2Trackers(trackers); len(got) != 20 || got[0].URL != "udp://tracker-00/announce" { + t.Fatalf("unexpected trackers: %#v", got) + } + + peers := aria2Peers([]arigo.Peer{ + {IP: "", Port: 6881}, + {IP: "203.0.113.10", Port: 6881, DownloadSpeed: 10, UploadSpeed: 2, BitField: "f"}, + }, nil) + if len(peers) != 1 || peers[0].Address != "203.0.113.10:6881" || *peers[0].DownloadBps != 10 || *peers[0].UploadBps != 2 { + t.Fatalf("unexpected peers: %#v", peers) + } +} + +func TestAria2FilesDetailAndResultHelpers(t *testing.T) { + taskDir := t.TempDir() + contentPath := filepath.Join(taskDir, "Torrent", "movie.mkv") + if err := os.MkdirAll(filepath.Dir(contentPath), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(contentPath, []byte("movie"), 0o600); err != nil { + t.Fatal(err) + } + files := []arigo.File{ + {Path: "[METADATA]fixture", Length: 1, CompletedLength: 1, Selected: true}, + {Path: contentPath, Length: 5, CompletedLength: 5, Selected: true}, + } + converted := aria2Files(taskDir, "Torrent", files) + if len(converted) != 1 || converted[0].Path != "movie.mkv" || converted[0].Size != 5 || *converted[0].CompletedBytes != 5 || !*converted[0].Selected { + t.Fatalf("unexpected converted files: %#v", converted) + } + result, err := resultFromAria2Files(downloadTask("task-1", "magnet", "magnet:?xt=urn:btih:abc"), taskDir, "Torrent", files) + if err != nil { + t.Fatal(err) + } + if result.Path != filepath.Join(taskDir, "Torrent") || result.Name != "Torrent" || !result.IsDir { + t.Fatalf("unexpected result: %#v", result) + } + status := arigo.Status{ + GID: "gid-1", + Status: arigo.StatusActive, + TotalLength: 10, + CompletedLength: 5, + DownloadSpeed: 5, + UploadLength: 2, + UploadSpeed: 1, + Connections: 3, + NumSeeders: 4, + Dir: taskDir, + InfoHash: "abcdef", + Files: files, + BitTorrent: arigo.BitTorrentStatus{ + Info: arigo.BitTorrentStatusInfo{Name: "Torrent"}, + AnnounceList: [][]string{{"udp://tracker/announce"}}, + }, + } + detail := aria2Detail(status, []arigo.Peer{{IP: "203.0.113.10", Port: 6881, Seeder: false}}, nil) + if detail.Engine != "aria2" || detail.Phase != "downloading" || detail.Torrent == nil || detail.Torrent.InfoHash != "abcdef" || detail.ETASeconds == nil || *detail.ETASeconds != 1 { + t.Fatalf("unexpected detail: %#v", detail) + } +} + +func TestSelectAria2SeedStatusPrefersCompletedPayloadOverMetadata(t *testing.T) { + infoHash := "f8f8044d5dfeef2719dcda6ce42dba1c4eb9ea22" + taskDir := filepath.Join(t.TempDir(), "task-1") + metadata := arigo.Status{ + GID: "metadata-gid", + Status: arigo.StatusCompleted, + InfoHash: infoHash, + Dir: taskDir, + TotalLength: 17596, + CompletedLength: 17596, + Files: []arigo.File{{Path: "[METADATA]", Length: 17596, CompletedLength: 17596, Selected: true}}, + } + payload := arigo.Status{ + GID: "payload-gid", + Status: arigo.StatusActive, + InfoHash: infoHash, + Dir: taskDir, + TotalLength: 100, + CompletedLength: 100, + Files: []arigo.File{{Path: filepath.Join(taskDir, "album", "track.flac"), Length: 100, CompletedLength: 100, Selected: true}}, + } + + got, ok := selectAria2SeedStatus([]arigo.Status{metadata, payload}, downloader.SeedRef{InfoHash: infoHash}, taskDir) + if !ok || got.GID != "payload-gid" { + t.Fatalf("expected payload seed, ok=%v got=%s", ok, got.GID) + } +} + +func TestAria2FilesReportsRelativeTorrentPaths(t *testing.T) { + taskDir := filepath.Join(t.TempDir(), "task-1") + files := aria2Files(taskDir, "album", []arigo.File{ + {Path: filepath.Join(taskDir, "album", "disc-1", "track.flac"), Length: 100, CompletedLength: 50, Selected: true}, + {Path: filepath.Join(t.TempDir(), "outside.flac"), Length: 10, CompletedLength: 10, Selected: true}, + {Path: "[METADATA]info", Length: 1}, + }) + + if len(files) != 2 { + t.Fatalf("expected two visible files, got %#v", files) + } + if files[0].Path != "disc-1/track.flac" || files[1].Path != "outside.flac" { + t.Fatalf("unexpected relative paths: %#v", files) + } +} + +func TestAria2DetailIncludesPeersAndOmitsEmptyETA(t *testing.T) { + detail := aria2Detail( + arigo.Status{ + TotalLength: 1000, + CompletedLength: 250, + DownloadSpeed: 200, + Connections: 44, + NumSeeders: 4, + BitTorrent: arigo.BitTorrentStatus{ + AnnounceList: [][]string{{"udp://tracker.example:1337/announce"}}, + Info: arigo.BitTorrentStatusInfo{Name: "fixture.torrent"}, + }, + }, + []arigo.Peer{ + {IP: "192.0.2.10", Port: 6881, DownloadSpeed: 1024, UploadSpeed: 256, Seeder: true}, + {IP: "192.0.2.11", Port: 6882, DownloadSpeed: 2048, UploadSpeed: 512, Seeder: false}, + }, + nil, + ) + + if detail.Torrent == nil || detail.Torrent.Peers == nil || *detail.Torrent.Peers != 2 { + t.Fatalf("expected peer count 2, got %#v", detail.Torrent) + } + if len(detail.Peers) != 2 || detail.Peers[0].Address != "192.0.2.10:6881" { + t.Fatalf("expected peer samples, got %#v", detail.Peers) + } + if detail.ETASeconds == nil || *detail.ETASeconds != 4 { + t.Fatalf("expected ETA seconds 4, got %#v", detail.ETASeconds) + } + + noSpeed := aria2Detail(arigo.Status{TotalLength: 1000, CompletedLength: 250}, nil, nil) + if noSpeed.ETASeconds != nil { + t.Fatalf("expected empty ETA without download speed, got %#v", noSpeed.ETASeconds) + } +} + +func TestResultFromAria2Files(t *testing.T) { + dir := t.TempDir() + taskDir := filepath.Join(dir, "task-1") + if err := os.MkdirAll(filepath.Join(taskDir, "payload"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(taskDir, "payload", "a.txt"), []byte("a"), 0o644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(taskDir, "payload", "b.txt"), []byte("b"), 0o644); err != nil { + t.Fatal(err) + } + + result, err := resultFromAria2Files( + downloader.DownloadTask{ID: "task-1"}, + taskDir, + "payload", + []arigo.File{ + {Path: filepath.Join(taskDir, "payload", "a.txt"), Length: 1, Selected: true}, + {Path: filepath.Join(taskDir, "payload", "b.txt"), Length: 1, Selected: true}, + }, + ) + if err != nil { + t.Fatal(err) + } + if result.Path != filepath.Join(taskDir, "payload") || result.Name != "payload" || result.Size != 2 { + t.Fatalf("unexpected aria2 result: %#v", result) + } +} + +func TestResultFromAria2FilesWrapsSingleFileBTTask(t *testing.T) { + taskDir := filepath.Join(t.TempDir(), "task-1") + if err := os.MkdirAll(taskDir, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(taskDir, "movie.mkv"), []byte("movie"), 0o644); err != nil { + t.Fatal(err) + } + + result, err := resultFromAria2Files( + downloadTaskWithName("task-1", "magnet", "magnet:?xt=urn:btih:abc", "movie.torrent"), + taskDir, + "Iron.Lung.2026.1080p.WEBRip.10Bit.DDP.5.1.x265-NeoNoir", + []arigo.File{{Path: filepath.Join(taskDir, "movie.mkv"), Length: 5, Selected: true}}, + ) + if err != nil { + t.Fatal(err) + } + if !result.IsDir || result.Path != taskDir || result.Size != 5 { + t.Fatalf("unexpected single-file torrent result: %#v", result) + } + if result.Name != "Iron.Lung.2026.1080p.WEBRip.10Bit.DDP.5.1.x265-NeoNoir" { + t.Fatalf("expected torrent name wrapper dir, got %s", result.Name) + } +} diff --git a/cmd/pkg/downloaders/core/layout.go b/cmd/pkg/downloaders/core/layout.go new file mode 100644 index 00000000..4b9b9bb0 --- /dev/null +++ b/cmd/pkg/downloaders/core/layout.go @@ -0,0 +1,234 @@ +package core + +import ( + "net/url" + "os" + "path/filepath" + "strings" + + "github.com/saltbo/zpan/internal/downloader" + "github.com/saltbo/zpan/pkg/system" +) + +type DownloadedFile struct { + Path string + RelativePath string +} + +func ResultFromPath(task downloader.DownloadTask, path string, fallbackName string) (downloader.Result, error) { + info, err := os.Stat(path) + if err != nil { + candidate := filepath.Join(path, fallbackName) + if fallbackName != "" { + if _, statErr := os.Stat(candidate); statErr == nil { + return ResultFromPath(task, candidate, fallbackName) + } + } + return downloader.Result{}, err + } + if !info.IsDir() { + return resultFromFile(task, path) + } + entries, err := os.ReadDir(path) + if err != nil { + return downloader.Result{}, err + } + visible := make([]os.DirEntry, 0, len(entries)) + for _, entry := range entries { + if strings.HasPrefix(entry.Name(), ".") { + continue + } + if !entry.IsDir() && system.IsDownloadSidecarPath(entry.Name()) { + continue + } + visible = append(visible, entry) + } + if len(visible) == 1 && !visible[0].IsDir() { + return resultFromFile(task, filepath.Join(path, visible[0].Name())) + } + if len(visible) == 1 && visible[0].IsDir() { + return ResultFromPath(task, filepath.Join(path, visible[0].Name()), visible[0].Name()) + } + size, err := system.DirectorySize(path) + if err != nil { + return downloader.Result{}, err + } + return downloader.Result{Path: path, Name: outputName(task, fallbackName), Size: size, IsDir: true}, nil +} + +func ResultFromDownloadedFiles(task downloader.DownloadTask, taskDir string, fallbackName string, files []DownloadedFile) (downloader.Result, error) { + if len(files) == 1 && !hasPathSeparator(files[0].RelativePath) { + if task.SourceType() != "http" { + size, err := system.DirectorySize(taskDir) + if err != nil { + return downloader.Result{}, err + } + return downloader.Result{Path: taskDir, Name: singleFileTorrentFolderName(task, files[0].RelativePath, fallbackName), Size: size, IsDir: true}, nil + } + return resultFromFile(task, files[0].Path) + } + root, ok := singleTopLevelDirectory(files) + if ok { + path := filepath.Join(taskDir, root) + size, err := system.DirectorySize(path) + if err != nil { + return downloader.Result{}, err + } + return downloader.Result{Path: path, Name: outputName(task, root), Size: size, IsDir: true}, nil + } + size, err := system.DirectorySize(taskDir) + if err != nil { + return downloader.Result{}, err + } + return downloader.Result{Path: taskDir, Name: outputName(task, fallbackName), Size: size, IsDir: true}, nil +} + +func singleFileTorrentFolderName(task downloader.DownloadTask, filePath string, fallbackName string) string { + if name := requestedOutputName(task); name != "" { + return name + } + if name := payloadFallbackName(fallbackName); name != "" { + return name + } + base := filepath.Base(filePath) + if base != "" && base != "." && base != string(filepath.Separator) { + ext := filepath.Ext(base) + if ext != "" { + base = strings.TrimSuffix(base, ext) + } + if base != "" { + return base + } + } + return outputName(task, fallbackName) +} + +func payloadFallbackName(fallbackName string) string { + name := strings.TrimSpace(fallbackName) + if name == "" || system.IsDownloadSidecarPath(name) { + return "" + } + name = filepath.Base(name) + if name == "." || name == string(filepath.Separator) { + return "" + } + return name +} + +func singleTopLevelDirectory(files []DownloadedFile) (string, bool) { + var root string + for _, file := range files { + segments := splitRelativePath(file.RelativePath) + if len(segments) < 2 { + return "", false + } + if root == "" { + root = segments[0] + continue + } + if segments[0] != root { + return "", false + } + } + return root, root != "" +} + +func splitRelativePath(path string) []string { + normalized := filepath.ToSlash(filepath.Clean(path)) + if normalized == "." || normalized == "/" { + return nil + } + parts := strings.Split(strings.Trim(normalized, "/"), "/") + out := parts[:0] + for _, part := range parts { + if part != "" && part != "." { + out = append(out, part) + } + } + return out +} + +func StripTorrentRoot(path string, torrentName string) string { + parts := splitRelativePath(path) + if len(parts) < 2 { + return filepath.ToSlash(filepath.Clean(path)) + } + if torrentName == "" || parts[0] != torrentName { + return filepath.ToSlash(filepath.Clean(path)) + } + return strings.Join(parts[1:], "/") +} + +func hasPathSeparator(path string) bool { + return len(splitRelativePath(path)) > 1 +} + +func OutputName(task downloader.DownloadTask, fallback string) string { + return outputName(task, fallback) +} + +func RequestedOutputName(task downloader.DownloadTask) string { + return requestedOutputName(task) +} + +func FilenameFromURL(parsed *url.URL) string { + return filenameFromURL(parsed) +} + +func resultFromFile(task downloader.DownloadTask, path string) (downloader.Result, error) { + info, err := os.Stat(path) + if err != nil { + return downloader.Result{}, err + } + return downloader.Result{Path: path, Name: outputName(task, filepath.Base(path)), Size: info.Size()}, nil +} + +func DownloadedPath(baseDir string, path string) (string, string) { + if filepath.IsAbs(path) { + abs := filepath.Clean(path) + rel, err := filepath.Rel(baseDir, abs) + if err != nil || isUnsafeRelativePath(rel) { + return abs, filepath.Base(abs) + } + return abs, rel + } + rel := filepath.Clean(path) + if isUnsafeRelativePath(rel) { + return filepath.Join(baseDir, filepath.Base(rel)), filepath.Base(rel) + } + return filepath.Join(baseDir, rel), rel +} + +func isUnsafeRelativePath(path string) bool { + return path == ".." || strings.HasPrefix(path, ".."+string(filepath.Separator)) || filepath.IsAbs(path) +} + +func outputName(task downloader.DownloadTask, fallback string) string { + name := requestedOutputName(task) + if name == "" { + name = strings.TrimSpace(fallback) + } + if name == "" || name == "." || name == string(filepath.Separator) { + name = task.ID + } + return filepath.Base(name) +} + +func requestedOutputName(task downloader.DownloadTask) string { + name := strings.TrimSpace(task.Name()) + if name == "" { + return "" + } + if task.SourceType() != "http" && system.IsDownloadSidecarPath(name) { + return "" + } + return filepath.Base(name) +} + +func filenameFromURL(parsed *url.URL) string { + name := filepath.Base(parsed.Path) + if name == "." || name == "/" { + return "" + } + return name +} diff --git a/cmd/pkg/downloaders/core/layout_test.go b/cmd/pkg/downloaders/core/layout_test.go new file mode 100644 index 00000000..8130ee85 --- /dev/null +++ b/cmd/pkg/downloaders/core/layout_test.go @@ -0,0 +1,256 @@ +package core + +import ( + "net/url" + "os" + "path/filepath" + "testing" + + "github.com/saltbo/zpan/internal/downloader" +) + +func downloadTask(id, sourceType, sourceURI string) downloader.DownloadTask { + return downloader.DownloadTask{ + ID: id, + Source: downloader.Source{Type: sourceType, URI: sourceURI}, + } +} + +func downloadTaskWithName(id, sourceType, sourceURI, name string) downloader.DownloadTask { + task := downloadTask(id, sourceType, sourceURI) + task.Destination.Name = name + return task +} + +func TestStripTorrentRoot(t *testing.T) { + cases := []struct { + name string + path string + torrentName string + want string + }{ + {name: "nested torrent root", path: "Album/Disc 1/track.flac", torrentName: "Album", want: "Disc 1/track.flac"}, + {name: "single file named like torrent", path: "Album", torrentName: "Album", want: "Album"}, + {name: "different root", path: "Other/track.flac", torrentName: "Album", want: "Other/track.flac"}, + {name: "empty torrent name", path: "Album/track.flac", torrentName: "", want: "Album/track.flac"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if got := StripTorrentRoot(tc.path, tc.torrentName); got != tc.want { + t.Fatalf("expected %q, got %q", tc.want, got) + } + }) + } +} + +func TestResultFromPathReturnsDirectory(t *testing.T) { + dir := t.TempDir() + taskDir := filepath.Join(dir, "task-1") + if err := os.MkdirAll(filepath.Join(taskDir, "folder"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(taskDir, "folder", "a.txt"), []byte("a"), 0o644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(taskDir, "folder", "b.txt"), []byte("b"), 0o644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(taskDir, "fixture.torrent"), []byte("torrent"), 0o644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(taskDir, "folder.aria2"), []byte("control"), 0o644); err != nil { + t.Fatal(err) + } + + result, err := ResultFromPath(downloadTaskWithName("task-1", "http", "https://example.com/bundle", "bundle"), taskDir, "bundle") + if err != nil { + t.Fatal(err) + } + if !result.IsDir { + t.Fatal("expected directory result") + } + if result.Name != "bundle" { + t.Fatalf("expected bundle, got %s", result.Name) + } + if result.Size != 2 { + t.Fatalf("expected directory size 2, got %d", result.Size) + } + if result.Path != filepath.Join(taskDir, "folder") { + t.Fatalf("expected content dir path, got %s", result.Path) + } +} + +func TestResultFromDownloadedFilesWrapsMultipleTopLevelEntries(t *testing.T) { + dir := t.TempDir() + taskDir := filepath.Join(dir, "task-1") + if err := os.MkdirAll(filepath.Join(taskDir, "folder"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(taskDir, "folder", "a.txt"), []byte("a"), 0o644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(taskDir, "root.txt"), []byte("b"), 0o644); err != nil { + t.Fatal(err) + } + + result, err := ResultFromDownloadedFiles(downloadTaskWithName("task-1", "http", "https://example.com/bundle", "bundle"), taskDir, "fallback", []DownloadedFile{ + {Path: filepath.Join(taskDir, "folder", "a.txt"), RelativePath: filepath.Join("folder", "a.txt")}, + {Path: filepath.Join(taskDir, "root.txt"), RelativePath: "root.txt"}, + }) + if err != nil { + t.Fatal(err) + } + if result.Path != taskDir { + t.Fatalf("expected task dir wrapper path, got %s", result.Path) + } + if result.Name != "bundle" { + t.Fatalf("expected bundle wrapper name, got %s", result.Name) + } +} + +func TestOutputNameIgnoresTorrentTaskNameForBT(t *testing.T) { + name := OutputName( + downloadTaskWithName("task-1", "magnet", "magnet:?xt=urn:btih:abc", "movie.torrent"), + "movie.mkv", + ) + + if name != "movie.mkv" { + t.Fatalf("expected payload fallback name, got %s", name) + } +} + +func TestOutputNameAllowsHTTPDownloadName(t *testing.T) { + name := OutputName( + downloadTaskWithName("task-1", "http", "https://example.com/movie.torrent", "movie.torrent"), + "download", + ) + + if name != "movie.torrent" { + t.Fatalf("expected HTTP task name to be preserved, got %s", name) + } +} + +func TestResultFromPathSingleFileAndFallbackCandidate(t *testing.T) { + dir := t.TempDir() + file := filepath.Join(dir, "payload.bin") + if err := os.WriteFile(file, []byte("payload"), 0o644); err != nil { + t.Fatal(err) + } + result, err := ResultFromPath(downloadTaskWithName("task-1", "http", "", ""), file, "") + if err != nil { + t.Fatal(err) + } + if result.Path != file || result.Name != "payload.bin" || result.Size != int64(len("payload")) || result.IsDir { + t.Fatalf("unexpected single file result: %#v", result) + } + + if _, err := ResultFromPath(downloadTask("task-1", "http", ""), filepath.Join(dir, "missing"), "absent.bin"); err == nil { + t.Fatal("expected missing path error") + } +} + +func TestResultFromDownloadedFilesSingleFileVariants(t *testing.T) { + dir := t.TempDir() + file := filepath.Join(dir, "movie.mkv") + if err := os.WriteFile(file, []byte("movie"), 0o644); err != nil { + t.Fatal(err) + } + httpResult, err := ResultFromDownloadedFiles(downloadTask("task-1", "http", ""), dir, "", []DownloadedFile{{Path: file, RelativePath: "movie.mkv"}}) + if err != nil { + t.Fatal(err) + } + if httpResult.Path != file || httpResult.IsDir { + t.Fatalf("expected HTTP single file result, got %#v", httpResult) + } + btResult, err := ResultFromDownloadedFiles(downloadTask("task-1", "magnet", ""), dir, "Movie", []DownloadedFile{{Path: file, RelativePath: "movie.mkv"}}) + if err != nil { + t.Fatal(err) + } + if btResult.Path != dir || !btResult.IsDir || btResult.Name != "Movie" { + t.Fatalf("expected BT folder wrapper, got %#v", btResult) + } +} + +func TestResultFromDownloadedFilesSingleTopLevelDirectory(t *testing.T) { + dir := t.TempDir() + album := filepath.Join(dir, "Album") + if err := os.MkdirAll(album, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(album, "track.flac"), []byte("track"), 0o644); err != nil { + t.Fatal(err) + } + result, err := ResultFromDownloadedFiles(downloadTask("task-1", "magnet", ""), dir, "", []DownloadedFile{{Path: filepath.Join(album, "track.flac"), RelativePath: filepath.Join("Album", "track.flac")}}) + if err != nil { + t.Fatal(err) + } + if result.Path != album || result.Name != "Album" || !result.IsDir { + t.Fatalf("expected album directory result, got %#v", result) + } +} + +func TestDownloadedPathAndURLHelpers(t *testing.T) { + base := t.TempDir() + absInside := filepath.Join(base, "dir", "file.txt") + path, rel := DownloadedPath(base, absInside) + if path != absInside || rel != filepath.Join("dir", "file.txt") { + t.Fatalf("unexpected inside path: %q %q", path, rel) + } + absOutside := filepath.Join(filepath.Dir(base), "outside.txt") + path, rel = DownloadedPath(base, absOutside) + if path != absOutside || rel != filepath.Base(absOutside) { + t.Fatalf("unexpected outside path: %q %q", path, rel) + } + path, rel = DownloadedPath(base, "../escape.txt") + if path != filepath.Join(base, "escape.txt") || rel != "escape.txt" { + t.Fatalf("unexpected unsafe path handling: %q %q", path, rel) + } + parsed, err := url.Parse("https://example.com/files/movie.mkv?token=1") + if err != nil { + t.Fatal(err) + } + if FilenameFromURL(parsed) != "movie.mkv" { + t.Fatalf("unexpected URL filename: %q", FilenameFromURL(parsed)) + } + parsed, _ = url.Parse("https://example.com/") + if FilenameFromURL(parsed) != "" { + t.Fatalf("expected empty URL filename, got %q", FilenameFromURL(parsed)) + } +} + +func TestRequestedOutputNameAndFallbacks(t *testing.T) { + if RequestedOutputName(downloadTaskWithName("task-1", "http", "", " dir/file.bin ")) != "file.bin" { + t.Fatal("expected requested HTTP output basename") + } + if RequestedOutputName(downloadTaskWithName("task-1", "magnet", "", "payload.torrent")) != "" { + t.Fatal("expected BT sidecar output name to be ignored") + } + if OutputName(downloadTask("task-1", "http", ""), "") != "task-1" { + t.Fatal("expected task id output fallback") + } +} + +func TestLayoutPrivateFallbackBranches(t *testing.T) { + task := downloadTask("task-1", "magnet", "") + if got := singleFileTorrentFolderName(task, "movie.mkv", ""); got != "movie" { + t.Fatalf("expected basename without extension, got %q", got) + } + if got := singleFileTorrentFolderName(task, ".", ""); got != "task-1" { + t.Fatalf("expected task id fallback, got %q", got) + } + if got := payloadFallbackName("payload.torrent"); got != "" { + t.Fatalf("expected sidecar fallback to be ignored, got %q", got) + } + if got := payloadFallbackName("/"); got != "" { + t.Fatalf("expected root fallback to be ignored, got %q", got) + } + if root, ok := singleTopLevelDirectory([]DownloadedFile{{RelativePath: "Album/one.flac"}, {RelativePath: "Other/two.flac"}}); ok || root != "" { + t.Fatalf("expected mixed roots to be rejected, got %q %v", root, ok) + } + if parts := splitRelativePath("."); parts != nil { + t.Fatalf("expected empty path parts, got %#v", parts) + } + if _, err := resultFromFile(task, filepath.Join(t.TempDir(), "missing.bin")); err == nil { + t.Fatal("expected missing file error") + } +} diff --git a/cmd/pkg/downloaders/core/peer.go b/cmd/pkg/downloaders/core/peer.go new file mode 100644 index 00000000..6df72e24 --- /dev/null +++ b/cmd/pkg/downloaders/core/peer.go @@ -0,0 +1,13 @@ +package core + +import ( + "github.com/saltbo/zpan/internal/downloader" + "github.com/saltbo/zpan/pkg/geoip" +) + +func ApplyPeerRegion(peer *downloader.Peer, ip string, fallbackCountryCode string, resolver geoip.Resolver) { + if peer == nil { + return + } + peer.CountryCode, peer.RegionCode = geoip.NormalizeRegion(ip, fallbackCountryCode, resolver) +} diff --git a/cmd/pkg/downloaders/core/trackers.go b/cmd/pkg/downloaders/core/trackers.go new file mode 100644 index 00000000..18273ac7 --- /dev/null +++ b/cmd/pkg/downloaders/core/trackers.go @@ -0,0 +1,95 @@ +package core + +import ( + "context" + "errors" + "fmt" + "io" + "log/slog" + "net/http" + "strings" + "sync" + "time" +) + +const btTrackerListURL = "https://raw.githubusercontent.com/XIU2/TrackersListCollection/master/best_aria2.txt" + +var defaultBTTrackers = []string{ + "udp://tracker.opentrackr.org:1337/announce", + "udp://open.demonii.com:1337/announce", + "udp://open.stealth.si:80/announce", + "udp://tracker.torrent.eu.org:451/announce", + "udp://exodus.desync.com:6969/announce", + "udp://tracker.openbittorrent.com:6969/announce", + "udp://opentracker.i2p.rocks:6969/announce", + "udp://tracker.dler.org:6969/announce", + "http://tracker.openbittorrent.com:80/announce", + "udp://tracker.moeking.me:6969/announce", +} + +var ( + cachedBTTrackers string + loadBTTrackers sync.Once +) + +func BTTrackers(configured string) string { + if trackers := strings.TrimSpace(configured); trackers != "" { + return trackers + } + loadBTTrackers.Do(func() { + cachedBTTrackers = FetchBTTrackers() + }) + return cachedBTTrackers +} + +func FetchBTTrackers() string { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + trackers, err := fetchBTTrackers(ctx, http.DefaultClient, btTrackerListURL) + if err != nil { + slog.Warn("fetch bt trackers failed, using bundled fallback", "url", btTrackerListURL, "err", err) + return DefaultBTTrackers() + } + return trackers +} + +func fetchBTTrackers(ctx context.Context, client *http.Client, url string) (string, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return "", err + } + resp, err := client.Do(req) + if err != nil { + return "", err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return "", fmt.Errorf("unexpected status %s", resp.Status) + } + body, err := io.ReadAll(io.LimitReader(resp.Body, 1<<20)) + if err != nil { + return "", err + } + if list := strings.TrimSpace(string(body)); list != "" { + return list, nil + } + return "", errors.New("empty tracker list") +} + +func DefaultBTTrackers() string { + return strings.Join(defaultBTTrackers, ",") +} + +func BTTrackersForQBittorrent(trackers string) string { + parts := strings.FieldsFunc(trackers, func(r rune) bool { + return r == ',' || r == '\n' || r == '\r' + }) + out := make([]string, 0, len(parts)) + for _, part := range parts { + tracker := strings.TrimSpace(part) + if tracker != "" { + out = append(out, tracker) + } + } + return strings.Join(out, "|") +} diff --git a/cmd/pkg/downloaders/core/trackers_test.go b/cmd/pkg/downloaders/core/trackers_test.go new file mode 100644 index 00000000..eece223c --- /dev/null +++ b/cmd/pkg/downloaders/core/trackers_test.go @@ -0,0 +1,102 @@ +package core + +import ( + "context" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/saltbo/zpan/internal/downloader" +) + +func TestFetchBTTrackers(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte(" udp://tracker.example:1337/announce \n")) + })) + defer server.Close() + + trackers, err := fetchBTTrackers(context.Background(), server.Client(), server.URL) + if err != nil { + t.Fatal(err) + } + if trackers != "udp://tracker.example:1337/announce" { + t.Fatalf("unexpected trackers: %q", trackers) + } +} + +func TestFetchBTTrackersReturnsErrors(t *testing.T) { + t.Run("status", func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.Error(w, "nope", http.StatusBadGateway) + })) + defer server.Close() + + _, err := fetchBTTrackers(context.Background(), server.Client(), server.URL) + if err == nil || !strings.Contains(err.Error(), "502") { + t.Fatalf("expected status error, got %v", err) + } + }) + + t.Run("empty", func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {})) + defer server.Close() + + _, err := fetchBTTrackers(context.Background(), server.Client(), server.URL) + if err == nil || !strings.Contains(err.Error(), "empty") { + t.Fatalf("expected empty response error, got %v", err) + } + }) +} + +func TestBTTrackersForQBittorrent(t *testing.T) { + got := BTTrackersForQBittorrent(" udp://one/announce,\n udp://two/announce\r\nudp://three/announce ") + want := "udp://one/announce|udp://two/announce|udp://three/announce" + if got != want { + t.Fatalf("expected %q, got %q", want, got) + } +} + +func TestBTTrackersUsesConfiguredValueWithoutFetch(t *testing.T) { + got := BTTrackers(" udp://configured.example:1337/announce ") + if got != "udp://configured.example:1337/announce" { + t.Fatalf("unexpected configured trackers: %q", got) + } +} + +func TestDefaultBTTrackers(t *testing.T) { + got := DefaultBTTrackers() + if got == "" { + t.Fatal("expected bundled trackers") + } + if !strings.Contains(got, "udp://tracker.opentrackr.org:1337/announce") { + t.Fatalf("expected bundled tracker list, got %q", got) + } + if strings.Contains(got, "\n") { + t.Fatalf("default trackers should be comma-separated, got %q", got) + } +} + +type regionResolver struct{} + +func (regionResolver) LookupPeerRegion(ip string) (string, string) { + if ip == "203.0.113.10" { + return "US", "CA" + } + return "", "" +} + +func TestApplyPeerRegion(t *testing.T) { + ApplyPeerRegion(nil, "203.0.113.10", "", regionResolver{}) + + peer := downloader.Peer{} + ApplyPeerRegion(&peer, "203.0.113.10", "", regionResolver{}) + if peer.CountryCode != "US" || peer.RegionCode != "CA" { + t.Fatalf("expected resolver region, got %#v", peer) + } + peer = downloader.Peer{} + ApplyPeerRegion(&peer, "198.51.100.10", "JP", regionResolver{}) + if peer.CountryCode != "JP" || peer.RegionCode != "" { + t.Fatalf("expected fallback country, got %#v", peer) + } +} diff --git a/cmd/pkg/downloaders/httpdl/http.go b/cmd/pkg/downloaders/httpdl/http.go new file mode 100644 index 00000000..cc02ef5a --- /dev/null +++ b/cmd/pkg/downloaders/httpdl/http.go @@ -0,0 +1,253 @@ +package httpdl + +import ( + "context" + "errors" + "fmt" + "io" + nethttp "net/http" + "net/url" + "os" + "path/filepath" + "strconv" + "time" + + "github.com/saltbo/zpan/internal/downloader" + "github.com/saltbo/zpan/pkg/downloaders/core" +) + +func init() { + downloader.Register("http", true, nil, New) +} + +func New(cfg downloader.Config) (downloader.Downloader, error) { + return HTTP{Dir: cfg.DownloadDir}, nil +} + +type HTTP struct { + Dir string +} + +type progressWriter struct { + progress downloader.ProgressReporter + total *int64 + downloaded int64 + lastBytes int64 + lastAt time.Time +} + +func newDownloadProgressWriter(progress downloader.ProgressReporter, total *int64, downloaded int64) *progressWriter { + return &progressWriter{progress: progress, total: total, downloaded: downloaded, lastBytes: downloaded, lastAt: time.Now()} +} + +func (w *progressWriter) Write(data []byte) (int, error) { + n := len(data) + w.downloaded += int64(n) + now := time.Now() + if now.Sub(w.lastAt) >= time.Second { + bps := int64(float64(w.downloaded-w.lastBytes) / now.Sub(w.lastAt).Seconds()) + if err := w.progress(downloader.ProgressUpdate{ + Downloaded: w.downloaded, + Total: w.total, + Bps: bps, + Runtime: &downloader.TaskRuntime{Engine: "http", Phase: "downloading"}, + }); err != nil { + return n, err + } + w.lastBytes = w.downloaded + w.lastAt = now + } + return n, nil +} + +func (w *progressWriter) Downloaded() int64 { + return w.downloaded +} + +func (h HTTP) Name() string { + return "http" +} + +func (h HTTP) Capabilities() downloader.Capabilities { + return downloader.Capabilities{SourceTypes: []string{"http"}} +} + +func (h HTTP) Start(ctx context.Context) error { + return nil +} + +func (h HTTP) Stop(ctx context.Context) error { + return nil +} + +func (h HTTP) Check(ctx context.Context) error { + if err := os.MkdirAll(h.Dir, 0o755); err != nil { + return err + } + file, err := os.CreateTemp(h.Dir, ".zpan-check-*") + if err != nil { + return err + } + path := file.Name() + if err := file.Close(); err != nil { + return err + } + return os.Remove(path) +} + +func (h HTTP) ResetTask(ctx context.Context, task downloader.DownloadTask) error { + if task.SourceType() != "http" { + return nil + } + return os.RemoveAll(filepath.Join(h.Dir, task.ID)) +} + +func (h HTTP) InspectTask(ctx context.Context, task downloader.DownloadTask) (downloader.TaskSnapshot, bool, error) { + if task.SourceType() != "http" { + return downloader.TaskSnapshot{}, false, nil + } + size, ok := completedHTTPCheckpoint(task) + if !ok { + return downloader.TaskSnapshot{}, false, nil + } + path, name, err := h.outputPath(task) + if err != nil { + return downloader.TaskSnapshot{}, false, err + } + info, err := os.Stat(path) + if err != nil { + return downloader.TaskSnapshot{}, false, fmt.Errorf("inspect http completed file: %w", err) + } + if info.IsDir() { + return downloader.TaskSnapshot{}, false, fmt.Errorf("inspect http completed file: %s is a directory", path) + } + if info.Size() != size { + return downloader.TaskSnapshot{}, false, fmt.Errorf("inspect http completed file: size mismatch path=%s expected=%d actual=%d", path, size, info.Size()) + } + result := downloader.Result{Path: path, Name: name, Size: size} + return downloader.TaskSnapshot{ + State: downloader.TaskStateCompleted, + Downloaded: size, + Total: &size, + Runtime: &downloader.TaskRuntime{Engine: "http", Phase: "completed"}, + Result: &result, + }, true, nil +} + +func (h HTTP) Download(ctx context.Context, task downloader.DownloadTask, progress downloader.ProgressReporter) (downloader.Result, error) { + if task.SourceType() != "http" { + return downloader.Result{}, errors.New("http engine only supports http sources") + } + taskDir := filepath.Join(h.Dir, task.ID) + if err := os.MkdirAll(taskDir, 0o755); err != nil { + return downloader.Result{}, err + } + + path, name, err := h.outputPath(task) + if err != nil { + return downloader.Result{}, err + } + existingSize, err := existingFileSize(path) + if err != nil { + return downloader.Result{}, err + } + req, err := nethttp.NewRequestWithContext(ctx, nethttp.MethodGet, task.SourceURI(), nil) + if err != nil { + return downloader.Result{}, err + } + if existingSize > 0 { + req.Header.Set("Range", "bytes="+strconv.FormatInt(existingSize, 10)+"-") + } + + res, err := nethttp.DefaultClient.Do(req) + if err != nil { + return downloader.Result{}, err + } + defer res.Body.Close() + if res.StatusCode == nethttp.StatusRequestedRangeNotSatisfiable && existingSize > 0 { + return core.ResultFromPath(task, path, name) + } + if res.StatusCode < 200 || res.StatusCode >= 300 { + return downloader.Result{}, errors.New(res.Status) + } + + appendExisting := existingSize > 0 && res.StatusCode == nethttp.StatusPartialContent + if existingSize > 0 && !appendExisting { + existingSize = 0 + } + file, err := openOutputFile(path, appendExisting) + if err != nil { + return downloader.Result{}, err + } + defer file.Close() + + var total *int64 + if res.ContentLength > 0 { + value := res.ContentLength + existingSize + total = &value + } + counter := newDownloadProgressWriter(progress, total, existingSize) + if _, err := io.Copy(file, io.TeeReader(res.Body, counter)); err != nil { + return downloader.Result{}, err + } + if err := progress(downloader.ProgressUpdate{ + Downloaded: counter.Downloaded(), + Total: total, + Runtime: &downloader.TaskRuntime{Engine: "http", Phase: "completed"}, + }); err != nil { + return downloader.Result{}, err + } + return downloader.Result{Path: path, Name: name, Size: counter.Downloaded()}, nil +} + +func (h HTTP) outputPath(task downloader.DownloadTask) (string, string, error) { + parsed, err := httpURL(task.SourceURI()) + if err != nil { + return "", "", err + } + name := core.OutputName(task, core.FilenameFromURL(parsed)) + return filepath.Join(h.Dir, task.ID, name), name, nil +} + +func completedHTTPCheckpoint(task downloader.DownloadTask) (int64, bool) { + total := task.Status.Progress.Download.TotalBytes + if total == nil || *total <= 0 { + return 0, false + } + if task.Status.Progress.Download.Bytes != *total { + return 0, false + } + return *total, true +} + +func httpURL(raw string) (*url.URL, error) { + parsed, err := url.Parse(raw) + if err != nil { + return nil, err + } + if parsed.Scheme != "http" && parsed.Scheme != "https" { + return nil, fmt.Errorf("unsupported http source url: %s", raw) + } + return parsed, nil +} + +func existingFileSize(path string) (int64, error) { + info, err := os.Stat(path) + if err == nil { + if info.IsDir() { + return 0, nil + } + return info.Size(), nil + } + if os.IsNotExist(err) { + return 0, nil + } + return 0, err +} + +func openOutputFile(path string, appendExisting bool) (*os.File, error) { + if appendExisting { + return os.OpenFile(path, os.O_WRONLY|os.O_APPEND, 0o644) + } + return os.Create(path) +} diff --git a/cmd/pkg/downloaders/httpdl/http_test.go b/cmd/pkg/downloaders/httpdl/http_test.go new file mode 100644 index 00000000..60748065 --- /dev/null +++ b/cmd/pkg/downloaders/httpdl/http_test.go @@ -0,0 +1,337 @@ +package httpdl + +import ( + "context" + "errors" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strconv" + "strings" + "testing" + + "github.com/saltbo/zpan/internal/downloader" +) + +func downloadTask(id, sourceType, sourceURI string) downloader.DownloadTask { + return downloader.DownloadTask{ID: id, Source: downloader.Source{Type: sourceType, URI: sourceURI}} +} + +func completedDownloadTask(id, sourceType, sourceURI string, size int64) downloader.DownloadTask { + task := downloadTask(id, sourceType, sourceURI) + task.Status.Progress.Download = downloader.TransferProgress{Bytes: size, TotalBytes: &size} + return task +} + +func TestHTTPDownload(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Length", "11") + _, _ = w.Write([]byte("hello world")) + })) + defer server.Close() + + dir := t.TempDir() + progressCalls := 0 + var lastDetail *downloader.TaskRuntime + result, err := (HTTP{Dir: dir}).Download( + context.Background(), + downloadTask("task-1", "http", server.URL+"/file.txt"), + func(update downloader.ProgressUpdate) error { + progressCalls++ + lastDetail = update.Runtime + return nil + }, + ) + if err != nil { + t.Fatal(err) + } + if result.Name != "file.txt" || result.Size != 11 { + t.Fatalf("unexpected result: %#v", result) + } + data, err := os.ReadFile(result.Path) + if err != nil { + t.Fatal(err) + } + if string(data) != "hello world" { + t.Fatalf("unexpected file content: %q", string(data)) + } + if progressCalls == 0 || lastDetail == nil || lastDetail.Engine != "http" { + t.Fatalf("expected http progress callback, calls=%d detail=%#v", progressCalls, lastDetail) + } +} + +func TestHTTPRejectsMagnet(t *testing.T) { + _, err := (HTTP{Dir: t.TempDir()}).Download( + context.Background(), + downloadTask("task-1", "magnet", "magnet:?xt=urn:btih:abc"), + func(update downloader.ProgressUpdate) error { return nil }, + ) + if err == nil { + t.Fatal("expected magnet to be rejected by HTTP engine") + } +} + +func TestHTTPDownloadResumesExistingFile(t *testing.T) { + var rangeHeader string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + rangeHeader = r.Header.Get("Range") + if rangeHeader != "bytes=5-" { + t.Fatalf("expected resume range bytes=5-, got %q", rangeHeader) + } + w.Header().Set("Content-Length", "6") + w.WriteHeader(http.StatusPartialContent) + _, _ = w.Write([]byte(" world")) + })) + defer server.Close() + + dir := t.TempDir() + taskDir := filepath.Join(dir, "task-1") + if err := os.MkdirAll(taskDir, 0o755); err != nil { + t.Fatal(err) + } + path := filepath.Join(taskDir, "file.txt") + if err := os.WriteFile(path, []byte("hello"), 0o644); err != nil { + t.Fatal(err) + } + + result, err := (HTTP{Dir: dir}).Download( + context.Background(), + downloadTask("task-1", "http", server.URL+"/file.txt"), + func(update downloader.ProgressUpdate) error { return nil }, + ) + if err != nil { + t.Fatal(err) + } + data, err := os.ReadFile(result.Path) + if err != nil { + t.Fatal(err) + } + if string(data) != "hello world" || result.Size != 11 { + t.Fatalf("expected resumed content size 11, got %q size=%d", string(data), result.Size) + } +} + +func TestHTTPInspectTaskUsesCompletedCheckpoint(t *testing.T) { + dir := t.TempDir() + taskDir := filepath.Join(dir, "task-1") + if err := os.MkdirAll(taskDir, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(taskDir, "payload.bin"), []byte("payload"), 0o644); err != nil { + t.Fatal(err) + } + total := int64(7) + + snapshot, found, err := (HTTP{Dir: dir}).InspectTask( + context.Background(), + completedDownloadTask("task-1", "http", "https://example.com/payload.bin", total), + ) + + if err != nil { + t.Fatal(err) + } + if !found || snapshot.State != downloader.TaskStateCompleted { + t.Fatalf("expected completed checkpoint, found=%v snapshot=%#v", found, snapshot) + } + if snapshot.Result == nil || snapshot.Result.Path != filepath.Join(taskDir, "payload.bin") || snapshot.Result.Size != 7 { + t.Fatalf("unexpected completed result: %#v", snapshot.Result) + } +} + +func TestHTTPInspectTaskDoesNotTrustLocalFileWithoutCompletedCheckpoint(t *testing.T) { + dir := t.TempDir() + taskDir := filepath.Join(dir, "task-1") + if err := os.MkdirAll(taskDir, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(taskDir, "payload.bin"), []byte("payload"), 0o644); err != nil { + t.Fatal(err) + } + + snapshot, found, err := (HTTP{Dir: dir}).InspectTask( + context.Background(), + downloadTask("task-1", "http", "https://example.com/payload.bin"), + ) + + if err != nil { + t.Fatal(err) + } + if found { + t.Fatalf("expected local file without checkpoint not to be trusted, got %#v", snapshot) + } +} + +func TestHTTPInspectTaskRejectsSizeMismatch(t *testing.T) { + dir := t.TempDir() + taskDir := filepath.Join(dir, "task-1") + if err := os.MkdirAll(taskDir, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(taskDir, "payload.bin"), []byte("payload"), 0o644); err != nil { + t.Fatal(err) + } + total := int64(8) + + _, _, err := (HTTP{Dir: dir}).InspectTask( + context.Background(), + completedDownloadTask("task-1", "http", "https://example.com/payload.bin", total), + ) + + if err == nil || !strings.Contains(err.Error(), "size mismatch") { + t.Fatalf("expected size mismatch error, got %v", err) + } +} + +func TestHTTPRangeNotSatisfiableReusesExistingFile(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if got := r.Header.Get("Range"); got != "bytes=7-" { + t.Fatalf("expected range request, got %q", got) + } + w.WriteHeader(http.StatusRequestedRangeNotSatisfiable) + })) + defer server.Close() + + dir := t.TempDir() + taskDir := filepath.Join(dir, "task-1") + if err := os.MkdirAll(taskDir, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(taskDir, "payload.bin"), []byte("payload"), 0o644); err != nil { + t.Fatal(err) + } + + result, err := (HTTP{Dir: dir}).Download( + context.Background(), + downloadTask("task-1", "http", server.URL+"/payload.bin"), + func(update downloader.ProgressUpdate) error { return nil }, + ) + if err != nil { + t.Fatal(err) + } + if result.Size != int64(len("payload")) { + t.Fatalf("expected existing size %d, got %d", len("payload"), result.Size) + } +} + +func TestHTTPPropagatesProgressError(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Length", strconv.Itoa(1<<20)) + _, _ = w.Write(make([]byte, 1<<20)) + })) + defer server.Close() + + want := errors.New("stop") + _, err := (HTTP{Dir: t.TempDir()}).Download( + context.Background(), + downloadTask("task-1", "http", server.URL+"/payload.bin"), + func(update downloader.ProgressUpdate) error { + if update.Runtime != nil && update.Runtime.Phase == "completed" { + return want + } + return nil + }, + ) + if !errors.Is(err, want) { + t.Fatalf("expected progress error, got %v", err) + } +} + +func TestHTTPMetadataAndHealth(t *testing.T) { + engine := HTTP{Dir: t.TempDir()} + if engine.Name() != "http" { + t.Fatalf("unexpected name %q", engine.Name()) + } + if got := engine.Capabilities().SourceTypes; len(got) != 1 || got[0] != "http" { + t.Fatalf("unexpected capabilities %#v", got) + } + if err := engine.Start(context.Background()); err != nil { + t.Fatal(err) + } + if err := engine.Stop(context.Background()); err != nil { + t.Fatal(err) + } + if err := engine.Check(context.Background()); err != nil { + t.Fatal(err) + } +} + +func TestHTTPResetAndInspectIgnoreNonHTTP(t *testing.T) { + dir := t.TempDir() + taskDir := filepath.Join(dir, "task-1") + if err := os.MkdirAll(taskDir, 0o755); err != nil { + t.Fatal(err) + } + if err := (HTTP{Dir: dir}).ResetTask(context.Background(), downloadTask("task-1", "magnet", "magnet:?xt=urn:btih:abc")); err != nil { + t.Fatal(err) + } + if _, err := os.Stat(taskDir); err != nil { + t.Fatal(err) + } + if err := (HTTP{Dir: dir}).ResetTask(context.Background(), downloadTask("task-1", "http", "https://example.com/file.bin")); err != nil { + t.Fatal(err) + } + if _, err := os.Stat(taskDir); !os.IsNotExist(err) { + t.Fatalf("expected task dir removal, got %v", err) + } + if _, found, err := (HTTP{Dir: dir}).InspectTask(context.Background(), downloadTask("task-1", "magnet", "magnet:?xt=urn:btih:abc")); err != nil || found { + t.Fatalf("expected non-http inspect ignored, found=%v err=%v", found, err) + } +} + +func TestHTTPDownloadErrorBranches(t *testing.T) { + if _, err := (HTTP{Dir: t.TempDir()}).Download(context.Background(), downloadTask("task-1", "http", "ftp://example.com/file.bin"), func(downloader.ProgressUpdate) error { return nil }); err == nil { + t.Fatal("expected unsupported URL error") + } + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusInternalServerError) + })) + defer server.Close() + if _, err := (HTTP{Dir: t.TempDir()}).Download(context.Background(), downloadTask("task-1", "http", server.URL+"/file.bin"), func(downloader.ProgressUpdate) error { return nil }); err == nil || !strings.Contains(err.Error(), "500") { + t.Fatalf("expected HTTP status error, got %v", err) + } +} + +func TestHTTPDownloadRestartsWhenServerIgnoresRange(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Header.Get("Range") == "" { + t.Fatal("expected range header") + } + w.Header().Set("Content-Length", "5") + _, _ = w.Write([]byte("fresh")) + })) + defer server.Close() + dir := t.TempDir() + taskDir := filepath.Join(dir, "task-1") + if err := os.MkdirAll(taskDir, 0o755); err != nil { + t.Fatal(err) + } + path := filepath.Join(taskDir, "file.bin") + if err := os.WriteFile(path, []byte("stale"), 0o644); err != nil { + t.Fatal(err) + } + result, err := (HTTP{Dir: dir}).Download(context.Background(), downloadTask("task-1", "http", server.URL+"/file.bin"), func(downloader.ProgressUpdate) error { return nil }) + if err != nil { + t.Fatal(err) + } + data, err := os.ReadFile(result.Path) + if err != nil { + t.Fatal(err) + } + if string(data) != "fresh" || result.Size != 5 { + t.Fatalf("expected restarted content, got %q size=%d", string(data), result.Size) + } +} + +func TestHTTPInspectTaskRejectsDirectoryResult(t *testing.T) { + dir := t.TempDir() + taskDir := filepath.Join(dir, "task-1") + if err := os.MkdirAll(filepath.Join(taskDir, "payload.bin"), 0o755); err != nil { + t.Fatal(err) + } + total := int64(7) + _, _, err := (HTTP{Dir: dir}).InspectTask(context.Background(), completedDownloadTask("task-1", "http", "https://example.com/payload.bin", total)) + if err == nil || !strings.Contains(err.Error(), "is a directory") { + t.Fatalf("expected directory result error, got %v", err) + } +} diff --git a/cmd/internal/engine/live_download_test.go b/cmd/pkg/downloaders/live_download_test.go similarity index 89% rename from cmd/internal/engine/live_download_test.go rename to cmd/pkg/downloaders/live_download_test.go index ed5de8d0..4a092f70 100644 --- a/cmd/internal/engine/live_download_test.go +++ b/cmd/pkg/downloaders/live_download_test.go @@ -1,4 +1,4 @@ -package engine +package downloaders_test import ( "bytes" @@ -18,17 +18,18 @@ import ( "testing" "time" - "github.com/saltbo/zpan/internal/client" + "github.com/saltbo/zpan/internal/downloader" + "github.com/saltbo/zpan/pkg/downloaders/aria2" + "github.com/saltbo/zpan/pkg/downloaders/httpdl" + "github.com/saltbo/zpan/pkg/downloaders/qbittorrent" ) -func liveTask(id, sourceType, sourceURI, name string) client.DownloadTask { - return client.DownloadTask{ - ID: id, - Spec: client.DownloadTaskSpec{ - Source: client.DownloadTaskSource{Type: sourceType, URI: sourceURI}, - Destination: client.DownloadTaskDestination{Name: name}, - Labels: client.DownloadTaskLabels{Tags: []string{}}, - }, +func liveTask(id, sourceType, sourceURI, name string) downloader.DownloadTask { + return downloader.DownloadTask{ + ID: id, + Source: downloader.Source{Type: sourceType, URI: sourceURI}, + Destination: downloader.Destination{Name: name}, + Labels: downloader.Labels{Tags: []string{}}, } } @@ -130,7 +131,7 @@ func TestLiveDownloadThreeSourceTypes(t *testing.T) { httpResult := runLiveDownload( t, ctx, - HTTP{Dir: filepath.Join(root, "http")}, + httpdl.HTTP{Dir: filepath.Join(root, "http")}, liveTask("live-http", "http", httpServer.URL+"/"+fixtureName, "http-fixture.txt"), ) magnetURL := fmt.Sprintf( @@ -142,13 +143,13 @@ func TestLiveDownloadThreeSourceTypes(t *testing.T) { magnetResult := runLiveDownload( t, ctx, - Aria2{URL: fmt.Sprintf("ws://127.0.0.1:%d/jsonrpc", rpcPort), Dir: filepath.Join(root, "magnet")}, + &aria2.Aria2{URL: fmt.Sprintf("ws://127.0.0.1:%d/jsonrpc", rpcPort), Dir: filepath.Join(root, "magnet")}, liveTask("live-magnet", "magnet", magnetURL, "magnet-fixture.txt"), ) torrentResult := runLiveDownload( t, ctx, - Aria2{URL: fmt.Sprintf("ws://127.0.0.1:%d/jsonrpc", rpcPort), Dir: filepath.Join(root, "torrent-url")}, + &aria2.Aria2{URL: fmt.Sprintf("ws://127.0.0.1:%d/jsonrpc", rpcPort), Dir: filepath.Join(root, "torrent-url")}, liveTask("live-torrent-url", "torrent_url", torrentServer.URL+"/fixture.torrent", "torrent-url-fixture.txt"), ) @@ -240,7 +241,7 @@ func TestLiveQBittorrentDownloadTorrentURL(t *testing.T) { qbit := startQBittorrentForTest(t, ctx, qbittorrentBinary, filepath.Join(root, "qbit-profile"), webUIPort) defer stopProcess(qbit) - result := runLiveDownload(t, ctx, QBittorrent{ + result := runLiveDownload(t, ctx, &qbittorrent.QBittorrent{ URL: "http://127.0.0.1:" + strconv.Itoa(webUIPort), Dir: filepath.Join(root, "qbit-downloads"), }, liveTask("live-qbit-torrent-url", "torrent_url", torrentServer.URL+"/fixture.torrent", "qbit-fixture.txt")) @@ -249,14 +250,14 @@ func TestLiveQBittorrentDownloadTorrentURL(t *testing.T) { t.Logf("qBittorrent torrent_url result: %s (%d bytes)", result.Path, result.Size) } -func runLiveDownload(t *testing.T, ctx context.Context, downloader Engine, task client.DownloadTask) Result { +func runLiveDownload(t *testing.T, ctx context.Context, engine downloader.Downloader, task downloader.DownloadTask) downloader.Result { t.Helper() var lastDownloaded int64 - result, err := downloader.Download(ctx, task, func(downloaded int64, total *int64, bps int64, detail *client.DownloadTaskRuntime) error { - if downloaded < lastDownloaded { - t.Fatalf("download progress moved backwards: %d < %d", downloaded, lastDownloaded) + result, err := engine.Download(ctx, task, func(update downloader.ProgressUpdate) error { + if update.Downloaded < lastDownloaded { + t.Fatalf("download progress moved backwards: %d < %d", update.Downloaded, lastDownloaded) } - lastDownloaded = downloaded + lastDownloaded = update.Downloaded return nil }) if err != nil { @@ -298,7 +299,7 @@ WebUI\Port=%d if err := cmd.Start(); err != nil { t.Fatal(err) } - engine := QBittorrent{URL: "http://127.0.0.1:" + strconv.Itoa(webUIPort), Dir: t.TempDir()} + engine := qbittorrent.QBittorrent{URL: "http://127.0.0.1:" + strconv.Itoa(webUIPort), Dir: t.TempDir()} deadline := time.Now().Add(30 * time.Second) var lastErr error for time.Now().Before(deadline) { diff --git a/cmd/internal/engine/qbittorrent.go b/cmd/pkg/downloaders/qbittorrent/qbittorrent.go similarity index 68% rename from cmd/internal/engine/qbittorrent.go rename to cmd/pkg/downloaders/qbittorrent/qbittorrent.go index fb55cb53..6e729753 100644 --- a/cmd/internal/engine/qbittorrent.go +++ b/cmd/pkg/downloaders/qbittorrent/qbittorrent.go @@ -1,4 +1,4 @@ -package engine +package qbittorrent import ( "context" @@ -12,10 +12,37 @@ import ( "strings" "time" - qbittorrent "github.com/autobrr/go-qbittorrent" - "github.com/saltbo/zpan/internal/client" + "github.com/autobrr/go-qbittorrent" + "github.com/saltbo/zpan/internal/downloader" + "github.com/saltbo/zpan/pkg/downloaders/core" + "github.com/saltbo/zpan/pkg/downloaders/httpdl" + "github.com/saltbo/zpan/pkg/geoip" + "github.com/saltbo/zpan/pkg/system" ) +func init() { + downloader.Register("qbittorrent", false, configured, New) +} + +func configured(cfg downloader.Config) bool { + return cfg.QBittorrent.Configured +} + +func New(cfg downloader.Config) (downloader.Downloader, error) { + return &QBittorrent{ + URL: cfg.QBittorrent.URL, + Username: cfg.QBittorrent.Username, + Password: cfg.QBittorrent.Password, + Dir: cfg.DownloadDir, + StateDir: cfg.StateDir, + ListenPort: cfg.BTListenPort, + RetainSeed: cfg.SeedEnabled, + BtTrackers: cfg.Aria2.BtTrackers, + Managed: !cfg.QBittorrent.Configured && (cfg.Engine == "" || cfg.Engine == "auto" || cfg.Engine == "qbittorrent"), + GeoIP: cfg.GeoIP, + }, nil +} + type QBittorrent struct { URL string Username string @@ -24,54 +51,83 @@ type QBittorrent struct { StateDir string ListenPort int RetainSeed bool - GeoIP PeerGeoIPResolver + BtTrackers string + Managed bool + GeoIP geoip.Resolver + cmd *exec.Cmd } func (q QBittorrent) Name() string { return "qbittorrent" } -func (q QBittorrent) Capabilities() []string { - return []string{"http", "magnet", "torrent"} +func (q QBittorrent) Capabilities() downloader.Capabilities { + return downloader.Capabilities{SourceTypes: []string{"magnet", "torrent", "torrent_url"}} } -func (q QBittorrent) Start(ctx context.Context) (*exec.Cmd, error) { - path, err := lookPathAny("qbittorrent-nox", "qbittorrent") +func (q *QBittorrent) Start(ctx context.Context) error { + if !q.Managed { + return nil + } + path, err := system.LookPathAny("qbittorrent-nox", "qbittorrent") if err != nil { - return nil, err + return err } args, err := q.startArgs(path) if err != nil { - return nil, err + return err } cmd := exec.Command(path, args...) - configureEngineProcess(cmd) + system.ConfigureProcess(cmd) if err := cmd.Start(); err != nil { - return nil, err + return err } - return cmd, nil + q.cmd = cmd + return nil +} + +func (q *QBittorrent) Stop(ctx context.Context) error { + if q.cmd == nil { + return nil + } + var errs []error + if q.cmd.Process != nil { + if err := q.cmd.Process.Kill(); err != nil { + errs = append(errs, fmt.Errorf("kill qbittorrent process: %w", err)) + } + } + done := make(chan error, 1) + go func() { done <- q.cmd.Wait() }() + select { + case <-done: + case <-ctx.Done(): + errs = append(errs, ctx.Err()) + } + q.cmd = nil + return errors.Join(errs...) } func (q QBittorrent) startArgs(path string) ([]string, error) { - webURL, err := parseLocalEngineURL(q.URL, "8080") + webURL, err := system.ParseLocalEngineURL(q.URL, "8080") if err != nil { return nil, err } args := []string{} if q.StateDir != "" { profileDir := filepath.Join(q.StateDir, "qbittorrent") - if err := writeQBittorrentManagedConfig(profileDir, q.Dir, webURL.port, listenPortString(q.ListenPort)); err != nil { + trackers := core.BTTrackers(q.BtTrackers) + if err := writeQBittorrentManagedConfig(profileDir, q.Dir, webURL.Port, listenPortString(q.ListenPort), trackers); err != nil { return nil, err } args = append(args, "--profile="+profileDir) } - if strings.Contains(filepathBase(path), "qbittorrent-nox") { - args = append(args, "--webui-port="+webURL.port) + if strings.Contains(system.FilepathBase(path), "qbittorrent-nox") { + args = append(args, "--webui-port="+webURL.Port) } return args, nil } -func writeQBittorrentManagedConfig(profileDir string, downloadDir string, webUIPort string, listenPort string) error { +func writeQBittorrentManagedConfig(profileDir string, downloadDir string, webUIPort string, listenPort string, trackers string) error { configDir := filepath.Join(profileDir, "qBittorrent", "config") if err := os.MkdirAll(configDir, 0o755); err != nil { return err @@ -80,17 +136,29 @@ func writeQBittorrentManagedConfig(profileDir string, downloadDir string, webUIP Accepted=true [Preferences] +BitTorrent\DHT=true +BitTorrent\LSD=true +BitTorrent\PeX=true Connection\PortRangeMin=%s Downloads\SavePath=%s/ +Session\AddTrackers=%s +Session\AddTrackersEnabled=true WebUI\Address=127.0.0.1 WebUI\AuthSubnetWhitelist=127.0.0.1 WebUI\AuthSubnetWhitelistEnabled=true WebUI\LocalHostAuth=false WebUI\Port=%s -`, listenPort, filepath.ToSlash(downloadDir), webUIPort) +`, listenPort, filepath.ToSlash(downloadDir), strings.ReplaceAll(core.BTTrackersForQBittorrent(trackers), "|", `\n`), webUIPort) return os.WriteFile(filepath.Join(configDir, "qBittorrent.conf"), []byte(config), 0o600) } +func listenPortString(port int) string { + if port == 0 { + return "6881" + } + return fmt.Sprint(port) +} + func (q QBittorrent) Check(ctx context.Context) error { req, err := http.NewRequestWithContext(ctx, http.MethodGet, strings.TrimRight(q.URL, "/")+"/api/v2/app/version", nil) if err != nil { @@ -114,9 +182,9 @@ func (q QBittorrent) Check(ctx context.Context) error { return nil } -func (q QBittorrent) ResetTask(ctx context.Context, task client.DownloadTask) error { +func (q QBittorrent) ResetTask(ctx context.Context, task downloader.DownloadTask) error { if task.SourceType() == "http" { - return HTTP{Dir: q.Dir}.ResetTask(ctx, task) + return httpdl.HTTP{Dir: q.Dir}.ResetTask(ctx, task) } qbt, err := q.login(ctx) if err != nil { @@ -155,7 +223,7 @@ func (q QBittorrent) ResetTask(ctx context.Context, task client.DownloadTask) er return os.RemoveAll(taskDir) } -func (q QBittorrent) RestoreSeed(ctx context.Context, ref SeedRef) (*Seed, error) { +func (q QBittorrent) RestoreSeed(ctx context.Context, ref downloader.SeedRef) (*downloader.Seed, error) { qbt, err := q.login(ctx) if err != nil { return nil, err @@ -183,7 +251,7 @@ func (q QBittorrent) RestoreSeed(ctx context.Context, ref SeedRef) (*Seed, error path = filepath.Join(q.Dir, ref.TaskID) } _ = qbt.StartCtx(ctx, []string{torrent.Hash}) - return &Seed{ + return &downloader.Seed{ Engine: "qbittorrent", ID: torrent.Hash, InfoHash: torrent.Hash, @@ -193,17 +261,17 @@ func (q QBittorrent) RestoreSeed(ctx context.Context, ref SeedRef) (*Seed, error }, nil } -func (q QBittorrent) InspectTask(ctx context.Context, task client.DownloadTask) (TaskSnapshot, bool, error) { +func (q QBittorrent) InspectTask(ctx context.Context, task downloader.DownloadTask) (downloader.TaskSnapshot, bool, error) { if task.SourceType() == "http" { - return HTTP{Dir: q.Dir}.InspectTask(ctx, task) + return httpdl.HTTP{Dir: q.Dir}.InspectTask(ctx, task) } qbt, err := q.login(ctx) if err != nil { - return TaskSnapshot{}, false, err + return downloader.TaskSnapshot{}, false, err } torrent, ok, err := q.findTask(ctx, qbt, task) if err != nil || !ok { - return TaskSnapshot{}, ok, err + return downloader.TaskSnapshot{}, ok, err } return q.snapshotTask(ctx, qbt, task, torrent) } @@ -221,42 +289,42 @@ func (q QBittorrent) login(ctx context.Context) (*qbittorrent.Client, error) { return qbt, nil } -func (q QBittorrent) Download(ctx context.Context, task client.DownloadTask, progress Progress) (Result, error) { +func (q QBittorrent) Download(ctx context.Context, task downloader.DownloadTask, progress downloader.ProgressReporter) (downloader.Result, error) { if task.SourceType() == "http" { - return HTTP{Dir: q.Dir}.Download(ctx, task, progress) + return httpdl.HTTP{Dir: q.Dir}.Download(ctx, task, progress) } taskDir := filepath.Join(q.Dir, task.ID) if err := os.MkdirAll(taskDir, 0o755); err != nil { - return Result{}, err + return downloader.Result{}, err } qbt, err := q.login(ctx) if err != nil { - return Result{}, err + return downloader.Result{}, err } tag := qbittorrentTrackingTag(task.ID) torrent, ok, err := q.findTask(ctx, qbt, task) if err != nil { - return Result{}, err + return downloader.Result{}, err } if ok { _ = qbt.StartCtx(ctx, []string{torrent.Hash}) torrent, err = waitQBittorrent(ctx, qbt, tag, progress, q.GeoIP) if err != nil { - return Result{}, err + return downloader.Result{}, err } return q.resultFromTorrent(ctx, qbt, task, taskDir, torrent) } - options := qbittorrentAddOptions(task, taskDir, tag) + options := q.qbittorrentAddOptions(task, taskDir, tag) if _, err := qbt.AddTorrentFromUrlCtx(ctx, task.SourceURI(), options); err != nil { - return Result{}, err + return downloader.Result{}, err } torrent, err = waitQBittorrent(ctx, qbt, tag, progress, q.GeoIP) if err != nil { - return Result{}, err + return downloader.Result{}, err } return q.resultFromTorrent(ctx, qbt, task, taskDir, torrent) } @@ -264,16 +332,16 @@ func (q QBittorrent) Download(ctx context.Context, task client.DownloadTask, pro func (q QBittorrent) resultFromTorrent( ctx context.Context, qbt *qbittorrent.Client, - task client.DownloadTask, + task downloader.DownloadTask, taskDir string, torrent qbittorrent.Torrent, -) (Result, error) { +) (downloader.Result, error) { result, err := resultFromQBittorrentFiles(ctx, qbt, task, taskDir, torrent) if err != nil { - return Result{}, err + return downloader.Result{}, err } if q.RetainSeed { - result.Seed = &Seed{ + result.Seed = &downloader.Seed{ Engine: "qbittorrent", ID: torrent.Hash, InfoHash: torrent.Hash, @@ -290,9 +358,9 @@ func (q QBittorrent) resultFromTorrent( func (q QBittorrent) snapshotTask( ctx context.Context, qbt *qbittorrent.Client, - task client.DownloadTask, + task downloader.DownloadTask, torrent qbittorrent.Torrent, -) (TaskSnapshot, bool, error) { +) (downloader.TaskSnapshot, bool, error) { total := torrent.TotalSize if total <= 0 { total = torrent.Size @@ -301,22 +369,22 @@ func (q QBittorrent) snapshotTask( if total > 0 { totalPtr = &total } - snapshot := TaskSnapshot{ + snapshot := downloader.TaskSnapshot{ State: qbittorrentTaskState(torrent), Downloaded: torrent.Completed, Total: totalPtr, Bps: torrent.DlSpeed, Runtime: qbittorrentDetail(ctx, qbt, torrent, q.GeoIP), } - if snapshot.State != TaskStateCompleted { + if snapshot.State != downloader.TaskStateCompleted { return snapshot, true, nil } result, err := resultFromQBittorrentFiles(ctx, qbt, task, filepath.Join(q.Dir, task.ID), torrent) if err != nil { - return TaskSnapshot{}, false, err + return downloader.TaskSnapshot{}, false, err } if q.RetainSeed { - result.Seed = &Seed{ + result.Seed = &downloader.Seed{ Engine: "qbittorrent", ID: torrent.Hash, InfoHash: torrent.Hash, @@ -329,21 +397,21 @@ func (q QBittorrent) snapshotTask( return snapshot, true, nil } -func qbittorrentTaskState(torrent qbittorrent.Torrent) TaskState { +func qbittorrentTaskState(torrent qbittorrent.Torrent) downloader.TaskState { total := torrent.TotalSize if total <= 0 { total = torrent.Size } if torrent.Progress >= 1 || (torrent.AmountLeft == 0 && total > 0) { - return TaskStateCompleted + return downloader.TaskStateCompleted } if isQBittorrentErrorState(torrent.State) { - return TaskStateFailed + return downloader.TaskStateFailed } - return TaskStateDownloading + return downloader.TaskStateDownloading } -func (q QBittorrent) findTask(ctx context.Context, qbt *qbittorrent.Client, task client.DownloadTask) (qbittorrent.Torrent, bool, error) { +func (q QBittorrent) findTask(ctx context.Context, qbt *qbittorrent.Client, task downloader.DownloadTask) (qbittorrent.Torrent, bool, error) { tag := qbittorrentTrackingTag(task.ID) torrents, err := qbt.GetTorrentsCtx(ctx, qbittorrent.TorrentFilterOptions{Tag: tag}) if err != nil { @@ -365,7 +433,7 @@ func (q QBittorrent) findTask(ctx context.Context, qbt *qbittorrent.Client, task return qbittorrent.Torrent{}, false, nil } -func qbittorrentAddOptions(task client.DownloadTask, taskDir string, trackingTag string) map[string]string { +func (q QBittorrent) qbittorrentAddOptions(task downloader.DownloadTask, taskDir string, trackingTag string) map[string]string { category := "zpan" if task.Category() != "" { category = task.Category() @@ -379,9 +447,12 @@ func qbittorrentAddOptions(task client.DownloadTask, taskDir string, trackingTag LimitSeedTime: 0, SequentialDownload: false, }).Prepare() - if name := requestedOutputName(task); name != "" { + if name := core.RequestedOutputName(task); name != "" { options["rename"] = name } + if trackers := core.BTTrackersForQBittorrent(core.BTTrackers(q.BtTrackers)); trackers != "" { + options["trackers"] = trackers + } return options } @@ -414,18 +485,18 @@ func (q QBittorrent) cleanupSeed(hash string, localPath string) func(context.Con } } -func (q QBittorrent) seedSnapshot(hash string) func(context.Context) (SeedSnapshot, error) { - return func(ctx context.Context) (SeedSnapshot, error) { +func (q QBittorrent) seedSnapshot(hash string) func(context.Context) (downloader.SeedSnapshot, error) { + return func(ctx context.Context) (downloader.SeedSnapshot, error) { qbt, err := q.login(ctx) if err != nil { - return SeedSnapshot{}, err + return downloader.SeedSnapshot{}, err } torrents, err := qbt.GetTorrentsCtx(ctx, qbittorrent.TorrentFilterOptions{Hashes: []string{hash}}) if err != nil { - return SeedSnapshot{}, err + return downloader.SeedSnapshot{}, err } if len(torrents) == 0 { - return SeedSnapshot{}, fmt.Errorf("qbittorrent torrent %s not found", hash) + return downloader.SeedSnapshot{}, fmt.Errorf("qbittorrent torrent %s not found", hash) } torrent := torrents[0] total := torrent.TotalSize @@ -438,7 +509,7 @@ func (q QBittorrent) seedSnapshot(hash string) func(context.Context) (SeedSnapsh } detail := qbittorrentDetail(ctx, qbt, torrent, q.GeoIP) detail.Phase = "seeding" - return SeedSnapshot{ + return downloader.SeedSnapshot{ Downloaded: torrent.Completed, Total: totalPtr, Bps: torrent.DlSpeed, @@ -451,8 +522,8 @@ func waitQBittorrent( ctx context.Context, qbt *qbittorrent.Client, tag string, - progress Progress, - geoIP PeerGeoIPResolver, + progress downloader.ProgressReporter, + geoIP geoip.Resolver, ) (qbittorrent.Torrent, error) { ticker := time.NewTicker(time.Second) defer ticker.Stop() @@ -477,7 +548,12 @@ func waitQBittorrent( if total > 0 { totalPtr = &total } - if err := progress(torrent.Completed, totalPtr, torrent.DlSpeed, qbittorrentDetail(ctx, qbt, torrent, geoIP)); err != nil { + if err := progress(downloader.ProgressUpdate{ + Downloaded: torrent.Completed, + Total: totalPtr, + Bps: torrent.DlSpeed, + Runtime: qbittorrentDetail(ctx, qbt, torrent, geoIP), + }); err != nil { _ = qbt.StopCtx(ctx, []string{torrent.Hash}) return qbittorrent.Torrent{}, err } @@ -495,8 +571,8 @@ func qbittorrentDetail( ctx context.Context, qbt *qbittorrent.Client, torrent qbittorrent.Torrent, - geoIP PeerGeoIPResolver, -) *client.DownloadTaskRuntime { + geoIP geoip.Resolver, +) *downloader.TaskRuntime { connections := int64(torrent.NumSeeds + torrent.NumLeechs) seeders := torrent.NumSeeds leechers := torrent.NumLeechs @@ -507,20 +583,20 @@ func qbittorrentDetail( if torrent.ETA >= 0 { eta = &torrent.ETA } - detail := &client.DownloadTaskRuntime{ + detail := &downloader.TaskRuntime{ Engine: "qbittorrent", Phase: qbittorrentPhase(string(torrent.State)), State: string(torrent.State), ETASeconds: eta, Connections: &connections, - Torrent: &client.DownloadTaskTorrentRuntime{ + Torrent: &downloader.TorrentRuntime{ InfoHash: torrent.Hash, Name: torrent.Name, Seeders: &seeders, Leechers: &leechers, Peers: &peers, }, - Seeding: &client.DownloadTaskSeedingRuntime{ + Seeding: &downloader.SeedingRuntime{ UploadedBytes: &uploaded, UploadBytesPerSecond: &uploadBps, }, @@ -552,7 +628,7 @@ func qbittorrentPhase(state string) string { } } -func qbittorrentTrackers(ctx context.Context, qbt *qbittorrent.Client, torrent qbittorrent.Torrent) []client.DownloadTaskTracker { +func qbittorrentTrackers(ctx context.Context, qbt *qbittorrent.Client, torrent qbittorrent.Torrent) []downloader.Tracker { trackers := torrent.Trackers if len(trackers) == 0 && torrent.Hash != "" { loaded, err := qbt.GetTorrentTrackersCtx(ctx, torrent.Hash) @@ -560,12 +636,12 @@ func qbittorrentTrackers(ctx context.Context, qbt *qbittorrent.Client, torrent q trackers = loaded } } - out := make([]client.DownloadTaskTracker, 0, min(len(trackers), 20)) + out := make([]downloader.Tracker, 0, min(len(trackers), 20)) for _, tracker := range trackers { peers := int64(tracker.NumPeers) seeds := int64(tracker.NumSeeds) leechers := int64(tracker.NumLeechers) - out = append(out, client.DownloadTaskTracker{ + out = append(out, downloader.Tracker{ URL: tracker.Url, Status: fmt.Sprint(tracker.Status), Peers: &peers, @@ -580,7 +656,7 @@ func qbittorrentTrackers(ctx context.Context, qbt *qbittorrent.Client, torrent q return out } -func qbittorrentPeers(ctx context.Context, qbt *qbittorrent.Client, hash string, geoIP PeerGeoIPResolver) []client.DownloadTaskPeer { +func qbittorrentPeers(ctx context.Context, qbt *qbittorrent.Client, hash string, geoIP geoip.Resolver) []downloader.Peer { if hash == "" { return nil } @@ -588,7 +664,7 @@ func qbittorrentPeers(ctx context.Context, qbt *qbittorrent.Client, hash string, if err != nil || peers == nil { return nil } - out := make([]client.DownloadTaskPeer, 0, min(len(peers.Peers), 20)) + out := make([]downloader.Peer, 0, min(len(peers.Peers), 20)) for address, peer := range peers.Peers { progress := peer.Progress down := peer.DownSpeed @@ -597,14 +673,14 @@ func qbittorrentPeers(ctx context.Context, qbt *qbittorrent.Client, hash string, if peer.IP != "" && peer.Port > 0 { label = fmt.Sprintf("%s:%d", peer.IP, peer.Port) } - item := client.DownloadTaskPeer{ + item := downloader.Peer{ Address: label, Client: peer.Client, Progress: &progress, DownloadBps: &down, UploadBps: &up, } - applyPeerRegion(&item, peer.IP, peer.CountryCode, geoIP) + core.ApplyPeerRegion(&item, peer.IP, peer.CountryCode, geoIP) out = append(out, item) if len(out) >= 20 { break @@ -613,7 +689,7 @@ func qbittorrentPeers(ctx context.Context, qbt *qbittorrent.Client, hash string, return out } -func qbittorrentFiles(ctx context.Context, qbt *qbittorrent.Client, torrent qbittorrent.Torrent) []client.DownloadTaskFile { +func qbittorrentFiles(ctx context.Context, qbt *qbittorrent.Client, torrent qbittorrent.Torrent) []downloader.File { if torrent.Hash == "" { return nil } @@ -621,15 +697,15 @@ func qbittorrentFiles(ctx context.Context, qbt *qbittorrent.Client, torrent qbit if err != nil || files == nil { return nil } - out := make([]client.DownloadTaskFile, 0, min(len(*files), 50)) + out := make([]downloader.File, 0, min(len(*files), 50)) for _, file := range *files { if file.Size <= 0 { continue } completed := int64(float64(file.Size) * float64(file.Progress)) selected := file.Priority > 0 - out = append(out, client.DownloadTaskFile{ - Path: stripTorrentRoot(file.Name, torrent.Name), + out = append(out, downloader.File{ + Path: core.StripTorrentRoot(file.Name, torrent.Name), Size: file.Size, CompletedBytes: &completed, Selected: &selected, @@ -644,26 +720,26 @@ func qbittorrentFiles(ctx context.Context, qbt *qbittorrent.Client, torrent qbit func resultFromQBittorrentFiles( ctx context.Context, qbt *qbittorrent.Client, - task client.DownloadTask, + task downloader.DownloadTask, taskDir string, torrent qbittorrent.Torrent, -) (Result, error) { +) (downloader.Result, error) { files, err := qbt.GetFilesInformationCtx(ctx, torrent.Hash) if err != nil || files == nil { - return resultFromPath(task, taskDir, torrent.Name) + return core.ResultFromPath(task, taskDir, torrent.Name) } - downloaded := make([]downloadedFile, 0, len(*files)) + downloaded := make([]core.DownloadedFile, 0, len(*files)) for _, file := range *files { if file.Priority == 0 || file.Size <= 0 { continue } - abs, rel := downloadedPath(taskDir, file.Name) - downloaded = append(downloaded, downloadedFile{path: abs, relativePath: rel}) + abs, rel := core.DownloadedPath(taskDir, file.Name) + downloaded = append(downloaded, core.DownloadedFile{Path: abs, RelativePath: rel}) } if len(downloaded) == 0 { - return resultFromPath(task, taskDir, torrent.Name) + return core.ResultFromPath(task, taskDir, torrent.Name) } - return resultFromDownloadedFiles(task, taskDir, torrent.Name, downloaded) + return core.ResultFromDownloadedFiles(task, taskDir, torrent.Name, downloaded) } func isQBittorrentErrorState(state qbittorrent.TorrentState) bool { diff --git a/cmd/pkg/downloaders/qbittorrent/qbittorrent_test.go b/cmd/pkg/downloaders/qbittorrent/qbittorrent_test.go new file mode 100644 index 00000000..f4438b1f --- /dev/null +++ b/cmd/pkg/downloaders/qbittorrent/qbittorrent_test.go @@ -0,0 +1,316 @@ +package qbittorrent + +import ( + "context" + "fmt" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/autobrr/go-qbittorrent" + "github.com/saltbo/zpan/internal/downloader" +) + +func downloadTask(id, sourceType, sourceURI string) downloader.DownloadTask { + return downloader.DownloadTask{ID: id, Source: downloader.Source{Type: sourceType, URI: sourceURI}, Labels: downloader.Labels{Tags: []string{}}} +} + +func downloadTaskWithName(id, sourceType, sourceURI, name string) downloader.DownloadTask { + task := downloadTask(id, sourceType, sourceURI) + task.Destination.Name = name + return task +} + +func TestQBittorrentStartArgsWritesManagedListenPort(t *testing.T) { + stateDir := t.TempDir() + downloadDir := t.TempDir() + args, err := (QBittorrent{ + URL: "http://127.0.0.1:8080", + Dir: downloadDir, + StateDir: stateDir, + ListenPort: 51413, + BtTrackers: "udp://custom.example:1337/announce,udp://custom2.example:1337/announce", + }).startArgs("/usr/bin/qbittorrent-nox") + if err != nil { + t.Fatal(err) + } + joined := strings.Join(args, " ") + if !strings.Contains(joined, "--profile="+filepath.Join(stateDir, "qbittorrent")) { + t.Fatalf("expected managed profile, got %v", args) + } + if !strings.Contains(joined, "--webui-port=8080") { + t.Fatalf("expected webui port, got %v", args) + } + content, err := os.ReadFile(filepath.Join(stateDir, "qbittorrent", "qBittorrent", "config", "qBittorrent.conf")) + if err != nil { + t.Fatal(err) + } + text := string(content) + if !strings.Contains(text, `Connection\PortRangeMin=51413`) { + t.Fatalf("expected custom listen port, got:\n%s", text) + } + if !strings.Contains(text, `Downloads\SavePath=`+filepath.ToSlash(downloadDir)+`/`) { + t.Fatalf("expected custom download dir, got:\n%s", text) + } + if !strings.Contains(text, `BitTorrent\DHT=true`) || + !strings.Contains(text, `BitTorrent\PeX=true`) || + !strings.Contains(text, `Session\AddTrackersEnabled=true`) || + !strings.Contains(text, `Session\AddTrackers=udp://custom.example:1337/announce\nudp://custom2.example:1337/announce`) { + t.Fatalf("expected managed BT discovery config, got:\n%s", text) + } +} + +func TestQBittorrentStartArgsWithoutManagedProfile(t *testing.T) { + args, err := (QBittorrent{URL: "http://127.0.0.1:9090"}).startArgs("/usr/bin/qbittorrent") + if err != nil { + t.Fatal(err) + } + if len(args) != 0 { + t.Fatalf("expected no args without state dir and nox binary, got %v", args) + } + if _, err := (QBittorrent{URL: "://bad-url"}).startArgs("/usr/bin/qbittorrent-nox"); err == nil { + t.Fatal("expected invalid URL error") + } + if got := listenPortString(0); got != "6881" { + t.Fatalf("expected default listen port, got %q", got) + } + if got := listenPortString(51413); got != "51413" { + t.Fatalf("expected custom listen port, got %q", got) + } +} + +func TestQBittorrentConstructorAndMetadata(t *testing.T) { + cfg := downloader.Config{ + Engine: "qbittorrent", + DownloadDir: t.TempDir(), + StateDir: t.TempDir(), + BTListenPort: 51413, + SeedEnabled: true, + QBittorrent: downloader.QBittorrentConfig{ + URL: "http://127.0.0.1:8080", + Username: "u", + Password: "p", + }, + } + d, err := New(cfg) + if err != nil { + t.Fatal(err) + } + q := d.(*QBittorrent) + if q.Name() != "qbittorrent" { + t.Fatalf("unexpected name %q", q.Name()) + } + if got := q.Capabilities().SourceTypes; len(got) != 3 || got[0] != "magnet" { + t.Fatalf("unexpected capabilities %#v", got) + } + if !q.Managed || !q.RetainSeed || q.ListenPort != 51413 || q.Username != "u" || q.Password != "p" { + t.Fatalf("unexpected constructed downloader: %#v", q) + } + q.Managed = false + if err := q.Start(context.Background()); err != nil { + t.Fatal(err) + } + if err := q.Stop(context.Background()); err != nil { + t.Fatal(err) + } + if !configured(downloader.Config{QBittorrent: downloader.QBittorrentConfig{Configured: true}}) { + t.Fatal("expected configured qbit config") + } +} + +func TestQBittorrentStartAndCheckErrors(t *testing.T) { + if err := (&QBittorrent{Managed: true}).Start(context.Background()); err == nil { + t.Fatal("expected missing binary error") + } + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.Error(w, "down", http.StatusBadGateway) + })) + defer server.Close() + if err := (QBittorrent{URL: server.URL}).Check(context.Background()); err == nil || !strings.Contains(err.Error(), "502") { + t.Fatalf("expected bad gateway check error, got %v", err) + } + empty := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {})) + defer empty.Close() + if err := (QBittorrent{URL: empty.URL}).Check(context.Background()); err == nil || !strings.Contains(err.Error(), "did not return") { + t.Fatalf("expected empty version error, got %v", err) + } +} + +func TestQBittorrentCheckUsesWebAPIVersion(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/api/v2/app/version" { + t.Fatalf("unexpected path: %s", r.URL.Path) + } + _, _ = w.Write([]byte("5.0.0")) + })) + defer server.Close() + + if err := (QBittorrent{URL: server.URL, Dir: t.TempDir()}).Check(context.Background()); err != nil { + t.Fatal(err) + } +} + +func TestQBittorrentHTTPDelegatesToBuiltin(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Length", "11") + _, _ = w.Write([]byte("hello qbit!")) + })) + defer server.Close() + + result, err := (QBittorrent{URL: "http://127.0.0.1:1", Dir: t.TempDir()}).Download( + context.Background(), + downloadTask("task-1", "http", server.URL+"/file.txt"), + func(update downloader.ProgressUpdate) error { return nil }, + ) + if err != nil { + t.Fatal(err) + } + data, err := os.ReadFile(result.Path) + if err != nil { + t.Fatal(err) + } + if string(data) != "hello qbit!" { + t.Fatalf("unexpected file content: %q", string(data)) + } +} + +func TestQBittorrentAddOptionsPassThroughTaskClassification(t *testing.T) { + task := downloadTaskWithName("task-1", "magnet", "magnet:?xt=urn:btih:abc", "fixture") + task.Labels.Category = "movies" + task.Labels.Tags = []string{"4k", "private"} + options := (QBittorrent{BtTrackers: "udp://tracker.example:1337/announce"}).qbittorrentAddOptions(task, "/tmp/zpan/task-1", qbittorrentTrackingTag("task-1")) + + if options["category"] != "movies" { + t.Fatalf("expected task category, got %q", options["category"]) + } + if options["tags"] != "ztid=task-1,4k,private" { + t.Fatalf("expected tracking and task tags, got %q", options["tags"]) + } + if options["rename"] != "fixture" { + t.Fatalf("expected rename option, got %q", options["rename"]) + } + if options["trackers"] != "udp://tracker.example:1337/announce" { + t.Fatalf("expected trackers option, got %q", options["trackers"]) + } +} + +func TestQBittorrentAddOptionsDefaultsCategory(t *testing.T) { + options := (QBittorrent{BtTrackers: "udp://tracker.example:1337/announce"}).qbittorrentAddOptions(downloader.DownloadTask{ID: "task-1"}, "/tmp/zpan/task-1", qbittorrentTrackingTag("task-1")) + if options["category"] != "zpan" { + t.Fatalf("expected default category, got %q", options["category"]) + } + if options["tags"] != "ztid=task-1" { + t.Fatalf("expected tracking tag, got %q", options["tags"]) + } +} + +func TestQBittorrentAddOptionsIgnoresTorrentTaskName(t *testing.T) { + options := (QBittorrent{BtTrackers: "udp://tracker.example:1337/announce"}).qbittorrentAddOptions( + downloadTaskWithName("task-1", "magnet", "magnet:?xt=urn:btih:abc", "movie.torrent"), + "/tmp/zpan/task-1", + qbittorrentTrackingTag("task-1"), + ) + if _, ok := options["rename"]; ok { + t.Fatalf("expected torrent task name to be ignored, got rename=%q", options["rename"]) + } +} + +func TestQBittorrentTaskState(t *testing.T) { + if got := qbittorrentTaskState(qbittorrent.Torrent{ + State: qbittorrent.TorrentState("stalledUP"), + Progress: 1, + AmountLeft: 0, + TotalSize: 100, + }); got != downloader.TaskStateCompleted { + t.Fatalf("expected seeding torrent to be completed, got %s", got) + } + if got := qbittorrentTaskState(qbittorrent.Torrent{ + State: qbittorrent.TorrentState("downloading"), + Progress: 0.5, + AmountLeft: 50, + TotalSize: 100, + }); got != downloader.TaskStateDownloading { + t.Fatalf("expected partial torrent to be downloading, got %s", got) + } + if got := qbittorrentTaskState(qbittorrent.Torrent{ + State: qbittorrent.TorrentState("missingFiles"), + Progress: 0.5, + AmountLeft: 50, + TotalSize: 100, + }); got != downloader.TaskStateFailed { + t.Fatalf("expected missing files torrent to be failed, got %s", got) + } +} + +func TestQBittorrentPhaseVariants(t *testing.T) { + tests := map[string]string{ + "metaDL": "metadata", + "stalledUP": "seeding", + "uploading": "seeding", + "missingFiles": "error", + "error": "error", + "pausedDL": "downloading", + "downloading": "downloading", + } + for state, want := range tests { + if got := qbittorrentPhase(state); got != want { + t.Fatalf("expected phase %q for %q, got %q", want, state, got) + } + } +} + +func TestQBittorrentTrackersMapsTorrentTrackersAndLimits(t *testing.T) { + trackers := make([]qbittorrent.TorrentTracker, 0, 25) + for i := 0; i < 25; i++ { + trackers = append(trackers, qbittorrent.TorrentTracker{ + Url: fmt.Sprintf("udp://tracker-%02d/announce", i), + Status: qbittorrent.TrackerStatus(i), + NumPeers: i + 1, + NumSeeds: i + 2, + NumLeechers: i + 3, + Message: "ok", + }) + } + out := qbittorrentTrackers(context.Background(), nil, qbittorrent.Torrent{Trackers: trackers}) + if len(out) != 20 { + t.Fatalf("expected tracker limit 20, got %d", len(out)) + } + if out[0].URL != "udp://tracker-00/announce" || out[0].Status != "0" || *out[0].Peers != 1 || *out[0].Seeds != 2 || *out[0].Leechers != 3 { + t.Fatalf("unexpected first tracker: %#v", out[0]) + } +} + +func TestQBittorrentTagHelpers(t *testing.T) { + if !torrentHasTag("one, ztid=task-1 ,two", qbittorrentTrackingTag("task-1")) { + t.Fatal("expected tracking tag match") + } + if torrentHasTag("one,two", qbittorrentTrackingTag("task-1")) { + t.Fatal("did not expect missing tracking tag match") + } + if !isQBittorrentErrorState(qbittorrent.TorrentState("missingFiles")) { + t.Fatal("expected missingFiles to be an error state") + } + if isQBittorrentErrorState(qbittorrent.TorrentState("downloading")) { + t.Fatal("did not expect downloading to be an error state") + } +} + +func TestQBittorrentDetailOmitsSeedingETA(t *testing.T) { + detail := qbittorrentDetail(context.Background(), nil, qbittorrent.Torrent{ + State: qbittorrent.TorrentState("stalledUP"), + ETA: 3600, + Progress: 1, + AmountLeft: 0, + TotalSize: 100, + }, nil) + + if detail.Phase != "seeding" { + t.Fatalf("expected seeding phase, got %s", detail.Phase) + } + if detail.ETASeconds != nil { + t.Fatalf("expected seeding detail without ETA, got %#v", detail.ETASeconds) + } +} diff --git a/cmd/internal/engine/geoip.go b/cmd/pkg/geoip/geoip.go similarity index 70% rename from cmd/internal/engine/geoip.go rename to cmd/pkg/geoip/geoip.go index 2f136061..20ea094d 100644 --- a/cmd/internal/engine/geoip.go +++ b/cmd/pkg/geoip/geoip.go @@ -1,4 +1,4 @@ -package engine +package geoip import ( "errors" @@ -8,20 +8,19 @@ import ( "strings" "github.com/oschwald/geoip2-golang" - "github.com/saltbo/zpan/internal/client" ) var geoIPCodePattern = regexp.MustCompile(`^[A-Z0-9-]+$`) -type PeerGeoIPResolver interface { +type Resolver interface { LookupPeerRegion(ip string) (countryCode string, regionCode string) } -type GeoIPResolver struct { +type DB struct { db *geoip2.Reader } -func OpenGeoIPResolver(path string) (*GeoIPResolver, error) { +func Open(path string) (*DB, error) { if path == "" { return nil, nil } @@ -32,17 +31,17 @@ func OpenGeoIPResolver(path string) (*GeoIPResolver, error) { } return nil, err } - return &GeoIPResolver{db: db}, nil + return &DB{db: db}, nil } -func (r *GeoIPResolver) Close() error { +func (r *DB) Close() error { if r == nil || r.db == nil { return nil } return r.db.Close() } -func (r *GeoIPResolver) LookupPeerRegion(ip string) (string, string) { +func (r *DB) LookupPeerRegion(ip string) (string, string) { if r == nil || r.db == nil { return "", "" } @@ -65,10 +64,7 @@ func (r *GeoIPResolver) LookupPeerRegion(ip string) (string, string) { return country, region } -func applyPeerRegion(peer *client.DownloadTaskPeer, ip string, fallbackCountryCode string, geoIP PeerGeoIPResolver) { - if peer == nil { - return - } +func NormalizeRegion(ip string, fallbackCountryCode string, geoIP Resolver) (string, string) { country, region := "", "" if geoIP != nil { country, region = geoIP.LookupPeerRegion(ip) @@ -76,8 +72,7 @@ func applyPeerRegion(peer *client.DownloadTaskPeer, ip string, fallbackCountryCo if country == "" { country = fallbackCountryCode } - peer.CountryCode = normalizeGeoIPCode(country, 2) - peer.RegionCode = normalizeGeoIPCode(region, 16) + return normalizeGeoIPCode(country, 2), normalizeGeoIPCode(region, 16) } func normalizeGeoIPCode(value string, maxLen int) string { diff --git a/cmd/pkg/geoip/geoip_test.go b/cmd/pkg/geoip/geoip_test.go new file mode 100644 index 00000000..153d401a --- /dev/null +++ b/cmd/pkg/geoip/geoip_test.go @@ -0,0 +1,72 @@ +package geoip + +import ( + "os" + "path/filepath" + "testing" +) + +type fakeResolver struct{} + +func (fakeResolver) LookupPeerRegion(ip string) (string, string) { + if ip == "203.0.113.10" { + return "us", "ca" + } + return "", "" +} + +func TestNormalizeRegionUsesGeoIPAndFallbackCountry(t *testing.T) { + country, region := NormalizeRegion("203.0.113.10", "", fakeResolver{}) + if country != "US" || region != "CA" { + t.Fatalf("expected US/CA from geoip, got %s/%s", country, region) + } + + country, region = NormalizeRegion("198.51.100.10", "jp", fakeResolver{}) + if country != "JP" || region != "" { + t.Fatalf("expected JP fallback country, got %s/%s", country, region) + } +} + +func TestOpenHandlesEmptyMissingAndInvalidDatabases(t *testing.T) { + db, err := Open("") + if err != nil { + t.Fatal(err) + } + if db != nil { + t.Fatalf("expected nil db for empty path, got %#v", db) + } + + db, err = Open(filepath.Join(t.TempDir(), "missing.mmdb")) + if err != nil { + t.Fatal(err) + } + if db != nil { + t.Fatalf("expected nil db for missing path, got %#v", db) + } + + path := filepath.Join(t.TempDir(), "invalid.mmdb") + if err := os.WriteFile(path, []byte("not a maxmind database"), 0o600); err != nil { + t.Fatal(err) + } + if _, err := Open(path); err == nil { + t.Fatal("expected invalid database error") + } +} + +func TestDBCloseAndLookupEmptyDatabase(t *testing.T) { + if err := ((*DB)(nil)).Close(); err != nil { + t.Fatal(err) + } + db := &DB{} + if err := db.Close(); err != nil { + t.Fatal(err) + } + country, region := db.LookupPeerRegion("203.0.113.10") + if country != "" || region != "" { + t.Fatalf("expected empty region for nil reader, got %s/%s", country, region) + } + country, region = ((*DB)(nil)).LookupPeerRegion("203.0.113.10") + if country != "" || region != "" { + t.Fatalf("expected empty region for nil DB, got %s/%s", country, region) + } +} diff --git a/cmd/internal/worker/disk_unix.go b/cmd/pkg/system/disk_unix.go similarity index 91% rename from cmd/internal/worker/disk_unix.go rename to cmd/pkg/system/disk_unix.go index ff263bd0..457ea442 100644 --- a/cmd/internal/worker/disk_unix.go +++ b/cmd/pkg/system/disk_unix.go @@ -1,6 +1,6 @@ //go:build !windows -package worker +package system import ( "os" @@ -9,7 +9,7 @@ import ( "golang.org/x/sys/unix" ) -func freeDiskBytes(path string) (int64, error) { +func FreeDiskBytes(path string) (int64, error) { statPath, err := existingStatPath(path) if err != nil { return 0, err diff --git a/cmd/pkg/system/disk_unix_test.go b/cmd/pkg/system/disk_unix_test.go new file mode 100644 index 00000000..21958c2c --- /dev/null +++ b/cmd/pkg/system/disk_unix_test.go @@ -0,0 +1,62 @@ +//go:build !windows + +package system + +import ( + "os" + "path/filepath" + "testing" + + "golang.org/x/sys/unix" +) + +func TestFreeDiskBytesReportsPathFreeDiskExactly(t *testing.T) { + downloadDir := t.TempDir() + var stat unix.Statfs_t + if err := unix.Statfs(downloadDir, &stat); err != nil { + t.Fatalf("statfs %s: %v", downloadDir, err) + } + want := int64(stat.Bavail) * int64(stat.Bsize) + + got, err := FreeDiskBytes(downloadDir) + if err != nil { + t.Fatal(err) + } + if got != want { + t.Fatalf("expected free disk %d, got %d", want, got) + } +} + +func TestExistingStatPathFallsBackToExistingParent(t *testing.T) { + root := t.TempDir() + missing := filepath.Join(root, "missing", "child") + got, err := existingStatPath(missing) + if err != nil { + t.Fatal(err) + } + if got != root { + t.Fatalf("expected root fallback %s, got %s", root, got) + } + got, err = existingStatPath("") + if err != nil { + t.Fatal(err) + } + if got != "." { + t.Fatalf("expected current directory fallback, got %s", got) + } +} + +func TestExistingStatPathReturnsStatErrors(t *testing.T) { + if os.Geteuid() == 0 { + t.Skip("root can stat unreadable directories") + } + root := t.TempDir() + locked := filepath.Join(root, "locked") + if err := os.Mkdir(locked, 0o000); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.Chmod(locked, 0o700) }) + if _, err := existingStatPath(filepath.Join(locked, "child")); err == nil { + t.Fatal("expected stat permission error") + } +} diff --git a/cmd/internal/worker/disk_windows.go b/cmd/pkg/system/disk_windows.go similarity index 92% rename from cmd/internal/worker/disk_windows.go rename to cmd/pkg/system/disk_windows.go index 6ac16856..7f3bebce 100644 --- a/cmd/internal/worker/disk_windows.go +++ b/cmd/pkg/system/disk_windows.go @@ -1,6 +1,6 @@ //go:build windows -package worker +package system import ( "os" @@ -9,7 +9,7 @@ import ( "golang.org/x/sys/windows" ) -func freeDiskBytes(path string) (int64, error) { +func FreeDiskBytes(path string) (int64, error) { statPath, err := existingStatPath(path) if err != nil { return 0, err diff --git a/cmd/pkg/system/files.go b/cmd/pkg/system/files.go new file mode 100644 index 00000000..41d435ca --- /dev/null +++ b/cmd/pkg/system/files.go @@ -0,0 +1,51 @@ +package system + +import ( + "os" + "path/filepath" + "strings" +) + +func DirectorySize(path string) (int64, error) { + var total int64 + err := filepath.WalkDir(path, func(entryPath string, entry os.DirEntry, err error) error { + if err != nil { + return err + } + if entryPath == path { + return nil + } + if strings.HasPrefix(entry.Name(), ".") { + if entry.IsDir() { + return filepath.SkipDir + } + return nil + } + if entry.IsDir() { + return nil + } + if IsDownloadSidecarPath(entry.Name()) { + return nil + } + info, err := entry.Info() + if err != nil { + return err + } + total += info.Size() + return nil + }) + return total, err +} + +func IsDownloadSidecarPath(path string) bool { + base := filepath.Base(path) + ext := filepath.Ext(base) + return IsAria2MetadataPath(path) || + IsAria2MetadataPath(base) || + strings.EqualFold(ext, ".torrent") || + strings.EqualFold(ext, ".aria2") +} + +func IsAria2MetadataPath(path string) bool { + return strings.HasPrefix(path, "[MEMORY]") || strings.HasPrefix(path, "[METADATA]") +} diff --git a/cmd/pkg/system/files_test.go b/cmd/pkg/system/files_test.go new file mode 100644 index 00000000..5fdde015 --- /dev/null +++ b/cmd/pkg/system/files_test.go @@ -0,0 +1,57 @@ +package system + +import ( + "os" + "path/filepath" + "testing" +) + +func TestDirectorySizeSkipsHiddenAndSidecarFiles(t *testing.T) { + root := t.TempDir() + if err := os.WriteFile(filepath.Join(root, "movie.mkv"), []byte("movie"), 0o644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(root, "movie.mkv.aria2"), []byte("sidecar"), 0o644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(root, "payload.torrent"), []byte("torrent"), 0o644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(root, ".hidden"), []byte("hidden"), 0o644); err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(filepath.Join(root, ".hidden-dir"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(root, ".hidden-dir", "ignored.bin"), []byte("ignored"), 0o644); err != nil { + t.Fatal(err) + } + + size, err := DirectorySize(root) + if err != nil { + t.Fatal(err) + } + if size != int64(len("movie")) { + t.Fatalf("expected visible payload size, got %d", size) + } +} + +func TestDownloadSidecarDetection(t *testing.T) { + cases := map[string]bool{ + "payload.torrent": true, + "payload.TORRENT": true, + "payload.aria2": true, + "[MEMORY]abc": true, + "dir/[METADATA]abc": true, + "payload.mkv": false, + "payload.torrent/file": false, + } + for path, want := range cases { + if got := IsDownloadSidecarPath(path); got != want { + t.Fatalf("IsDownloadSidecarPath(%q)=%v, want %v", path, got, want) + } + } + if !IsAria2MetadataPath("[METADATA]abc") || IsAria2MetadataPath("abc[METADATA]") { + t.Fatal("unexpected aria2 metadata detection") + } +} diff --git a/cmd/internal/host/hostname.go b/cmd/pkg/system/hostname.go similarity index 96% rename from cmd/internal/host/hostname.go rename to cmd/pkg/system/hostname.go index ac88ed97..591832ca 100644 --- a/cmd/internal/host/hostname.go +++ b/cmd/pkg/system/hostname.go @@ -1,4 +1,4 @@ -package host +package system import ( "os" diff --git a/cmd/internal/host/hostname_test.go b/cmd/pkg/system/hostname_test.go similarity index 97% rename from cmd/internal/host/hostname_test.go rename to cmd/pkg/system/hostname_test.go index f0476c95..ab5bb835 100644 --- a/cmd/internal/host/hostname_test.go +++ b/cmd/pkg/system/hostname_test.go @@ -1,4 +1,4 @@ -package host +package system import ( "os" diff --git a/cmd/internal/engine/process.go b/cmd/pkg/system/process.go similarity index 66% rename from cmd/internal/engine/process.go rename to cmd/pkg/system/process.go index 0bfae4b0..1bf5fe98 100644 --- a/cmd/internal/engine/process.go +++ b/cmd/pkg/system/process.go @@ -1,7 +1,6 @@ -package engine +package system import ( - "context" "fmt" "net/url" "os/exec" @@ -9,15 +8,11 @@ import ( "strings" ) -type Starter interface { - Start(ctx context.Context) (*exec.Cmd, error) +type LocalEngineURL struct { + Port string } -type localEngineURL struct { - port string -} - -func lookPathAny(names ...string) (string, error) { +func LookPathAny(names ...string) (string, error) { var lastErr error for _, name := range names { path, err := exec.LookPath(name) @@ -29,7 +24,7 @@ func lookPathAny(names ...string) (string, error) { return "", lastErr } -func parseLocalEngineURL(raw string, defaultPort string) (localEngineURL, error) { +func ParseLocalEngineURL(raw string, defaultPort string) (LocalEngineURL, error) { normalized := raw if strings.HasPrefix(normalized, "ws://") { normalized = "http://" + strings.TrimPrefix(normalized, "ws://") @@ -39,23 +34,23 @@ func parseLocalEngineURL(raw string, defaultPort string) (localEngineURL, error) } parsed, err := url.Parse(normalized) if err != nil { - return localEngineURL{}, err + return LocalEngineURL{}, err } host := parsed.Hostname() if host != "" && host != "127.0.0.1" && host != "localhost" && host != "::1" { - return localEngineURL{}, fmt.Errorf("auto start only supports local engine URLs, got %s", host) + return LocalEngineURL{}, fmt.Errorf("auto start only supports local engine URLs, got %s", host) } port := parsed.Port() if port == "" { port = defaultPort } if _, err := strconv.Atoi(port); err != nil { - return localEngineURL{}, fmt.Errorf("invalid engine port %q", port) + return LocalEngineURL{}, fmt.Errorf("invalid engine port %q", port) } - return localEngineURL{port: port}, nil + return LocalEngineURL{Port: port}, nil } -func filepathBase(path string) string { +func FilepathBase(path string) string { parts := strings.FieldsFunc(path, func(r rune) bool { return r == '/' || r == '\\' }) if len(parts) == 0 { return path diff --git a/cmd/pkg/system/process_test.go b/cmd/pkg/system/process_test.go new file mode 100644 index 00000000..84839dd2 --- /dev/null +++ b/cmd/pkg/system/process_test.go @@ -0,0 +1,63 @@ +package system + +import ( + "os/exec" + "path/filepath" + "testing" +) + +func TestParseLocalEngineURL(t *testing.T) { + cases := []struct { + raw string + port string + }{ + {"", "6800"}, + {"http://127.0.0.1:6888/jsonrpc", "6888"}, + {"http://localhost", "6800"}, + {"ws://127.0.0.1:6801/jsonrpc", "6801"}, + {"wss://[::1]:6802/jsonrpc", "6802"}, + } + for _, tc := range cases { + got, err := ParseLocalEngineURL(tc.raw, "6800") + if err != nil { + t.Fatalf("ParseLocalEngineURL(%q): %v", tc.raw, err) + } + if got.Port != tc.port { + t.Fatalf("ParseLocalEngineURL(%q) port=%q, want %q", tc.raw, got.Port, tc.port) + } + } +} + +func TestParseLocalEngineURLRejectsRemoteAndInvalidPorts(t *testing.T) { + if _, err := ParseLocalEngineURL("http://example.com:6800", "6800"); err == nil { + t.Fatal("expected remote host rejection") + } + if _, err := ParseLocalEngineURL("http://127.0.0.1:bad", "6800"); err == nil { + t.Fatal("expected invalid port error") + } + if _, err := ParseLocalEngineURL("http://127.0.0.1", "bad"); err == nil { + t.Fatal("expected invalid default port error") + } +} + +func TestLookPathAnyAndFilepathBase(t *testing.T) { + path, err := LookPathAny("definitely-not-a-zpan-test-binary", "go") + if err != nil { + t.Fatal(err) + } + if filepath.Base(path) != "go" { + t.Fatalf("expected go binary, got %q", path) + } + if _, err := LookPathAny("definitely-not-a-zpan-test-binary"); err == nil { + t.Fatal("expected lookup error") + } + if FilepathBase(`C:\Program Files\qBittorrent\qbittorrent.exe`) != "qbittorrent.exe" { + t.Fatal("expected windows basename") + } + if FilepathBase("/usr/bin/aria2c") != "aria2c" { + t.Fatal("expected unix basename") + } + if _, err := exec.LookPath("go"); err != nil { + t.Fatal(err) + } +} diff --git a/cmd/internal/engine/process_unix.go b/cmd/pkg/system/process_unix.go similarity index 65% rename from cmd/internal/engine/process_unix.go rename to cmd/pkg/system/process_unix.go index c5b81c45..8f55f2b1 100644 --- a/cmd/internal/engine/process_unix.go +++ b/cmd/pkg/system/process_unix.go @@ -1,12 +1,12 @@ //go:build !windows -package engine +package system import ( "os/exec" "syscall" ) -func configureEngineProcess(cmd *exec.Cmd) { +func ConfigureProcess(cmd *exec.Cmd) { cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} } diff --git a/cmd/pkg/system/process_unix_test.go b/cmd/pkg/system/process_unix_test.go new file mode 100644 index 00000000..af3db957 --- /dev/null +++ b/cmd/pkg/system/process_unix_test.go @@ -0,0 +1,16 @@ +//go:build !windows + +package system + +import ( + "os/exec" + "testing" +) + +func TestConfigureProcessSetsProcessGroup(t *testing.T) { + cmd := exec.Command("go", "version") + ConfigureProcess(cmd) + if cmd.SysProcAttr == nil || !cmd.SysProcAttr.Setpgid { + t.Fatalf("expected process group configuration, got %#v", cmd.SysProcAttr) + } +} diff --git a/cmd/pkg/system/process_windows.go b/cmd/pkg/system/process_windows.go new file mode 100644 index 00000000..bc2571fc --- /dev/null +++ b/cmd/pkg/system/process_windows.go @@ -0,0 +1,7 @@ +//go:build windows + +package system + +import "os/exec" + +func ConfigureProcess(cmd *exec.Cmd) {} diff --git a/cmd/scripts/test-coverage.sh b/cmd/scripts/test-coverage.sh new file mode 100755 index 00000000..6a2c9783 --- /dev/null +++ b/cmd/scripts/test-coverage.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +set -euo pipefail + +threshold="${CMD_COVERAGE_MIN:-70}" +profile="$(mktemp "${TMPDIR:-/tmp}/zpan-cmd-coverage.XXXXXX")" +filtered="$(mktemp "${TMPDIR:-/tmp}/zpan-cmd-coverage-filtered.XXXXXX")" +trap 'rm -f "$profile" "$filtered"' EXIT + +go test ./... -coverprofile="$profile" +grep -v 'internal/openapi/client\.gen\.go' "$profile" >"$filtered" + +coverage="$(go tool cover -func="$filtered" | awk '/^total:/ { sub(/%$/, "", $3); print $3 }')" +if [ -z "$coverage" ]; then + echo "failed to read cmd coverage total" >&2 + exit 1 +fi + +awk -v coverage="$coverage" -v threshold="$threshold" 'BEGIN { + if (coverage + 0 < threshold + 0) { + printf("cmd coverage %.1f%% is below %.1f%%\n", coverage, threshold) > "/dev/stderr" + exit 1 + } + printf("cmd coverage %.1f%% meets %.1f%% threshold\n", coverage, threshold) +}' diff --git a/migrations/0052_rename-downloader-http-engine.sql b/migrations/0052_rename-downloader-http-engine.sql new file mode 100644 index 00000000..5096031a --- /dev/null +++ b/migrations/0052_rename-downloader-http-engine.sql @@ -0,0 +1,36 @@ +PRAGMA foreign_keys=OFF;--> statement-breakpoint +CREATE TABLE `__new_downloaders` ( + `id` text PRIMARY KEY NOT NULL, + `name` text NOT NULL, + `token_hash` text NOT NULL, + `token_jti` text NOT NULL, + `status` text DEFAULT 'offline' NOT NULL, + `enabled` integer DEFAULT true NOT NULL, + `version` text DEFAULT 'unknown' NOT NULL, + `hostname` text DEFAULT 'unknown' NOT NULL, + `platform` text DEFAULT 'unknown' NOT NULL, + `arch` text DEFAULT 'unknown' NOT NULL, + `engine` text DEFAULT 'http' NOT NULL, + `capabilities` text DEFAULT '[]' NOT NULL, + `max_concurrent_tasks` integer DEFAULT 1 NOT NULL, + `current_tasks` integer DEFAULT 0 NOT NULL, + `download_bps` integer DEFAULT 0 NOT NULL, + `upload_bps` integer DEFAULT 0 NOT NULL, + `free_disk_bytes` integer DEFAULT 0 NOT NULL, + `remote_download_credit_billing_enabled` integer DEFAULT false NOT NULL, + `remote_download_credit_unit_bytes` integer DEFAULT 104857600 NOT NULL, + `remote_download_credit_per_unit` integer DEFAULT 1 NOT NULL, + `last_heartbeat_at` integer, + `created_by` text NOT NULL, + `created_at` integer NOT NULL, + `updated_at` integer NOT NULL +); +--> statement-breakpoint +INSERT INTO `__new_downloaders`("id", "name", "token_hash", "token_jti", "status", "enabled", "version", "hostname", "platform", "arch", "engine", "capabilities", "max_concurrent_tasks", "current_tasks", "download_bps", "upload_bps", "free_disk_bytes", "remote_download_credit_billing_enabled", "remote_download_credit_unit_bytes", "remote_download_credit_per_unit", "last_heartbeat_at", "created_by", "created_at", "updated_at") SELECT "id", "name", "token_hash", "token_jti", "status", "enabled", "version", "hostname", "platform", "arch", "engine", "capabilities", "max_concurrent_tasks", "current_tasks", "download_bps", "upload_bps", "free_disk_bytes", "remote_download_credit_billing_enabled", "remote_download_credit_unit_bytes", "remote_download_credit_per_unit", "last_heartbeat_at", "created_by", "created_at", "updated_at" FROM `downloaders`;--> statement-breakpoint +DROP TABLE `downloaders`;--> statement-breakpoint +ALTER TABLE `__new_downloaders` RENAME TO `downloaders`;--> statement-breakpoint +PRAGMA foreign_keys=ON;--> statement-breakpoint +CREATE UNIQUE INDEX `downloaders_token_jti_unique` ON `downloaders` (`token_jti`);--> statement-breakpoint +CREATE INDEX `downloaders_status_idx` ON `downloaders` (`status`);--> statement-breakpoint +CREATE INDEX `downloaders_enabled_idx` ON `downloaders` (`enabled`);--> statement-breakpoint +CREATE INDEX `downloaders_created_idx` ON `downloaders` (`created_at`); \ No newline at end of file diff --git a/migrations/meta/0052_snapshot.json b/migrations/meta/0052_snapshot.json new file mode 100644 index 00000000..0ba37d2e --- /dev/null +++ b/migrations/meta/0052_snapshot.json @@ -0,0 +1,3918 @@ +{ + "version": "6", + "dialect": "sqlite", + "id": "fcd723fa-bb9f-4049-b9ed-6ac894da0b5e", + "prevId": "42a70d70-74f3-4d1a-8276-474cd350a695", + "tables": { + "activity_events": { + "name": "activity_events", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "action": { + "name": "action", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "target_type": { + "name": "target_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "target_id": { + "name": "target_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "target_name": { + "name": "target_name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "metadata": { + "name": "metadata", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "announcements": { + "name": "announcements", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "body": { + "name": "body", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "''" + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'draft'" + }, + "priority": { + "name": "priority", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "published_at": { + "name": "published_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "announcements_status_priority_idx": { + "name": "announcements_status_priority_idx", + "columns": [ + "status", + "priority" + ], + "isUnique": false + }, + "announcements_published_idx": { + "name": "announcements_published_idx", + "columns": [ + "published_at" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "background_jobs": { + "name": "background_jobs", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "target_folder": { + "name": "target_folder", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "target_path": { + "name": "target_path", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "metadata": { + "name": "metadata", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "input_bytes": { + "name": "input_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "output_bytes": { + "name": "output_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "processed_bytes": { + "name": "processed_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "file_count": { + "name": "file_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "current_filename": { + "name": "current_filename", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "error_message": { + "name": "error_message", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "result_metadata": { + "name": "result_metadata", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "retryable": { + "name": "retryable", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "cancelable": { + "name": "cancelable", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": true + }, + "retried_from_job_id": { + "name": "retried_from_job_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "started_at": { + "name": "started_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "finished_at": { + "name": "finished_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "background_jobs_org_created_idx": { + "name": "background_jobs_org_created_idx", + "columns": [ + "org_id", + "created_at" + ], + "isUnique": false + }, + "background_jobs_org_status_idx": { + "name": "background_jobs_org_status_idx", + "columns": [ + "org_id", + "status" + ], + "isUnique": false + }, + "background_jobs_org_type_idx": { + "name": "background_jobs_org_type_idx", + "columns": [ + "org_id", + "type" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "cloud_traffic_reports": { + "name": "cloud_traffic_reports", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "period": { + "name": "period", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "source": { + "name": "source", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "source_id": { + "name": "source_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "event_id": { + "name": "event_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bytes": { + "name": "bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "storage_id": { + "name": "storage_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "unit_bytes": { + "name": "unit_bytes", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "credits_per_unit": { + "name": "credits_per_unit", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "cloud_traffic_reports_event_uniq": { + "name": "cloud_traffic_reports_event_uniq", + "columns": [ + "event_id" + ], + "isUnique": true + }, + "cloud_traffic_reports_org_period_idx": { + "name": "cloud_traffic_reports_org_period_idx", + "columns": [ + "org_id", + "period" + ], + "isUnique": false + }, + "cloud_traffic_reports_status_idx": { + "name": "cloud_traffic_reports_status_idx", + "columns": [ + "status" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "download_tasks": { + "name": "download_tasks", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_by_user_id": { + "name": "created_by_user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "source_type": { + "name": "source_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "source_uri": { + "name": "source_uri", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "display_name": { + "name": "display_name", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "target_folder": { + "name": "target_folder", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "''" + }, + "category": { + "name": "category", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "tags": { + "name": "tags", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'[]'" + }, + "assigned_downloader_id": { + "name": "assigned_downloader_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "attempt": { + "name": "attempt", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 1 + }, + "billing_authorized_bytes": { + "name": "billing_authorized_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "billing_charged_bytes": { + "name": "billing_charged_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "billing_charged_credits": { + "name": "billing_charged_credits", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "billing_status": { + "name": "billing_status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'none'" + }, + "error_code": { + "name": "error_code", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "error_message": { + "name": "error_message", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "result_object_id": { + "name": "result_object_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "runtime": { + "name": "runtime", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "assigned_at": { + "name": "assigned_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "started_at": { + "name": "started_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "finished_at": { + "name": "finished_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "download_tasks_org_created_idx": { + "name": "download_tasks_org_created_idx", + "columns": [ + "org_id", + "created_at" + ], + "isUnique": false + }, + "download_tasks_org_status_idx": { + "name": "download_tasks_org_status_idx", + "columns": [ + "org_id", + "status" + ], + "isUnique": false + }, + "download_tasks_org_category_idx": { + "name": "download_tasks_org_category_idx", + "columns": [ + "org_id", + "category" + ], + "isUnique": false + }, + "download_tasks_org_tags_idx": { + "name": "download_tasks_org_tags_idx", + "columns": [ + "org_id", + "tags" + ], + "isUnique": false + }, + "download_tasks_downloader_idx": { + "name": "download_tasks_downloader_idx", + "columns": [ + "assigned_downloader_id", + "status" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "downloaders": { + "name": "downloaders", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "token_hash": { + "name": "token_hash", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "token_jti": { + "name": "token_jti", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'offline'" + }, + "enabled": { + "name": "enabled", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": true + }, + "version": { + "name": "version", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'unknown'" + }, + "hostname": { + "name": "hostname", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'unknown'" + }, + "platform": { + "name": "platform", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'unknown'" + }, + "arch": { + "name": "arch", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'unknown'" + }, + "engine": { + "name": "engine", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'http'" + }, + "capabilities": { + "name": "capabilities", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'[]'" + }, + "max_concurrent_tasks": { + "name": "max_concurrent_tasks", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 1 + }, + "current_tasks": { + "name": "current_tasks", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "download_bps": { + "name": "download_bps", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "upload_bps": { + "name": "upload_bps", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "free_disk_bytes": { + "name": "free_disk_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "remote_download_credit_billing_enabled": { + "name": "remote_download_credit_billing_enabled", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "remote_download_credit_unit_bytes": { + "name": "remote_download_credit_unit_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 104857600 + }, + "remote_download_credit_per_unit": { + "name": "remote_download_credit_per_unit", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 1 + }, + "last_heartbeat_at": { + "name": "last_heartbeat_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "downloaders_token_jti_unique": { + "name": "downloaders_token_jti_unique", + "columns": [ + "token_jti" + ], + "isUnique": true + }, + "downloaders_status_idx": { + "name": "downloaders_status_idx", + "columns": [ + "status" + ], + "isUnique": false + }, + "downloaders_enabled_idx": { + "name": "downloaders_enabled_idx", + "columns": [ + "enabled" + ], + "isUnique": false + }, + "downloaders_created_idx": { + "name": "downloaders_created_idx", + "columns": [ + "created_at" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "image_hosting_configs": { + "name": "image_hosting_configs", + "columns": { + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "custom_domain": { + "name": "custom_domain", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "cf_hostname_id": { + "name": "cf_hostname_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "domain_verified_at": { + "name": "domain_verified_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "referer_allowlist": { + "name": "referer_allowlist", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "image_hosting_configs_custom_domain_unique": { + "name": "image_hosting_configs_custom_domain_unique", + "columns": [ + "custom_domain" + ], + "isUnique": true + } + }, + "foreignKeys": { + "image_hosting_configs_org_id_organization_id_fk": { + "name": "image_hosting_configs_org_id_organization_id_fk", + "tableFrom": "image_hosting_configs", + "tableTo": "organization", + "columnsFrom": [ + "org_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "image_hostings": { + "name": "image_hostings", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "token": { + "name": "token", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "path": { + "name": "path", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "storage_id": { + "name": "storage_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "storage_key": { + "name": "storage_key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "size": { + "name": "size", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "mime": { + "name": "mime", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "width": { + "name": "width", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "height": { + "name": "height", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'draft'" + }, + "access_count": { + "name": "access_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "last_accessed_at": { + "name": "last_accessed_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "image_hostings_token_unique": { + "name": "image_hostings_token_unique", + "columns": [ + "token" + ], + "isUnique": true + }, + "image_hostings_org_path_uniq": { + "name": "image_hostings_org_path_uniq", + "columns": [ + "org_id", + "path" + ], + "isUnique": true + }, + "image_hostings_org_created_idx": { + "name": "image_hostings_org_created_idx", + "columns": [ + "org_id", + "created_at" + ], + "isUnique": false + }, + "image_hostings_token_idx": { + "name": "image_hostings_token_idx", + "columns": [ + "token" + ], + "isUnique": false + } + }, + "foreignKeys": { + "image_hostings_org_id_organization_id_fk": { + "name": "image_hostings_org_id_organization_id_fk", + "tableFrom": "image_hostings", + "tableTo": "organization", + "columnsFrom": [ + "org_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "image_hostings_storage_id_storages_id_fk": { + "name": "image_hostings_storage_id_storages_id_fk", + "tableFrom": "image_hostings", + "tableTo": "storages", + "columnsFrom": [ + "storage_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "invite_codes": { + "name": "invite_codes", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "code": { + "name": "code", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "used_by": { + "name": "used_by", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "used_at": { + "name": "used_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "invite_codes_code_unique": { + "name": "invite_codes_code_unique", + "columns": [ + "code" + ], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "license_bindings": { + "name": "license_bindings", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "cloud_binding_id": { + "name": "cloud_binding_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "cloud_store_id": { + "name": "cloud_store_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "instance_id": { + "name": "instance_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "cloud_account_id": { + "name": "cloud_account_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "cloud_account_email": { + "name": "cloud_account_email", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "refresh_token": { + "name": "refresh_token", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "cached_certificate": { + "name": "cached_certificate", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "cached_certificate_expires_at": { + "name": "cached_certificate_expires_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "bound_at": { + "name": "bound_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "disconnected_at": { + "name": "disconnected_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_refresh_at": { + "name": "last_refresh_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_refresh_error": { + "name": "last_refresh_error", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "license_bindings_active_uniq": { + "name": "license_bindings_active_uniq", + "columns": [ + "status" + ], + "isUnique": true, + "where": "status = 'active'" + }, + "license_bindings_cloud_binding_idx": { + "name": "license_bindings_cloud_binding_idx", + "columns": [ + "cloud_binding_id" + ], + "isUnique": false + }, + "license_bindings_instance_idx": { + "name": "license_bindings_instance_idx", + "columns": [ + "instance_id" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "matters": { + "name": "matters", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "alias": { + "name": "alias", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "size": { + "name": "size", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": 0 + }, + "dirtype": { + "name": "dirtype", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": 0 + }, + "parent": { + "name": "parent", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "''" + }, + "object": { + "name": "object", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "''" + }, + "storage_id": { + "name": "storage_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'draft'" + }, + "trashed_at": { + "name": "trashed_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "matters_alias_unique": { + "name": "matters_alias_unique", + "columns": [ + "alias" + ], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "notifications": { + "name": "notifications", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "body": { + "name": "body", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "''" + }, + "ref_type": { + "name": "ref_type", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "ref_id": { + "name": "ref_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "metadata": { + "name": "metadata", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "read_at": { + "name": "read_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "notifications_user_created_idx": { + "name": "notifications_user_created_idx", + "columns": [ + "user_id", + "created_at" + ], + "isUnique": false + }, + "notifications_user_read_idx": { + "name": "notifications_user_read_idx", + "columns": [ + "user_id", + "read_at" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "object_upload_sessions": { + "name": "object_upload_sessions", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "object_id": { + "name": "object_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "storage_id": { + "name": "storage_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "storage_key": { + "name": "storage_key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "upload_id": { + "name": "upload_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "part_size": { + "name": "part_size", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "on_conflict": { + "name": "on_conflict", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'fail'" + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "object_upload_sessions_object_idx": { + "name": "object_upload_sessions_object_idx", + "columns": [ + "org_id", + "object_id" + ], + "isUnique": false + }, + "object_upload_sessions_expires_idx": { + "name": "object_upload_sessions_expires_idx", + "columns": [ + "expires_at" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "org_quota_entitlements": { + "name": "org_quota_entitlements", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "resource_type": { + "name": "resource_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "entitlement_type": { + "name": "entitlement_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'grant'" + }, + "source": { + "name": "source", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "source_id": { + "name": "source_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bytes": { + "name": "bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "starts_at": { + "name": "starts_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "metadata": { + "name": "metadata", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "org_quota_entitlements_org_resource_idx": { + "name": "org_quota_entitlements_org_resource_idx", + "columns": [ + "org_id", + "resource_type", + "status" + ], + "isUnique": false + }, + "org_quota_entitlements_org_type_idx": { + "name": "org_quota_entitlements_org_type_idx", + "columns": [ + "org_id", + "resource_type", + "entitlement_type", + "status" + ], + "isUnique": false + }, + "org_quota_entitlements_active_plan_uniq": { + "name": "org_quota_entitlements_active_plan_uniq", + "columns": [ + "org_id", + "resource_type", + "entitlement_type" + ], + "isUnique": true, + "where": "status = 'active' AND entitlement_type = 'plan'" + }, + "org_quota_entitlements_source_resource_uniq": { + "name": "org_quota_entitlements_source_resource_uniq", + "columns": [ + "source", + "source_id", + "resource_type" + ], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "org_quotas": { + "name": "org_quotas", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "quota": { + "name": "quota", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "used": { + "name": "used", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "traffic_quota": { + "name": "traffic_quota", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "traffic_used": { + "name": "traffic_used", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "traffic_period": { + "name": "traffic_period", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'1970-01'" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "remote_download_usage_reports": { + "name": "remote_download_usage_reports", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "downloader_id": { + "name": "downloader_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "task_id": { + "name": "task_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "event_id": { + "name": "event_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "unit_index": { + "name": "unit_index", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "unit_bytes": { + "name": "unit_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "credits_per_unit": { + "name": "credits_per_unit", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "remote_download_usage_reports_event_id_unique": { + "name": "remote_download_usage_reports_event_id_unique", + "columns": [ + "event_id" + ], + "isUnique": true + }, + "remote_download_usage_task_unit_uniq": { + "name": "remote_download_usage_task_unit_uniq", + "columns": [ + "task_id", + "unit_index" + ], + "isUnique": true + }, + "remote_download_usage_org_idx": { + "name": "remote_download_usage_org_idx", + "columns": [ + "org_id" + ], + "isUnique": false + }, + "remote_download_usage_status_idx": { + "name": "remote_download_usage_status_idx", + "columns": [ + "status" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "share_recipients": { + "name": "share_recipients", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "share_id": { + "name": "share_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "recipient_user_id": { + "name": "recipient_user_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "recipient_email": { + "name": "recipient_email", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "share_recipients_share_id_idx": { + "name": "share_recipients_share_id_idx", + "columns": [ + "share_id" + ], + "isUnique": false + }, + "share_recipients_user_id_idx": { + "name": "share_recipients_user_id_idx", + "columns": [ + "recipient_user_id" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "shares": { + "name": "shares", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "token": { + "name": "token", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "matter_id": { + "name": "matter_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "creator_id": { + "name": "creator_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "password_hash": { + "name": "password_hash", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "download_limit": { + "name": "download_limit", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "views": { + "name": "views", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "downloads": { + "name": "downloads", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'active'" + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "shares_token_unique": { + "name": "shares_token_unique", + "columns": [ + "token" + ], + "isUnique": true + }, + "shares_creator_status_created_idx": { + "name": "shares_creator_status_created_idx", + "columns": [ + "creator_id", + "status", + "created_at" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "site_invitations": { + "name": "site_invitations", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "token": { + "name": "token", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "invited_by": { + "name": "invited_by", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "accepted_by": { + "name": "accepted_by", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "accepted_at": { + "name": "accepted_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "revoked_by": { + "name": "revoked_by", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "revoked_at": { + "name": "revoked_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "site_invitations_token_unique": { + "name": "site_invitations_token_unique", + "columns": [ + "token" + ], + "isUnique": true + }, + "site_invitations_email_idx": { + "name": "site_invitations_email_idx", + "columns": [ + "email" + ], + "isUnique": false + }, + "site_invitations_created_idx": { + "name": "site_invitations_created_idx", + "columns": [ + "created_at" + ], + "isUnique": false + }, + "site_invitations_expires_idx": { + "name": "site_invitations_expires_idx", + "columns": [ + "expires_at" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "storages": { + "name": "storages", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "provider": { + "name": "provider", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "''" + }, + "bucket": { + "name": "bucket", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "endpoint": { + "name": "endpoint", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "region": { + "name": "region", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'auto'" + }, + "access_key": { + "name": "access_key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "secret_key": { + "name": "secret_key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "file_path": { + "name": "file_path", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "''" + }, + "custom_host": { + "name": "custom_host", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "''" + }, + "capacity": { + "name": "capacity", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "egress_credit_billing_enabled": { + "name": "egress_credit_billing_enabled", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "egress_credit_unit_bytes": { + "name": "egress_credit_unit_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 104857600 + }, + "egress_credit_per_unit": { + "name": "egress_credit_per_unit", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 1 + }, + "force_path_style": { + "name": "force_path_style", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": true + }, + "used": { + "name": "used", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'active'" + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "system_options": { + "name": "system_options", + "columns": { + "key": { + "name": "key", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "value": { + "name": "value", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "''" + }, + "public": { + "name": "public", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "team_invite_links": { + "name": "team_invite_links", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "token": { + "name": "token", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'member'" + }, + "inviter_id": { + "name": "inviter_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "team_invite_links_token_unique": { + "name": "team_invite_links_token_unique", + "columns": [ + "token" + ], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "webdav_dead_properties": { + "name": "webdav_dead_properties", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "resource_path": { + "name": "resource_path", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "namespace": { + "name": "namespace", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "value": { + "name": "value", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "webdav_dead_properties_resource_prop_uniq": { + "name": "webdav_dead_properties_resource_prop_uniq", + "columns": [ + "org_id", + "resource_path", + "namespace", + "name" + ], + "isUnique": true + }, + "webdav_dead_properties_resource_idx": { + "name": "webdav_dead_properties_resource_idx", + "columns": [ + "org_id", + "resource_path" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "webdav_locks": { + "name": "webdav_locks", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "token": { + "name": "token", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "resource_path": { + "name": "resource_path", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "owner": { + "name": "owner", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "''" + }, + "depth": { + "name": "depth", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'infinity'" + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "webdav_locks_token_unique": { + "name": "webdav_locks_token_unique", + "columns": [ + "token" + ], + "isUnique": true + }, + "webdav_locks_resource_idx": { + "name": "webdav_locks_resource_idx", + "columns": [ + "org_id", + "resource_path" + ], + "isUnique": false + }, + "webdav_locks_expires_idx": { + "name": "webdav_locks_expires_idx", + "columns": [ + "expires_at" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "webhook_events": { + "name": "webhook_events", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "source": { + "name": "source", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'cloud'" + }, + "event_id": { + "name": "event_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "event_type": { + "name": "event_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'order.quota_changed'" + }, + "payload_hash": { + "name": "payload_hash", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "raw_payload": { + "name": "raw_payload", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "processed_at": { + "name": "processed_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "webhook_events_source_event_uniq": { + "name": "webhook_events_source_event_uniq", + "columns": [ + "source", + "event_id" + ], + "isUnique": true + }, + "webhook_events_source_created_idx": { + "name": "webhook_events_source_created_idx", + "columns": [ + "source", + "created_at" + ], + "isUnique": false + }, + "webhook_events_status_idx": { + "name": "webhook_events_status_idx", + "columns": [ + "status" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "account": { + "name": "account", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "account_id": { + "name": "account_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "provider_id": { + "name": "provider_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "access_token": { + "name": "access_token", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "refresh_token": { + "name": "refresh_token", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "id_token": { + "name": "id_token", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "access_token_expires_at": { + "name": "access_token_expires_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "refresh_token_expires_at": { + "name": "refresh_token_expires_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "scope": { + "name": "scope", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "password": { + "name": "password", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "account_userId_idx": { + "name": "account_userId_idx", + "columns": [ + "user_id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "account_user_id_user_id_fk": { + "name": "account_user_id_user_id_fk", + "tableFrom": "account", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "apikey": { + "name": "apikey", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "config_id": { + "name": "config_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'default'" + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "start": { + "name": "start", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "reference_id": { + "name": "reference_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "prefix": { + "name": "prefix", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "key": { + "name": "key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "refill_interval": { + "name": "refill_interval", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "refill_amount": { + "name": "refill_amount", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_refill_at": { + "name": "last_refill_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "enabled": { + "name": "enabled", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": true + }, + "rate_limit_enabled": { + "name": "rate_limit_enabled", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": true + }, + "rate_limit_time_window": { + "name": "rate_limit_time_window", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "rate_limit_max": { + "name": "rate_limit_max", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "request_count": { + "name": "request_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "remaining": { + "name": "remaining", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_request": { + "name": "last_request", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "permissions": { + "name": "permissions", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "metadata": { + "name": "metadata", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "apikey_config_id_idx": { + "name": "apikey_config_id_idx", + "columns": [ + "config_id" + ], + "isUnique": false + }, + "apikey_reference_id_idx": { + "name": "apikey_reference_id_idx", + "columns": [ + "reference_id" + ], + "isUnique": false + }, + "apikey_key_idx": { + "name": "apikey_key_idx", + "columns": [ + "key" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "deviceCode": { + "name": "deviceCode", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "device_code": { + "name": "device_code", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "user_code": { + "name": "user_code", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "client_id": { + "name": "client_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "scope": { + "name": "scope", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "last_polled_at": { + "name": "last_polled_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "polling_interval": { + "name": "polling_interval", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + } + }, + "indexes": { + "deviceCode_device_code_idx": { + "name": "deviceCode_device_code_idx", + "columns": [ + "device_code" + ], + "isUnique": false + }, + "deviceCode_user_code_idx": { + "name": "deviceCode_user_code_idx", + "columns": [ + "user_code" + ], + "isUnique": false + }, + "deviceCode_status_idx": { + "name": "deviceCode_status_idx", + "columns": [ + "status" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "invitation": { + "name": "invitation", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'pending'" + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + }, + "inviter_id": { + "name": "inviter_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "invitation_organizationId_idx": { + "name": "invitation_organizationId_idx", + "columns": [ + "organization_id" + ], + "isUnique": false + }, + "invitation_email_idx": { + "name": "invitation_email_idx", + "columns": [ + "email" + ], + "isUnique": false + } + }, + "foreignKeys": { + "invitation_organization_id_organization_id_fk": { + "name": "invitation_organization_id_organization_id_fk", + "tableFrom": "invitation", + "tableTo": "organization", + "columnsFrom": [ + "organization_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "invitation_inviter_id_user_id_fk": { + "name": "invitation_inviter_id_user_id_fk", + "tableFrom": "invitation", + "tableTo": "user", + "columnsFrom": [ + "inviter_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "member": { + "name": "member", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'member'" + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + } + }, + "indexes": { + "member_organizationId_idx": { + "name": "member_organizationId_idx", + "columns": [ + "organization_id" + ], + "isUnique": false + }, + "member_userId_idx": { + "name": "member_userId_idx", + "columns": [ + "user_id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "member_organization_id_organization_id_fk": { + "name": "member_organization_id_organization_id_fk", + "tableFrom": "member", + "tableTo": "organization", + "columnsFrom": [ + "organization_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "member_user_id_user_id_fk": { + "name": "member_user_id_user_id_fk", + "tableFrom": "member", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "organization": { + "name": "organization", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "slug": { + "name": "slug", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "logo": { + "name": "logo", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "metadata": { + "name": "metadata", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + } + }, + "indexes": { + "organization_slug_unique": { + "name": "organization_slug_unique", + "columns": [ + "slug" + ], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "session": { + "name": "session", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "token": { + "name": "token", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ip_address": { + "name": "ip_address", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "user_agent": { + "name": "user_agent", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "impersonated_by": { + "name": "impersonated_by", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "active_organization_id": { + "name": "active_organization_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "session_token_unique": { + "name": "session_token_unique", + "columns": [ + "token" + ], + "isUnique": true + }, + "session_userId_idx": { + "name": "session_userId_idx", + "columns": [ + "user_id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "session_user_id_user_id_fk": { + "name": "session_user_id_user_id_fk", + "tableFrom": "session", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "user": { + "name": "user", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "email_verified": { + "name": "email_verified", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "image": { + "name": "image", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "banned": { + "name": "banned", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": false + }, + "ban_reason": { + "name": "ban_reason", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "ban_expires": { + "name": "ban_expires", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "username": { + "name": "username", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "display_username": { + "name": "display_username", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + } + }, + "indexes": { + "user_email_unique": { + "name": "user_email_unique", + "columns": [ + "email" + ], + "isUnique": true + }, + "user_username_unique": { + "name": "user_username_unique", + "columns": [ + "username" + ], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "verification": { + "name": "verification", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "identifier": { + "name": "identifier", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "value": { + "name": "value", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + } + }, + "indexes": { + "verification_identifier_idx": { + "name": "verification_identifier_idx", + "columns": [ + "identifier" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + } + }, + "views": {}, + "enums": {}, + "_meta": { + "schemas": {}, + "tables": {}, + "columns": {} + }, + "internal": { + "indexes": {} + } +} \ No newline at end of file diff --git a/migrations/meta/_journal.json b/migrations/meta/_journal.json index 50a6ca77..b5813f49 100644 --- a/migrations/meta/_journal.json +++ b/migrations/meta/_journal.json @@ -358,6 +358,13 @@ "when": 1782331014772, "tag": "0051_nostalgic_cerebro", "breakpoints": true + }, + { + "idx": 52, + "version": "6", + "when": 1782761700959, + "tag": "0052_rename-downloader-http-engine", + "breakpoints": true } ] } \ No newline at end of file diff --git a/server/db/schema.ts b/server/db/schema.ts index da2edd53..b478f8b8 100644 --- a/server/db/schema.ts +++ b/server/db/schema.ts @@ -300,7 +300,7 @@ export const downloaders = sqliteTable( hostname: text('hostname').notNull().default('unknown'), platform: text('platform').notNull().default('unknown'), arch: text('arch').notNull().default('unknown'), - engine: text('engine').notNull().default('builtin'), + engine: text('engine').notNull().default('http'), capabilities: text('capabilities').notNull().default('[]'), maxConcurrentTasks: integer('max_concurrent_tasks').notNull().default(1), currentTasks: integer('current_tasks').notNull().default(0), diff --git a/server/middleware/authz.integration.test.ts b/server/middleware/authz.integration.test.ts index d3b0fd74..40da9ff7 100644 --- a/server/middleware/authz.integration.test.ts +++ b/server/middleware/authz.integration.test.ts @@ -91,7 +91,7 @@ async function registerDownloader(app: TestApp, name: string): Promise { hostname: 'host', platform: 'linux', arch: 'x64', - engine: 'builtin', + engine: 'http', capabilities: [], maxConcurrentTasks: 1, currentTasks: 0, diff --git a/server/test/setup.ts b/server/test/setup.ts index 6220e6b2..cb5cc2a8 100644 --- a/server/test/setup.ts +++ b/server/test/setup.ts @@ -401,7 +401,7 @@ const APP_SCHEMA_SQL = ` hostname TEXT NOT NULL DEFAULT 'unknown', platform TEXT NOT NULL DEFAULT 'unknown', arch TEXT NOT NULL DEFAULT 'unknown', - engine TEXT NOT NULL DEFAULT 'builtin', + engine TEXT NOT NULL DEFAULT 'http', capabilities TEXT NOT NULL DEFAULT '[]', max_concurrent_tasks INTEGER NOT NULL DEFAULT 1, current_tasks INTEGER NOT NULL DEFAULT 0, diff --git a/shared/schemas/downloads.ts b/shared/schemas/downloads.ts index 74152f34..fc1d6fc1 100644 --- a/shared/schemas/downloads.ts +++ b/shared/schemas/downloads.ts @@ -2,7 +2,7 @@ import { z } from '@hono/zod-openapi' import { isSafeHttpUrl } from '../url-safety' export const downloaderStatusSchema = z.enum(['online', 'offline', 'disabled']) -export const downloaderEngineSchema = z.enum(['builtin', 'aria2', 'qbittorrent']) +export const downloaderEngineSchema = z.enum(['http', 'aria2', 'qbittorrent']) export const downloadTaskStatusSchema = z.enum([ 'queued', 'assigned', diff --git a/shared/types/index.ts b/shared/types/index.ts index 6eef6443..01fe5df2 100644 --- a/shared/types/index.ts +++ b/shared/types/index.ts @@ -210,7 +210,7 @@ export interface PaginatedResponse { } export type DownloaderStatus = 'online' | 'offline' | 'disabled' -export type DownloaderEngine = 'builtin' | 'aria2' | 'qbittorrent' +export type DownloaderEngine = 'http' | 'aria2' | 'qbittorrent' // `Downloader` is inferred from `downloaderSchema` (the wire contract) in // shared/schemas/downloads.ts — one source of truth for the OpenAPI document,