From 3f7a859e73c8ce5c26dc2d053a0e682fdb759aca Mon Sep 17 00:00:00 2001 From: saltbo Date: Fri, 5 Jun 2026 15:24:56 -0400 Subject: [PATCH] refactor(downloader): consolidate api contract generation --- .github/workflows/ci.yml | 5 + .github/workflows/release.yml | 5 + docs/openapi/downloader.json | 599 ++++++------ downloader/cmd/zpan-downloader/main.go | 11 +- downloader/internal/client/client.go | 293 ++++-- downloader/internal/client/client_test.go | 101 +- downloader/internal/engine/aria2.go | 584 ++++++++++++ downloader/internal/engine/engine.go | 829 +---------------- downloader/internal/engine/engine_test.go | 43 + downloader/internal/engine/http.go | 126 +++ downloader/internal/engine/process.go | 64 ++ downloader/internal/engine/qbittorrent.go | 455 ++++++++++ downloader/internal/openapi/client.gen.go | 756 ++++++++------- downloader/internal/worker/api.go | 143 +++ downloader/internal/worker/api_test.go | 46 + downloader/internal/worker/engines.go | 153 ++++ downloader/internal/worker/seeds.go | 181 ++++ downloader/internal/worker/uploader.go | 306 +++++++ downloader/internal/worker/worker.go | 859 ++---------------- downloader/internal/worker/worker_test.go | 8 +- downloader/oapi-codegen.yaml | 5 + package.json | 5 +- scripts/check-downloader-openapi.ts | 77 ++ server/openapi/downloader.ts | 97 +- .../routes/download-tasks.integration.test.ts | 29 + server/routes/download-tasks.ts | 50 +- server/routes/downloaders.ts | 48 +- server/services/downloads.ts | 752 +-------------- server/services/downloads/core.ts | 643 +++++++++++++ server/services/downloads/mappers.ts | 93 ++ server/services/downloads/types.ts | 20 + shared/schemas/downloads.ts | 105 ++- shared/schemas/index.ts | 41 +- 33 files changed, 4310 insertions(+), 3222 deletions(-) create mode 100644 downloader/internal/engine/aria2.go create mode 100644 downloader/internal/engine/http.go create mode 100644 downloader/internal/engine/process.go create mode 100644 downloader/internal/engine/qbittorrent.go create mode 100644 downloader/internal/worker/api.go create mode 100644 downloader/internal/worker/api_test.go create mode 100644 downloader/internal/worker/engines.go create mode 100644 downloader/internal/worker/seeds.go create mode 100644 downloader/internal/worker/uploader.go create mode 100644 downloader/oapi-codegen.yaml create mode 100644 scripts/check-downloader-openapi.ts create mode 100644 server/services/downloads/core.ts create mode 100644 server/services/downloads/mappers.ts create mode 100644 server/services/downloads/types.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5f6b92af..505ace1a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -49,9 +49,14 @@ jobs: with: node-version: 24 cache: pnpm + - uses: actions/setup-go@v6 + with: + go-version-file: downloader/go.mod + cache-dependency-path: downloader/go.sum - run: pnpm install --frozen-lockfile - run: pnpm lint - run: pnpm typecheck + - run: pnpm openapi:downloader:check - run: pnpm exec vitest run --project unit --coverage - uses: codecov/codecov-action@v5 if: always() diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index fa55b760..731572b3 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -18,9 +18,14 @@ jobs: with: node-version: 24 cache: pnpm + - uses: actions/setup-go@v6 + with: + go-version-file: downloader/go.mod + cache-dependency-path: downloader/go.sum - run: pnpm install --frozen-lockfile - run: pnpm lint - run: pnpm typecheck + - run: pnpm openapi:downloader:check - run: pnpm test - run: mkdir -p dist # required by vitest cloudflare config - run: pnpm test:cf diff --git a/docs/openapi/downloader.json b/docs/openapi/downloader.json index 180fc45b..c557d2a8 100644 --- a/docs/openapi/downloader.json +++ b/docs/openapi/downloader.json @@ -104,215 +104,6 @@ "device_code", "client_id" ] - }, - "ObjectDraft": { - "type": "object", - "properties": { - "id": { - "type": "string" - }, - "name": { - "type": "string" - }, - "uploadUrl": { - "type": "string" - }, - "contentDisposition": { - "type": "string" - } - }, - "required": [ - "id", - "name" - ] - }, - "ConfirmObjectRequest": { - "type": "object", - "properties": { - "action": { - "type": "string", - "enum": [ - "confirm" - ] - }, - "onConflict": { - "type": "string", - "enum": [ - "fail", - "rename" - ] - } - }, - "required": [ - "action" - ] - }, - "ObjectUploadSession": { - "type": "object", - "properties": { - "id": { - "type": "string" - }, - "objectId": { - "type": "string" - }, - "uploadId": { - "type": "string" - }, - "partSize": { - "type": "integer" - }, - "status": { - "type": "string", - "enum": [ - "active", - "completed", - "aborted" - ] - }, - "expiresAt": { - "type": "string" - }, - "createdAt": { - "type": "string" - }, - "updatedAt": { - "type": "string" - } - }, - "required": [ - "id", - "objectId", - "uploadId", - "partSize", - "status", - "expiresAt", - "createdAt", - "updatedAt" - ] - }, - "CreateObjectUploadSessionRequest": { - "type": "object", - "properties": { - "partSize": { - "type": "integer", - "minimum": 5242880, - "maximum": 536870912 - } - } - }, - "PresignObjectUploadPartsResponse": { - "type": "object", - "properties": { - "uploadId": { - "type": "string" - }, - "partSize": { - "type": "integer" - }, - "parts": { - "type": "array", - "items": { - "$ref": "#/components/schemas/PresignedObjectUploadPart" - } - } - }, - "required": [ - "uploadId", - "partSize", - "parts" - ] - }, - "PresignedObjectUploadPart": { - "type": "object", - "properties": { - "partNumber": { - "type": "integer" - }, - "url": { - "type": "string" - } - }, - "required": [ - "partNumber", - "url" - ] - }, - "PresignObjectUploadPartsRequest": { - "type": "object", - "properties": { - "partNumbers": { - "type": "array", - "items": { - "type": "integer", - "minimum": 1, - "maximum": 10000 - }, - "minItems": 1, - "maxItems": 100 - } - }, - "required": [ - "partNumbers" - ] - }, - "PatchObjectUploadSessionRequest": { - "anyOf": [ - { - "$ref": "#/components/schemas/CompleteObjectUploadSessionRequest" - }, - { - "$ref": "#/components/schemas/AbortObjectUploadSessionRequest" - } - ] - }, - "CompleteObjectUploadSessionRequest": { - "type": "object", - "properties": { - "action": { - "type": "string", - "enum": [ - "complete" - ] - }, - "parts": { - "type": "array", - "items": { - "type": "object", - "properties": { - "partNumber": { - "type": "integer" - }, - "etag": { - "type": "string" - } - }, - "required": [ - "partNumber", - "etag" - ] - }, - "minItems": 1 - } - }, - "required": [ - "action", - "parts" - ] - }, - "AbortObjectUploadSessionRequest": { - "type": "object", - "properties": { - "action": { - "type": "string", - "enum": [ - "abort" - ] - } - }, - "required": [ - "action" - ] } }, "parameters": {} @@ -638,11 +429,11 @@ }, "peerUploadedBytes": { "type": "integer", - "minimum": 0 + "format": "int64" }, "peerUploadBps": { "type": "integer", - "minimum": 0 + "format": "int64" }, "trackers": { "type": "array", @@ -700,11 +491,11 @@ }, "downloadBps": { "type": "integer", - "minimum": 0 + "format": "int64" }, "uploadBps": { "type": "integer", - "minimum": 0 + "format": "int64" } }, "required": [ @@ -724,11 +515,11 @@ }, "size": { "type": "integer", - "minimum": 0 + "format": "int64" }, "completedBytes": { "type": "integer", - "minimum": 0 + "format": "int64" }, "selected": { "type": "boolean" @@ -839,7 +630,8 @@ ] }, "targetFolder": { - "type": "string" + "type": "string", + "maxLength": 1024 }, "name": { "type": "string", @@ -1014,11 +806,11 @@ }, "peerUploadedBytes": { "type": "integer", - "minimum": 0 + "format": "int64" }, "peerUploadBps": { "type": "integer", - "minimum": 0 + "format": "int64" }, "trackers": { "type": "array", @@ -1076,11 +868,11 @@ }, "downloadBps": { "type": "integer", - "minimum": 0 + "format": "int64" }, "uploadBps": { "type": "integer", - "minimum": 0 + "format": "int64" } }, "required": [ @@ -1100,11 +892,11 @@ }, "size": { "type": "integer", - "minimum": 0 + "format": "int64" }, "completedBytes": { "type": "integer", - "minimum": 0 + "format": "int64" }, "selected": { "type": "boolean" @@ -1500,11 +1292,11 @@ }, "peerUploadedBytes": { "type": "integer", - "minimum": 0 + "format": "int64" }, "peerUploadBps": { "type": "integer", - "minimum": 0 + "format": "int64" }, "trackers": { "type": "array", @@ -1562,11 +1354,11 @@ }, "downloadBps": { "type": "integer", - "minimum": 0 + "format": "int64" }, "uploadBps": { "type": "integer", - "minimum": 0 + "format": "int64" } }, "required": [ @@ -1586,11 +1378,11 @@ }, "size": { "type": "integer", - "minimum": 0 + "format": "int64" }, "completedBytes": { "type": "integer", - "minimum": 0 + "format": "int64" }, "selected": { "type": "boolean" @@ -1688,24 +1480,24 @@ }, "downloadedBytes": { "type": "integer", - "minimum": 0 + "format": "int64" }, "storageUploadedBytes": { "type": "integer", - "minimum": 0 + "format": "int64" }, "totalBytes": { "type": "integer", - "nullable": true, - "minimum": 0 + "format": "int64", + "nullable": true }, "downloadBps": { "type": "integer", - "minimum": 0 + "format": "int64" }, "storageUploadBps": { "type": "integer", - "minimum": 0 + "format": "int64" }, "errorMessage": { "type": "string", @@ -1779,11 +1571,11 @@ }, "peerUploadedBytes": { "type": "integer", - "minimum": 0 + "format": "int64" }, "peerUploadBps": { "type": "integer", - "minimum": 0 + "format": "int64" }, "trackers": { "type": "array", @@ -1841,11 +1633,11 @@ }, "downloadBps": { "type": "integer", - "minimum": 0 + "format": "int64" }, "uploadBps": { "type": "integer", - "minimum": 0 + "format": "int64" } }, "required": [ @@ -1865,11 +1657,11 @@ }, "size": { "type": "integer", - "minimum": 0 + "format": "int64" }, "completedBytes": { "type": "integer", - "minimum": 0 + "format": "int64" }, "selected": { "type": "boolean" @@ -2034,11 +1826,11 @@ }, "peerUploadedBytes": { "type": "integer", - "minimum": 0 + "format": "int64" }, "peerUploadBps": { "type": "integer", - "minimum": 0 + "format": "int64" }, "trackers": { "type": "array", @@ -2096,11 +1888,11 @@ }, "downloadBps": { "type": "integer", - "minimum": 0 + "format": "int64" }, "uploadBps": { "type": "integer", - "minimum": 0 + "format": "int64" } }, "required": [ @@ -2120,11 +1912,11 @@ }, "size": { "type": "integer", - "minimum": 0 + "format": "int64" }, "completedBytes": { "type": "integer", - "minimum": 0 + "format": "int64" }, "selected": { "type": "boolean" @@ -2425,11 +2217,11 @@ }, "peerUploadedBytes": { "type": "integer", - "minimum": 0 + "format": "int64" }, "peerUploadBps": { "type": "integer", - "minimum": 0 + "format": "int64" }, "trackers": { "type": "array", @@ -2487,11 +2279,11 @@ }, "downloadBps": { "type": "integer", - "minimum": 0 + "format": "int64" }, "uploadBps": { "type": "integer", - "minimum": 0 + "format": "int64" } }, "required": [ @@ -2511,11 +2303,11 @@ }, "size": { "type": "integer", - "minimum": 0 + "format": "int64" }, "completedBytes": { "type": "integer", - "minimum": 0 + "format": "int64" }, "selected": { "type": "boolean" @@ -2710,18 +2502,15 @@ }, "downloadBps": { "type": "integer", - "minimum": 0, - "default": 0 + "format": "int64" }, "uploadBps": { "type": "integer", - "minimum": 0, - "default": 0 + "format": "int64" }, "freeDiskBytes": { "type": "integer", - "minimum": 0, - "default": 0 + "format": "int64" } }, "required": [ @@ -3051,18 +2840,15 @@ }, "downloadBps": { "type": "integer", - "minimum": 0, - "default": 0 + "format": "int64" }, "uploadBps": { "type": "integer", - "minimum": 0, - "default": 0 + "format": "int64" }, "freeDiskBytes": { "type": "integer", - "minimum": 0, - "default": 0 + "format": "int64" } }, "required": [ @@ -3446,14 +3232,15 @@ "minLength": 1 }, "size": { - "type": "number" + "type": "integer", + "minimum": 0 }, "parent": { "type": "string", "default": "" }, "dirtype": { - "type": "number", + "type": "integer", "default": 0 }, "onConflict": { @@ -3479,7 +3266,25 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ObjectDraft" + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "uploadUrl": { + "type": "string" + }, + "contentDisposition": { + "type": "string" + } + }, + "required": [ + "id", + "name" + ] } } } @@ -3489,7 +3294,25 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ObjectDraft" + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "uploadUrl": { + "type": "string" + }, + "contentDisposition": { + "type": "string" + } + }, + "required": [ + "id", + "name" + ] } } } @@ -3534,7 +3357,26 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ConfirmObjectRequest" + "type": "object", + "properties": { + "action": { + "type": "string", + "enum": [ + "confirm" + ] + }, + "onConflict": { + "type": "string", + "enum": [ + "fail", + "rename", + "replace" + ] + } + }, + "required": [ + "action" + ] } } } @@ -3545,7 +3387,25 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ObjectDraft" + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "uploadUrl": { + "type": "string" + }, + "contentDisposition": { + "type": "string" + } + }, + "required": [ + "id", + "name" + ] } } } @@ -3590,7 +3450,14 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/CreateObjectUploadSessionRequest" + "type": "object", + "properties": { + "partSize": { + "type": "integer", + "minimum": 5242880, + "maximum": 536870912 + } + } } } } @@ -3601,7 +3468,48 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ObjectUploadSession" + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "objectId": { + "type": "string" + }, + "uploadId": { + "type": "string" + }, + "partSize": { + "type": "integer" + }, + "status": { + "type": "string", + "enum": [ + "active", + "completed", + "aborted" + ] + }, + "expiresAt": { + "type": "string" + }, + "createdAt": { + "type": "string" + }, + "updatedAt": { + "type": "string" + } + }, + "required": [ + "id", + "objectId", + "uploadId", + "partSize", + "status", + "expiresAt", + "createdAt", + "updatedAt" + ] } } } @@ -3664,7 +3572,22 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/PresignObjectUploadPartsRequest" + "type": "object", + "properties": { + "partNumbers": { + "type": "array", + "items": { + "type": "integer", + "minimum": 1, + "maximum": 10000 + }, + "minItems": 1, + "maxItems": 100 + } + }, + "required": [ + "partNumbers" + ] } } } @@ -3675,7 +3598,38 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/PresignObjectUploadPartsResponse" + "type": "object", + "properties": { + "uploadId": { + "type": "string" + }, + "partSize": { + "type": "integer" + }, + "parts": { + "type": "array", + "items": { + "type": "object", + "properties": { + "partNumber": { + "type": "integer" + }, + "url": { + "type": "string" + } + }, + "required": [ + "partNumber", + "url" + ] + } + } + }, + "required": [ + "uploadId", + "partSize", + "parts" + ] } } } @@ -3738,7 +3692,59 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/PatchObjectUploadSessionRequest" + "oneOf": [ + { + "type": "object", + "properties": { + "action": { + "type": "string", + "enum": [ + "complete" + ] + }, + "parts": { + "type": "array", + "items": { + "type": "object", + "properties": { + "partNumber": { + "type": "integer", + "minimum": 1, + "maximum": 10000 + }, + "etag": { + "type": "string", + "minLength": 1 + } + }, + "required": [ + "partNumber", + "etag" + ] + }, + "minItems": 1 + } + }, + "required": [ + "action", + "parts" + ] + }, + { + "type": "object", + "properties": { + "action": { + "type": "string", + "enum": [ + "abort" + ] + } + }, + "required": [ + "action" + ] + } + ] } } } @@ -3749,7 +3755,48 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ObjectUploadSession" + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "objectId": { + "type": "string" + }, + "uploadId": { + "type": "string" + }, + "partSize": { + "type": "integer" + }, + "status": { + "type": "string", + "enum": [ + "active", + "completed", + "aborted" + ] + }, + "expiresAt": { + "type": "string" + }, + "createdAt": { + "type": "string" + }, + "updatedAt": { + "type": "string" + } + }, + "required": [ + "id", + "objectId", + "uploadId", + "partSize", + "status", + "expiresAt", + "createdAt", + "updatedAt" + ] } } } diff --git a/downloader/cmd/zpan-downloader/main.go b/downloader/cmd/zpan-downloader/main.go index 720df030..e9dcd18f 100644 --- a/downloader/cmd/zpan-downloader/main.go +++ b/downloader/cmd/zpan-downloader/main.go @@ -90,7 +90,11 @@ func runCommand(v *viper.Viper) *cobra.Command { ) ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) defer stop() - return worker.New(cfg).Run(ctx) + downloader, err := worker.New(cfg) + if err != nil { + return err + } + return downloader.Run(ctx) }, } } @@ -123,7 +127,10 @@ func loginCommand(v *viper.Viper, cfgFile *string) *cobra.Command { } } - api := client.New(serverURL, "") + api, err := client.New(serverURL, "") + if err != nil { + return err + } ctx := cmd.Context() code, err := api.RequestDeviceCode(ctx) if err != nil { diff --git a/downloader/internal/client/client.go b/downloader/internal/client/client.go index 107232f4..9069acfd 100644 --- a/downloader/internal/client/client.go +++ b/downloader/internal/client/client.go @@ -171,25 +171,21 @@ type CreateDownloaderResponse struct { Token string `json:"token"` } -func New(baseURL, token string) *Client { +func New(baseURL, token string) (*Client, error) { httpClient := &http.Client{Timeout: 60 * time.Second} api, err := openapi.NewClientWithResponses(strings.TrimRight(baseURL, "/"), openapi.WithHTTPClient(httpClient)) if err != nil { - panic(err) + return nil, err } return &Client{ baseURL: baseURL, token: token, api: api, - } + }, nil } func (c *Client) Heartbeat(ctx context.Context, heartbeat Heartbeat) error { - body, err := jsonBody(heartbeat) - if err != nil { - return err - } - res, err := c.api.PostApiDownloaderHeartbeatWithBodyWithResponse(ctx, "application/json", body, bearer(c.token)) + res, err := c.api.PostApiDownloaderHeartbeatWithResponse(ctx, heartbeatRequestBody(heartbeat), bearer(c.token)) if err != nil { return err } @@ -199,6 +195,8 @@ func (c *Client) Heartbeat(ctx context.Context, heartbeat Heartbeat) error { func (c *Client) AssignedTasks(ctx context.Context) ([]DownloadTask, error) { return c.assignedTasks(ctx, []openapi.GetApiDownloadTasksParamsStatus{ openapi.GetApiDownloadTasksParamsStatusAssigned, + openapi.GetApiDownloadTasksParamsStatusRunning, + openapi.GetApiDownloadTasksParamsStatusUploading, }) } @@ -227,11 +225,16 @@ func (c *Client) assignedTasks(ctx context.Context, statuses []openapi.GetApiDow if err := expectStatus("GET", "/api/download-tasks", res.StatusCode(), res.Body, http.StatusOK); err != nil { return nil, err } - var result Page[DownloadTask] - if err := decodeJSON(res.Body, &result); err != nil { - return nil, fmt.Errorf("GET /api/download-tasks failed: %w", err) + if res.JSON200 == nil { + return nil, fmt.Errorf("GET /api/download-tasks failed: empty response") + } + for _, item := range res.JSON200.Items { + task, err := downloadTaskFromOpenAPI(item) + if err != nil { + return nil, fmt.Errorf("GET /api/download-tasks failed: %w", err) + } + tasks = append(tasks, task) } - tasks = append(tasks, result.Items...) } return tasks, nil } @@ -284,38 +287,160 @@ func (c *Client) PollDeviceToken(ctx context.Context, deviceCode string) (Device } func (c *Client) CreateDownloader(ctx context.Context, accessToken string, req CreateDownloaderRequest) (CreateDownloaderResponse, error) { - body, err := jsonBody(req) - if err != nil { - return CreateDownloaderResponse{}, err - } - res, err := c.api.PostApiAdminDownloadersWithBodyWithResponse(ctx, "application/json", body, bearer(accessToken)) + res, err := c.api.PostApiAdminDownloadersWithResponse(ctx, createDownloaderRequestBody(req), bearer(accessToken)) if err != nil { return CreateDownloaderResponse{}, err } if err := expectStatus("POST", "/api/admin/downloaders", res.StatusCode(), res.Body, http.StatusCreated); err != nil { return CreateDownloaderResponse{}, err } - var out CreateDownloaderResponse - if err := decodeJSON(res.Body, &out); err != nil { - return CreateDownloaderResponse{}, fmt.Errorf("POST /api/admin/downloaders failed: %w", err) + if res.JSON201 == nil { + return CreateDownloaderResponse{}, fmt.Errorf("POST /api/admin/downloaders failed: empty response") } + out := CreateDownloaderResponse{Token: res.JSON201.Token} + out.Downloader.ID = res.JSON201.Downloader.Id return out, nil } -func (c *Client) UpdateTask(ctx context.Context, id string, patch TaskPatch) (DownloadTask, error) { - body, err := jsonBody(patch) +func heartbeatRequestBody(heartbeat Heartbeat) openapi.PostApiDownloaderHeartbeatJSONRequestBody { + return openapi.PostApiDownloaderHeartbeatJSONRequestBody{ + Arch: heartbeat.Arch, + Capabilities: heartbeat.Capabilities, + CurrentTasks: heartbeat.CurrentTasks, + DownloadBps: &heartbeat.DownloadBps, + Engine: openapi.PostApiDownloaderHeartbeatJSONBodyEngine(heartbeat.Engine), + FreeDiskBytes: &heartbeat.FreeDiskBytes, + Hostname: heartbeat.Hostname, + MaxConcurrentTasks: heartbeat.MaxConcurrentTasks, + Platform: heartbeat.Platform, + UploadBps: &heartbeat.UploadBps, + Version: heartbeat.Version, + } +} + +func createDownloaderRequestBody(req CreateDownloaderRequest) openapi.PostApiAdminDownloadersJSONRequestBody { + return openapi.PostApiAdminDownloadersJSONRequestBody{ + Name: req.Name, + Heartbeat: struct { + Arch string `json:"arch"` + Capabilities []string `json:"capabilities"` + CurrentTasks int `json:"currentTasks"` + DownloadBps *int64 `json:"downloadBps,omitempty"` + Engine openapi.PostApiAdminDownloadersJSONBodyHeartbeatEngine `json:"engine"` + FreeDiskBytes *int64 `json:"freeDiskBytes,omitempty"` + Hostname string `json:"hostname"` + MaxConcurrentTasks int `json:"maxConcurrentTasks"` + Platform string `json:"platform"` + UploadBps *int64 `json:"uploadBps,omitempty"` + Version string `json:"version"` + }{ + Arch: req.Heartbeat.Arch, + Capabilities: req.Heartbeat.Capabilities, + CurrentTasks: req.Heartbeat.CurrentTasks, + DownloadBps: &req.Heartbeat.DownloadBps, + Engine: openapi.PostApiAdminDownloadersJSONBodyHeartbeatEngine(req.Heartbeat.Engine), + FreeDiskBytes: &req.Heartbeat.FreeDiskBytes, + Hostname: req.Heartbeat.Hostname, + MaxConcurrentTasks: req.Heartbeat.MaxConcurrentTasks, + Platform: req.Heartbeat.Platform, + UploadBps: &req.Heartbeat.UploadBps, + Version: req.Heartbeat.Version, + }, + } +} + +func taskPatchRequestBody(patch TaskPatch) (openapi.PatchApiDownloadTasksIdJSONRequestBody, error) { + body := openapi.PatchApiDownloadTasksIdJSONRequestBody{ + DownloadBps: patch.DownloadBps, + DownloadedBytes: patch.DownloadedBytes, + ErrorMessage: patch.ErrorMessage, + ResultObjectId: patch.ResultObjectID, + StorageUploadBps: patch.StorageUploadBps, + StorageUploadedBytes: patch.StorageUploadedBytes, + TotalBytes: patch.TotalBytes, + } + if patch.Status != "" { + status := openapi.PatchApiDownloadTasksIdJSONBodyStatus(patch.Status) + body.Status = &status + } + if patch.Detail != nil { + data, err := json.Marshal(patch.Detail) + if err != nil { + return body, err + } + var detail struct { + Connections *int `json:"connections,omitempty"` + Engine *openapi.PatchApiDownloadTasksIdJSONBodyDetailEngine `json:"engine,omitempty"` + EngineState *string `json:"engineState,omitempty"` + EtaSeconds *int `json:"etaSeconds,omitempty"` + Files *[]struct { + CompletedBytes *int64 `json:"completedBytes,omitempty"` + Path string `json:"path"` + Selected *bool `json:"selected,omitempty"` + Size int64 `json:"size"` + } `json:"files,omitempty"` + InfoHash *string `json:"infoHash,omitempty"` + Leechers *int `json:"leechers,omitempty"` + Message *string `json:"message,omitempty"` + PeerSamples *[]struct { + Address string `json:"address"` + Client *string `json:"client,omitempty"` + DownloadBps *int64 `json:"downloadBps,omitempty"` + Progress *float32 `json:"progress,omitempty"` + UploadBps *int64 `json:"uploadBps,omitempty"` + } `json:"peerSamples,omitempty"` + PeerUploadBps *int64 `json:"peerUploadBps,omitempty"` + PeerUploadedBytes *int64 `json:"peerUploadedBytes,omitempty"` + Peers *int `json:"peers,omitempty"` + Phase *openapi.PatchApiDownloadTasksIdJSONBodyDetailPhase `json:"phase,omitempty"` + Seeders *int `json:"seeders,omitempty"` + TorrentName *string `json:"torrentName,omitempty"` + Trackers *[]struct { + Leechers *int `json:"leechers,omitempty"` + Message *string `json:"message,omitempty"` + Peers *int `json:"peers,omitempty"` + Seeds *int `json:"seeds,omitempty"` + Status *string `json:"status,omitempty"` + Url string `json:"url"` + } `json:"trackers,omitempty"` + } + if err := json.Unmarshal(data, &detail); err != nil { + return body, err + } + body.Detail = &detail + } + return body, nil +} + +func downloadTaskFromOpenAPI(value any) (DownloadTask, error) { + data, err := json.Marshal(value) if err != nil { return DownloadTask{}, err } - res, err := c.api.PatchApiDownloadTasksIdWithBodyWithResponse(ctx, id, "application/json", body, bearer(c.token)) + var task DownloadTask + if err := json.Unmarshal(data, &task); err != nil { + return DownloadTask{}, err + } + return task, nil +} + +func (c *Client) UpdateTask(ctx context.Context, id string, patch TaskPatch) (DownloadTask, error) { + body, err := taskPatchRequestBody(patch) + if err != nil { + return DownloadTask{}, err + } + res, err := c.api.PatchApiDownloadTasksIdWithResponse(ctx, id, body, bearer(c.token)) if err != nil { return DownloadTask{}, err } if err := expectStatus("PATCH", "/api/download-tasks/"+id, res.StatusCode(), res.Body, http.StatusOK); err != nil { return DownloadTask{}, err } - var task DownloadTask - if err := decodeJSON(res.Body, &task); err != nil { + if res.JSON200 == nil { + return DownloadTask{}, fmt.Errorf("PATCH /api/download-tasks/%s failed: empty response", id) + } + task, err := downloadTaskFromOpenAPI(*res.JSON200) + if err != nil { return DownloadTask{}, fmt.Errorf("PATCH /api/download-tasks/%s failed: %w", id, err) } return task, nil @@ -338,47 +463,47 @@ func (c *Client) createMatter( parent string, dirtype int, ) (ObjectDraft, error) { - body, err := jsonBody(struct { - Name string `json:"name"` - Type string `json:"type"` - Size int64 `json:"size"` - Parent string `json:"parent"` - Dirtype int `json:"dirtype"` - OnConflict string `json:"onConflict"` - }{ + sizeInt := int(size) + onConflict := openapi.PostApiObjectsJSONBodyOnConflictRename + res, err := c.api.PostApiObjectsWithResponse(ctx, openapi.PostApiObjectsJSONRequestBody{ Name: name, Type: contentType, - Size: size, - Parent: parent, - Dirtype: dirtype, - OnConflict: "rename", - }) - if err != nil { - return ObjectDraft{}, err - } - res, err := c.api.PostApiObjectsWithBodyWithResponse(ctx, "application/json", body, bearer(token)) + Size: &sizeInt, + Parent: &parent, + Dirtype: &dirtype, + OnConflict: &onConflict, + }, bearer(token)) if err != nil { return ObjectDraft{}, err } if err := expectStatus("POST", "/api/objects", res.StatusCode(), res.Body, http.StatusOK, http.StatusCreated); err != nil { return ObjectDraft{}, err } - var draft ObjectDraft - if err := decodeJSON(res.Body, &draft); err != nil { - return ObjectDraft{}, fmt.Errorf("POST /api/objects failed: %w", err) + if res.JSON200 != nil { + return ObjectDraft{ + ID: res.JSON200.Id, + Name: res.JSON200.Name, + UploadURL: derefString(res.JSON200.UploadUrl), + ContentDisposition: derefString(res.JSON200.ContentDisposition), + }, nil } - return draft, nil + if res.JSON201 != nil { + return ObjectDraft{ + ID: res.JSON201.Id, + Name: res.JSON201.Name, + UploadURL: derefString(res.JSON201.UploadUrl), + ContentDisposition: derefString(res.JSON201.ContentDisposition), + }, nil + } + return ObjectDraft{}, fmt.Errorf("POST /api/objects failed: empty response") } func (c *Client) ConfirmObject(ctx context.Context, token string, id string) error { - body, err := jsonBody(struct { - Action string `json:"action"` - OnConflict string `json:"onConflict"` - }{Action: "confirm", OnConflict: "rename"}) - if err != nil { - return err - } - res, err := c.api.PatchApiObjectsIdWithBodyWithResponse(ctx, id, "application/json", body, bearer(token)) + onConflict := openapi.PatchApiObjectsIdJSONBodyOnConflictRename + res, err := c.api.PatchApiObjectsIdWithResponse(ctx, id, openapi.PatchApiObjectsIdJSONRequestBody{ + Action: openapi.Confirm, + OnConflict: &onConflict, + }, bearer(token)) if err != nil { return err } @@ -386,13 +511,10 @@ func (c *Client) ConfirmObject(ctx context.Context, token string, id string) err } func (c *Client) CreateObjectUploadSession(ctx context.Context, token string, id string, partSize int64) (ObjectUploadSession, error) { - body, err := jsonBody(struct { - PartSize int64 `json:"partSize"` - }{PartSize: partSize}) - if err != nil { - return ObjectUploadSession{}, err - } - res, err := c.api.PostApiObjectsIdUploadsWithBodyWithResponse(ctx, id, "application/json", body, bearer(token)) + partSizeInt := int(partSize) + res, err := c.api.PostApiObjectsIdUploadsWithResponse(ctx, id, openapi.PostApiObjectsIdUploadsJSONRequestBody{ + PartSize: &partSizeInt, + }, bearer(token)) if err != nil { return ObjectUploadSession{}, err } @@ -411,13 +533,9 @@ func (c *Client) CreateObjectUploadSession(ctx context.Context, token string, id } func (c *Client) PresignObjectUploadParts(ctx context.Context, token string, id string, sessionID string, partNumbers []int) ([]PresignedObjectUploadPart, error) { - body, err := jsonBody(struct { - PartNumbers []int `json:"partNumbers"` - }{PartNumbers: partNumbers}) - if err != nil { - return nil, err - } - res, err := c.api.PostApiObjectsIdUploadsUploadSessionIdPartsWithBodyWithResponse(ctx, id, sessionID, "application/json", body, bearer(token)) + res, err := c.api.PostApiObjectsIdUploadsUploadSessionIdPartsWithResponse(ctx, id, sessionID, openapi.PostApiObjectsIdUploadsUploadSessionIdPartsJSONRequestBody{ + PartNumbers: partNumbers, + }, bearer(token)) if err != nil { return nil, err } @@ -436,10 +554,24 @@ func (c *Client) PresignObjectUploadParts(ctx context.Context, token string, id } func (c *Client) CompleteObjectUploadSession(ctx context.Context, token string, id string, sessionID string, parts []CompletedObjectUploadPart) error { - body, err := jsonBody(struct { - Action string `json:"action"` - Parts []CompletedObjectUploadPart `json:"parts"` - }{Action: "complete", Parts: parts}) + complete := openapi.PatchApiObjectsIdUploadsUploadSessionIdJSONBody0{ + Action: openapi.Complete, + Parts: make([]struct { + Etag string `json:"etag"` + PartNumber int `json:"partNumber"` + }, 0, len(parts)), + } + for _, part := range parts { + complete.Parts = append(complete.Parts, struct { + Etag string `json:"etag"` + PartNumber int `json:"partNumber"` + }{Etag: part.ETag, PartNumber: part.PartNumber}) + } + var union openapi.PatchApiObjectsIdUploadsUploadSessionIdJSONBody + if err := union.FromPatchApiObjectsIdUploadsUploadSessionIdJSONBody0(complete); err != nil { + return err + } + body, err := jsonBody(union) if err != nil { return err } @@ -451,9 +583,13 @@ func (c *Client) CompleteObjectUploadSession(ctx context.Context, token string, } func (c *Client) AbortObjectUploadSession(ctx context.Context, token string, id string, sessionID string) error { - body, err := jsonBody(struct { - Action string `json:"action"` - }{Action: "abort"}) + var union openapi.PatchApiObjectsIdUploadsUploadSessionIdJSONBody + if err := union.FromPatchApiObjectsIdUploadsUploadSessionIdJSONBody1(openapi.PatchApiObjectsIdUploadsUploadSessionIdJSONBody1{ + Action: openapi.Abort, + }); err != nil { + return err + } + body, err := jsonBody(union) if err != nil { return err } @@ -472,6 +608,13 @@ func jsonBody(value any) (*bytes.Reader, error) { return bytes.NewReader(data), nil } +func derefString(value *string) string { + if value == nil { + return "" + } + return *value +} + func decodeJSON(data []byte, out any) error { if len(data) == 0 { return fmt.Errorf("empty response body") diff --git a/downloader/internal/client/client_test.go b/downloader/internal/client/client_test.go index 03c60bb4..753f079d 100644 --- a/downloader/internal/client/client_test.go +++ b/downloader/internal/client/client_test.go @@ -6,6 +6,7 @@ import ( "net/http" "net/http/httptest" "reflect" + "sort" "testing" ) @@ -18,11 +19,12 @@ func TestCreateObjectUsesRenameConflictStrategy(t *testing.T) { if err := json.NewDecoder(r.Body).Decode(&body); err != nil { t.Fatal(err) } + w.Header().Set("Content-Type", "application/json") _ = json.NewEncoder(w).Encode(ObjectDraft{ID: "object-1", Name: "movie (1).mkv"}) })) defer server.Close() - _, err := New(server.URL, "token").CreateObject(context.Background(), "upload-token", "movie.mkv", 1024, "Downloads") + _, err := mustClient(t, server.URL, "token").CreateObject(context.Background(), "upload-token", "movie.mkv", 1024, "Downloads") if err != nil { t.Fatal(err) } @@ -31,6 +33,90 @@ func TestCreateObjectUsesRenameConflictStrategy(t *testing.T) { } } +func TestAssignedTasksFetchesRecoverableStatuses(t *testing.T) { + var statuses []string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/api/download-tasks" { + t.Fatalf("unexpected path: %s", r.URL.Path) + } + w.Header().Set("Content-Type", "application/json") + status := r.URL.Query().Get("status") + statuses = append(statuses, status) + _ = json.NewEncoder(w).Encode(Page[DownloadTask]{ + Items: []DownloadTask{{ID: "task-" + status, Status: status, UploadToken: "upload-token"}}, + }) + })) + defer server.Close() + + tasks, err := mustClient(t, server.URL, "token").AssignedTasks(context.Background()) + if err != nil { + t.Fatal(err) + } + sort.Strings(statuses) + expected := []string{"assigned", "running", "uploading"} + if !reflect.DeepEqual(statuses, expected) { + t.Fatalf("expected recoverable statuses %v, got %v", expected, statuses) + } + if len(tasks) != 3 { + t.Fatalf("expected three tasks, got %d", len(tasks)) + } +} + +func TestUpdateTaskUsesGeneratedRequestShape(t *testing.T) { + var body map[string]any + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPatch || r.URL.Path != "/api/download-tasks/task-1" { + t.Fatalf("unexpected request: %s %s", r.Method, r.URL.Path) + } + if err := json.NewDecoder(r.Body).Decode(&body); err != nil { + t.Fatal(err) + } + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(DownloadTask{ + ID: "task-1", + SourceType: "http", + SourceURI: "https://example.com/file.bin", + Name: "file.bin", + TargetFolder: "", + Tags: []string{}, + Status: "running", + DownloadedBytes: 1024, + StorageUploadedBytes: 0, + DownloadBps: 10, + StorageUploadBps: 0, + }) + })) + defer server.Close() + + downloadedBytes := int64(1024) + totalBytes := int64(2048) + etaSeconds := int64(30) + task, err := mustClient(t, server.URL, "token").UpdateTask(context.Background(), "task-1", TaskPatch{ + Status: "running", + DownloadedBytes: &downloadedBytes, + TotalBytes: &totalBytes, + Detail: &DownloadTaskDetail{ + Engine: "builtin", + Phase: "downloading", + ETASeconds: &etaSeconds, + Files: []DownloadTaskFile{{Path: "file.bin", Size: 2048, CompletedBytes: &downloadedBytes}}, + }, + }) + if err != nil { + t.Fatal(err) + } + if task.ID != "task-1" || task.Status != "running" { + t.Fatalf("unexpected task: %#v", task) + } + if body["status"] != "running" || body["downloadedBytes"] != float64(1024) || body["totalBytes"] != float64(2048) { + t.Fatalf("unexpected patch body: %#v", body) + } + detail, ok := body["detail"].(map[string]any) + if !ok || detail["engine"] != "builtin" || detail["phase"] != "downloading" || detail["etaSeconds"] != float64(30) { + t.Fatalf("unexpected detail body: %#v", body["detail"]) + } +} + func TestConfirmObjectUsesRenameConflictStrategy(t *testing.T) { var body map[string]any server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -44,7 +130,7 @@ func TestConfirmObjectUsesRenameConflictStrategy(t *testing.T) { })) defer server.Close() - if err := New(server.URL, "token").ConfirmObject(context.Background(), "upload-token", "object-1"); err != nil { + if err := mustClient(t, server.URL, "token").ConfirmObject(context.Background(), "upload-token", "object-1"); err != nil { t.Fatal(err) } if body["action"] != "confirm" { @@ -116,7 +202,7 @@ func TestMultipartUploadSessionClientMethods(t *testing.T) { })) defer server.Close() - api := New(server.URL, "token") + api := mustClient(t, server.URL, "token") session, err := api.CreateObjectUploadSession(context.Background(), "upload-token", "object-1", 64*1024*1024) if err != nil { t.Fatal(err) @@ -150,3 +236,12 @@ func TestMultipartUploadSessionClientMethods(t *testing.T) { t.Fatalf("unexpected patch bodies: complete=%#v abort=%#v", completeBody, abortBody) } } + +func mustClient(t *testing.T, baseURL string, token string) *Client { + t.Helper() + api, err := New(baseURL, token) + if err != nil { + t.Fatal(err) + } + return api +} diff --git a/downloader/internal/engine/aria2.go b/downloader/internal/engine/aria2.go new file mode 100644 index 00000000..eb5e172f --- /dev/null +++ b/downloader/internal/engine/aria2.go @@ -0,0 +1,584 @@ +package engine + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "strings" + "time" + + "github.com/Braurbeki/arigo" + "github.com/cenkalti/rpc2" + "github.com/saltbo/zpan/downloader/internal/client" +) + +type Aria2 struct { + URL string + Secret string + Dir string + RetainSeed bool +} + +func (a Aria2) Name() string { + return "aria2" +} + +func (a Aria2) Capabilities() []string { + return []string{"http", "magnet", "torrent"} +} + +func (a Aria2) Start(ctx context.Context) (*exec.Cmd, error) { + path, err := exec.LookPath("aria2c") + if err != nil { + return nil, err + } + rpcURL, err := parseLocalEngineURL(a.URL, "6800") + if err != nil { + return nil, err + } + args := []string{ + "--enable-rpc=true", + "--rpc-listen-all=false", + "--rpc-listen-port=" + rpcURL.port, + "--dir=" + a.Dir, + "--continue=true", + "--allow-overwrite=true", + "--auto-file-renaming=false", + } + if a.Secret != "" { + args = append(args, "--rpc-secret="+a.Secret) + } + cmd := exec.CommandContext(ctx, path, args...) + if err := cmd.Start(); err != nil { + return nil, err + } + go func() { _ = cmd.Wait() }() + return cmd, nil +} + +func (a Aria2) Check(ctx context.Context) error { + client, err := a.client(ctx) + if err != nil { + return err + } + defer client.Close() + version, err := client.GetVersion() + if err != nil { + return err + } + if version.Version == "" { + return errors.New("aria2 rpc did not return a version") + } + return nil +} + +func (a Aria2) Recover(ctx context.Context, task client.DownloadTask) (Result, bool, error) { + aria, err := a.client(ctx) + if err == nil { + defer aria.Close() + status, ok, findErr := a.findTask(ctx, &aria, task) + if findErr != nil { + return Result{}, false, findErr + } + if ok && string(status.Status) == string(arigo.StatusCompleted) { + files, err := a.getAria2Files(ctx, &aria, status.GID) + if err != nil { + return Result{}, false, err + } + result, err := resultFromAria2Files(task, filepath.Join(a.Dir, task.ID), status.BitTorrent.Info.Name, files) + return result, err == nil, err + } + } + return recoverFromTaskDir(task, a.Dir) +} + +func (a Aria2) Download(ctx context.Context, task client.DownloadTask, progress Progress) (Result, error) { + taskDir := filepath.Join(a.Dir, task.ID) + if err := os.MkdirAll(taskDir, 0o755); err != nil { + return Result{}, err + } + + aria, err := a.client(ctx) + if err != nil { + return Result{}, err + } + defer aria.Close() + + status, ok, err := a.findTask(ctx, &aria, task) + if err != nil { + return Result{}, err + } + if ok { + if string(status.Status) == string(arigo.StatusPaused) { + _ = aria.Unpause(status.GID) + } + return a.waitResult(ctx, &aria, task, taskDir, status.GID, progress) + } + + options := &arigo.Options{ + Dir: taskDir, + GID: aria2TaskGID(task.ID), + FollowTorrent: true, + BTSaveMetadata: true, + Continue: true, + SeedRatio: 0, + SeedTime: 0, + AllowOverwrite: true, + AutoFileRenaming: false, + } + if a.RetainSeed && task.SourceType != "http" { + options.SeedTime = 1000000 + } + if task.Name != "" && task.SourceType == "http" { + options.Out = task.Name + } + gid, err := aria.AddURI(arigo.URIs(task.SourceURI), options) + if err != nil { + status, ok, findErr := a.findTask(ctx, &aria, task) + if findErr != nil { + return Result{}, findErr + } + if !ok { + return Result{}, err + } + gid.GID = status.GID + } + return a.waitResult(ctx, &aria, task, taskDir, gid.GID, progress) +} + +func (a Aria2) waitResult(ctx context.Context, aria **arigo.Client, task client.DownloadTask, taskDir string, gid string, progress Progress) (Result, error) { + initialProgress := progress + if task.SourceType != "http" { + initialProgress = func(downloaded int64, total *int64, bps int64, detail *client.DownloadTaskDetail) error { return nil } + } + status, err := a.waitAria2(ctx, aria, gid, initialProgress) + if err != nil { + _ = (*aria).Remove(gid) + return Result{}, err + } + if len(status.FollowedBy) > 0 { + childGID := status.FollowedBy[0] + status, err = a.waitAria2(ctx, aria, childGID, progress) + if err != nil { + _ = (*aria).Remove(childGID) + return Result{}, err + } + } + files, err := a.getAria2Files(ctx, aria, status.GID) + if err != nil { + return Result{}, err + } + result, err := resultFromAria2Files(task, taskDir, status.BitTorrent.Info.Name, files) + if err != nil { + return Result{}, err + } + if a.RetainSeed && task.SourceType != "http" { + result.Seed = &Seed{ + Engine: "aria2", + ID: status.GID, + Path: taskDir, + Snapshot: a.seedSnapshot(status.GID), + Cleanup: a.cleanupSeed(status.GID, taskDir), + } + return result, nil + } + _ = (*aria).ForceRemove(status.GID) + _ = (*aria).RemoveDownloadResult(status.GID) + return result, nil +} + +func (a Aria2) client(ctx context.Context) (*arigo.Client, error) { + return arigo.DialContext(ctx, a.URL, a.Secret) +} + +func (a Aria2) findTask(ctx context.Context, aria **arigo.Client, task client.DownloadTask) (arigo.Status, bool, error) { + gid := aria2TaskGID(task.ID) + status, err := (*aria).TellStatus(gid) + if err == nil { + return status, true, nil + } + if isAria2RPCDisconnected(err) { + if err := a.reconnect(ctx, aria); err != nil { + return arigo.Status{}, false, err + } + status, err = (*aria).TellStatus(gid) + if err == nil { + return status, true, nil + } + } + statuses, err := a.taskStatuses(ctx, aria) + if err != nil { + return arigo.Status{}, false, err + } + taskDir := filepath.Clean(filepath.Join(a.Dir, task.ID)) + for _, status := range statuses { + if aria2StatusMatchesTask(status, taskDir, gid) { + return status, true, nil + } + } + return arigo.Status{}, false, nil +} + +func (a Aria2) taskStatuses(ctx context.Context, aria **arigo.Client) ([]arigo.Status, error) { + active, err := (*aria).TellActive() + if err != nil { + if !isAria2RPCDisconnected(err) { + return nil, err + } + if err := a.reconnect(ctx, aria); err != nil { + return nil, err + } + active, err = (*aria).TellActive() + if err != nil { + return nil, err + } + } + waiting, err := (*aria).TellWaiting(0, 1000) + if err != nil { + return nil, err + } + stopped, err := (*aria).TellStopped(0, 1000) + if err != nil { + return nil, err + } + statuses := make([]arigo.Status, 0, len(active)+len(waiting)+len(stopped)) + statuses = append(statuses, active...) + statuses = append(statuses, waiting...) + statuses = append(statuses, stopped...) + return statuses, nil +} + +func aria2TaskGID(taskID string) string { + sum := sha256.Sum256([]byte(taskID)) + return hex.EncodeToString(sum[:])[:16] +} + +func aria2StatusMatchesTask(status arigo.Status, taskDir string, gid string) bool { + if status.GID == gid || status.Following == gid || status.BelongsTo == gid { + return true + } + if filepath.Clean(status.Dir) == taskDir { + return true + } + for _, file := range status.Files { + if file.Path == "" { + continue + } + abs, _ := downloadedPath(taskDir, file.Path) + if strings.HasPrefix(filepath.Clean(abs), taskDir+string(filepath.Separator)) { + return true + } + } + return false +} + +func (a Aria2) seedSnapshot(gid string) func(context.Context) (SeedSnapshot, error) { + return func(ctx context.Context) (SeedSnapshot, error) { + aria, err := a.client(ctx) + if err != nil { + return SeedSnapshot{}, err + } + defer aria.Close() + status, err := aria.TellStatus(gid) + if err != nil { + return SeedSnapshot{}, err + } + peers := a.getAria2Peers(ctx, &aria, gid) + total := int64(status.TotalLength) + var totalPtr *int64 + if total > 0 { + totalPtr = &total + } + detail := aria2Detail(status, peers) + detail.Phase = "seeding" + return SeedSnapshot{ + Downloaded: int64(status.CompletedLength), + Total: totalPtr, + Bps: int64(status.DownloadSpeed), + Detail: detail, + }, nil + } +} + +func (a Aria2) cleanupSeed(gid string, localPath string) func(context.Context) error { + return func(ctx context.Context) error { + var errs []error + aria, err := a.client(ctx) + if err != nil { + errs = append(errs, err) + } else { + _ = aria.ForceRemove(gid) + _ = aria.RemoveDownloadResult(gid) + _ = aria.Close() + } + if err := os.RemoveAll(localPath); err != nil { + errs = append(errs, err) + } + return errors.Join(errs...) + } +} + +func (a Aria2) waitAria2(ctx context.Context, aria **arigo.Client, gid string, progress Progress) (arigo.Status, error) { + ticker := time.NewTicker(time.Second) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return arigo.Status{}, ctx.Err() + case <-ticker.C: + status, err := (*aria).TellStatus(gid) + if err != nil { + if isAria2RPCDisconnected(err) { + if err := a.reconnect(ctx, aria); err != nil { + return arigo.Status{}, err + } + continue + } + return arigo.Status{}, err + } + total := int64(status.TotalLength) + completed := int64(status.CompletedLength) + bps := int64(status.DownloadSpeed) + var totalPtr *int64 + if total > 0 { + totalPtr = &total + } + peers := a.getAria2Peers(ctx, aria, gid) + if err := progress(completed, totalPtr, bps, aria2Detail(status, peers)); err != nil { + _ = (*aria).ForcePause(gid) + return arigo.Status{}, err + } + switch string(status.Status) { + case "complete", string(arigo.StatusCompleted): + if len(status.FollowedBy) == 0 && !hasAria2LocalFile(status.Files) { + continue + } + return status, nil + case string(arigo.StatusActive): + if total > 0 && completed >= total && hasAria2LocalFile(status.Files) { + return status, nil + } + case string(arigo.StatusError), string(arigo.StatusRemoved): + if status.ErrorMessage != "" { + return arigo.Status{}, errors.New(status.ErrorMessage) + } + return arigo.Status{}, fmt.Errorf("aria2 download ended with status %s", status.Status) + } + } + } +} + +func (a Aria2) getAria2Peers(ctx context.Context, aria **arigo.Client, gid string) []arigo.Peer { + peers, err := (*aria).GetPeers(gid) + if err == nil { + return peers + } + if !isAria2RPCDisconnected(err) { + return nil + } + if err := a.reconnect(ctx, aria); err != nil { + return nil + } + peers, err = (*aria).GetPeers(gid) + if err != nil { + return nil + } + return peers +} + +func (a Aria2) getAria2Files(ctx context.Context, aria **arigo.Client, gid string) ([]arigo.File, error) { + files, err := (*aria).GetFiles(gid) + if err == nil { + return files, nil + } + if !isAria2RPCDisconnected(err) { + return nil, err + } + if err := a.reconnect(ctx, aria); err != nil { + return nil, err + } + return (*aria).GetFiles(gid) +} + +func (a Aria2) reconnect(ctx context.Context, aria **arigo.Client) error { + _ = (*aria).Close() + next, err := a.client(ctx) + if err != nil { + return err + } + *aria = next + return nil +} + +func isAria2RPCDisconnected(err error) bool { + return errors.Is(err, rpc2.ErrShutdown) || errors.Is(err, io.ErrClosedPipe) || strings.Contains(err.Error(), "connection is shut down") +} + +func aria2Detail(status arigo.Status, peers []arigo.Peer) *client.DownloadTaskDetail { + connections := int64(status.Connections) + seeders := int64(status.NumSeeders) + peerCount := int64(len(peers)) + leechers := aria2Leechers(peers) + uploaded := int64(status.UploadLength) + uploadBps := int64(status.UploadSpeed) + detail := &client.DownloadTaskDetail{ + Engine: "aria2", + Phase: aria2Phase(string(status.Status), status.FollowedBy), + EngineState: string(status.Status), + ETASeconds: aria2ETA(status), + Connections: &connections, + InfoHash: status.InfoHash, + TorrentName: status.BitTorrent.Info.Name, + Seeders: &seeders, + Leechers: leechers, + Peers: &peerCount, + PeerUploadedBytes: &uploaded, + PeerUploadBps: &uploadBps, + Trackers: aria2Trackers(status.BitTorrent.AnnounceList), + PeerSamples: aria2Peers(peers), + Files: aria2Files(status.Files), + } + if status.ErrorMessage != "" { + detail.Message = status.ErrorMessage + } + return detail +} + +func aria2ETA(status arigo.Status) *int64 { + total := int64(status.TotalLength) + completed := int64(status.CompletedLength) + bps := int64(status.DownloadSpeed) + if total <= 0 || completed >= total || bps <= 0 { + return nil + } + remaining := total - completed + eta := (remaining + bps - 1) / bps + return &eta +} + +func aria2Phase(state string, followedBy []string) string { + switch state { + case string(arigo.StatusWaiting): + if len(followedBy) > 0 { + return "metadata" + } + return "downloading" + case string(arigo.StatusActive): + return "downloading" + case "complete", string(arigo.StatusCompleted): + return "completed" + case string(arigo.StatusError), string(arigo.StatusRemoved): + return "error" + default: + return "downloading" + } +} + +func aria2Trackers(announceList [][]string) []client.DownloadTaskTracker { + trackers := make([]client.DownloadTaskTracker, 0, 20) + seen := map[string]struct{}{} + for _, tier := range announceList { + for _, url := range tier { + if url == "" { + continue + } + if _, exists := seen[url]; exists { + continue + } + seen[url] = struct{}{} + trackers = append(trackers, client.DownloadTaskTracker{ + URL: url, + Status: "announce", + Message: "aria2 exposes announce URLs only", + }) + if len(trackers) >= 20 { + return trackers + } + } + } + return trackers +} + +func aria2Leechers(peers []arigo.Peer) *int64 { + if len(peers) == 0 { + return nil + } + var count int64 + for _, peer := range peers { + if !peer.Seeder { + count++ + } + } + return &count +} + +func aria2Peers(peers []arigo.Peer) []client.DownloadTaskPeer { + out := make([]client.DownloadTaskPeer, 0, min(len(peers), 20)) + for _, peer := range peers { + if peer.IP == "" { + continue + } + down := int64(peer.DownloadSpeed) + up := int64(peer.UploadSpeed) + out = append(out, client.DownloadTaskPeer{ + Address: fmt.Sprintf("%s:%d", peer.IP, peer.Port), + DownloadBps: &down, + UploadBps: &up, + }) + if len(out) >= 20 { + break + } + } + return out +} + +func aria2Files(files []arigo.File) []client.DownloadTaskFile { + out := make([]client.DownloadTaskFile, 0, min(len(files), 50)) + for _, file := range files { + if file.Path == "" || isAria2MetadataPath(file.Path) { + continue + } + size := int64(file.Length) + completed := int64(file.CompletedLength) + selected := file.Selected + out = append(out, client.DownloadTaskFile{ + Path: file.Path, + Size: size, + CompletedBytes: &completed, + Selected: &selected, + }) + if len(out) >= 50 { + break + } + } + return out +} + +func hasAria2LocalFile(files []arigo.File) bool { + for _, file := range files { + if file.Path != "" && !isAria2MetadataPath(file.Path) { + return true + } + } + return false +} + +func resultFromAria2Files(task client.DownloadTask, taskDir string, fallbackName string, files []arigo.File) (Result, error) { + downloaded := make([]downloadedFile, 0, len(files)) + for _, file := range files { + if file.Selected && file.Length > 0 && !isDownloadSidecarPath(file.Path) { + abs, rel := downloadedPath(taskDir, file.Path) + downloaded = append(downloaded, downloadedFile{path: abs, relativePath: rel}) + } + } + if len(downloaded) == 0 { + return resultFromPath(task, taskDir, fallbackName) + } + return resultFromDownloadedFiles(task, taskDir, fallbackName, downloaded) +} diff --git a/downloader/internal/engine/engine.go b/downloader/internal/engine/engine.go index 73332c7b..f9b8f1e4 100644 --- a/downloader/internal/engine/engine.go +++ b/downloader/internal/engine/engine.go @@ -2,19 +2,12 @@ package engine import ( "context" - "errors" - "fmt" - "io" - "net/http" "net/url" "os" "path/filepath" "strings" "time" - "github.com/Braurbeki/arigo" - qbittorrent "github.com/autobrr/go-qbittorrent" - "github.com/cenkalti/rpc2" "github.com/saltbo/zpan/downloader/internal/client" ) @@ -44,376 +37,13 @@ type SeedSnapshot struct { type Progress func(downloaded int64, total *int64, bps int64, detail *client.DownloadTaskDetail) error type Engine interface { + Name() string + Capabilities() []string Check(ctx context.Context) error + Recover(ctx context.Context, task client.DownloadTask) (Result, bool, error) Download(ctx context.Context, task client.DownloadTask, progress Progress) (Result, error) } -type HTTP struct { - Dir string -} - -func (h HTTP) Check(ctx context.Context) error { - if err := os.MkdirAll(h.Dir, 0o755); err != nil { - return err - } - file, err := os.CreateTemp(h.Dir, ".zpan-check-*") - if err != nil { - return err - } - path := file.Name() - if err := file.Close(); err != nil { - return err - } - return os.Remove(path) -} - -func (h HTTP) Download(ctx context.Context, task client.DownloadTask, progress Progress) (Result, error) { - if task.SourceType != "http" { - return Result{}, errors.New("http engine only supports http sources") - } - taskDir := filepath.Join(h.Dir, task.ID) - if err := os.MkdirAll(taskDir, 0o755); err != nil { - return Result{}, err - } - - req, err := http.NewRequestWithContext(ctx, http.MethodGet, task.SourceURI, nil) - if err != nil { - return Result{}, err - } - res, err := http.DefaultClient.Do(req) - if err != nil { - return Result{}, err - } - defer res.Body.Close() - if res.StatusCode < 200 || res.StatusCode >= 300 { - return Result{}, errors.New(res.Status) - } - - name := outputName(task, filenameFromURL(req.URL)) - path := filepath.Join(taskDir, name) - file, err := os.Create(path) - if err != nil { - return Result{}, err - } - defer file.Close() - - var total *int64 - if res.ContentLength > 0 { - total = &res.ContentLength - } - counter := &progressWriter{progress: progress, total: total, lastAt: time.Now()} - if _, err := io.Copy(file, io.TeeReader(res.Body, counter)); err != nil { - return Result{}, err - } - if err := progress(counter.downloaded, total, 0, &client.DownloadTaskDetail{Engine: "builtin", Phase: "completed"}); err != nil { - return Result{}, err - } - return Result{Path: path, Name: name, Size: counter.downloaded}, nil -} - -type Aria2 struct { - URL string - Secret string - Dir string - RetainSeed bool -} - -func (a Aria2) Check(ctx context.Context) error { - client, err := a.client(ctx) - if err != nil { - return err - } - defer client.Close() - version, err := client.GetVersion() - if err != nil { - return err - } - if version.Version == "" { - return errors.New("aria2 rpc did not return a version") - } - return nil -} - -func (a Aria2) Download(ctx context.Context, task client.DownloadTask, progress Progress) (Result, error) { - taskDir := filepath.Join(a.Dir, task.ID) - if err := os.MkdirAll(taskDir, 0o755); err != nil { - return Result{}, err - } - - aria, err := a.client(ctx) - if err != nil { - return Result{}, err - } - defer aria.Close() - - options := &arigo.Options{ - Dir: taskDir, - FollowTorrent: true, - BTSaveMetadata: true, - SeedRatio: 0, - SeedTime: 0, - AllowOverwrite: true, - AutoFileRenaming: false, - } - if a.RetainSeed && task.SourceType != "http" { - options.SeedTime = 1000000 - } - if task.Name != "" && task.SourceType == "http" { - options.Out = task.Name - } - gid, err := aria.AddURI(arigo.URIs(task.SourceURI), options) - if err != nil { - return Result{}, err - } - initialProgress := progress - if task.SourceType != "http" { - initialProgress = func(downloaded int64, total *int64, bps int64, detail *client.DownloadTaskDetail) error { return nil } - } - status, err := a.waitAria2(ctx, &aria, gid.GID, initialProgress) - if err != nil { - _ = aria.Remove(gid.GID) - return Result{}, err - } - if len(status.FollowedBy) > 0 { - childGID := status.FollowedBy[0] - status, err = a.waitAria2(ctx, &aria, childGID, progress) - if err != nil { - _ = aria.Remove(childGID) - return Result{}, err - } - } - files, err := a.getAria2Files(ctx, &aria, status.GID) - if err != nil { - return Result{}, err - } - result, err := resultFromAria2Files(task, taskDir, status.BitTorrent.Info.Name, files) - if err != nil { - return Result{}, err - } - if a.RetainSeed && task.SourceType != "http" { - result.Seed = &Seed{ - Engine: "aria2", - ID: status.GID, - Path: taskDir, - Snapshot: a.seedSnapshot(status.GID), - Cleanup: a.cleanupSeed(status.GID, taskDir), - } - return result, nil - } - _ = aria.ForceRemove(status.GID) - _ = aria.RemoveDownloadResult(status.GID) - return result, nil -} - -func (a Aria2) client(ctx context.Context) (*arigo.Client, error) { - return arigo.DialContext(ctx, a.URL, a.Secret) -} - -func (a Aria2) seedSnapshot(gid string) func(context.Context) (SeedSnapshot, error) { - return func(ctx context.Context) (SeedSnapshot, error) { - aria, err := a.client(ctx) - if err != nil { - return SeedSnapshot{}, err - } - defer aria.Close() - status, err := aria.TellStatus(gid) - if err != nil { - return SeedSnapshot{}, err - } - peers := a.getAria2Peers(ctx, &aria, gid) - total := int64(status.TotalLength) - var totalPtr *int64 - if total > 0 { - totalPtr = &total - } - detail := aria2Detail(status, peers) - detail.Phase = "seeding" - return SeedSnapshot{ - Downloaded: int64(status.CompletedLength), - Total: totalPtr, - Bps: int64(status.DownloadSpeed), - Detail: detail, - }, nil - } -} - -func (a Aria2) cleanupSeed(gid string, localPath string) func(context.Context) error { - return func(ctx context.Context) error { - var errs []error - aria, err := a.client(ctx) - if err != nil { - errs = append(errs, err) - } else { - _ = aria.ForceRemove(gid) - _ = aria.RemoveDownloadResult(gid) - _ = aria.Close() - } - if err := os.RemoveAll(localPath); err != nil { - errs = append(errs, err) - } - return errors.Join(errs...) - } -} - -type QBittorrent struct { - URL string - Username string - Password string - Dir string - RetainSeed bool -} - -func (q QBittorrent) Check(ctx context.Context) error { - req, err := http.NewRequestWithContext(ctx, http.MethodGet, strings.TrimRight(q.URL, "/")+"/api/v2/app/version", nil) - if err != nil { - return err - } - res, err := (&http.Client{Timeout: 2 * time.Second}).Do(req) - if err != nil { - return err - } - defer res.Body.Close() - if res.StatusCode < 200 || res.StatusCode >= 300 { - return fmt.Errorf("qbittorrent web api returned %s", res.Status) - } - version, err := io.ReadAll(io.LimitReader(res.Body, 256)) - if err != nil { - return err - } - if strings.TrimSpace(string(version)) == "" { - return errors.New("qbittorrent web api did not return a version") - } - return nil -} - -func (q QBittorrent) login(ctx context.Context) (*qbittorrent.Client, error) { - qbt := qbittorrent.NewClient(qbittorrent.Config{ - Host: q.URL, - Username: q.Username, - Password: q.Password, - Timeout: 10, - }) - if err := qbt.LoginCtx(ctx); err != nil { - return nil, err - } - return qbt, nil -} - -func (q QBittorrent) Download(ctx context.Context, task client.DownloadTask, progress Progress) (Result, error) { - if task.SourceType == "http" { - return HTTP{Dir: q.Dir}.Download(ctx, task, progress) - } - taskDir := filepath.Join(q.Dir, task.ID) - if err := os.MkdirAll(taskDir, 0o755); err != nil { - return Result{}, err - } - - qbt, err := q.login(ctx) - if err != nil { - return Result{}, err - } - - tag := qbittorrentTrackingTag(task.ID) - options := qbittorrentAddOptions(task, taskDir, tag) - if _, err := qbt.AddTorrentFromUrlCtx(ctx, task.SourceURI, options); err != nil { - return Result{}, err - } - - torrent, err := waitQBittorrent(ctx, qbt, tag, progress) - if err != nil { - return Result{}, err - } - result, err := resultFromQBittorrentFiles(ctx, qbt, task, taskDir, torrent) - if err != nil { - return Result{}, err - } - if q.RetainSeed { - result.Seed = &Seed{ - Engine: "qbittorrent", - ID: torrent.Hash, - Path: taskDir, - Snapshot: q.seedSnapshot(torrent.Hash), - Cleanup: q.cleanupSeed(torrent.Hash, taskDir), - } - return result, nil - } - _ = qbt.DeleteTorrentsCtx(ctx, []string{torrent.Hash}, false) - return result, nil -} - -func qbittorrentAddOptions(task client.DownloadTask, taskDir string, trackingTag string) map[string]string { - category := "zpan" - if task.Category != "" { - category = task.Category - } - tags := append([]string{trackingTag}, task.Tags...) - options := (&qbittorrent.TorrentAddOptions{ - SavePath: taskDir, - Category: category, - Tags: strings.Join(tags, ","), - LimitRatio: 0, - LimitSeedTime: 0, - SequentialDownload: false, - }).Prepare() - if name := requestedOutputName(task); name != "" { - options["rename"] = name - } - return options -} - -func qbittorrentTrackingTag(taskID string) string { - return "ztid=" + taskID -} - -func (q QBittorrent) cleanupSeed(hash string, localPath string) func(context.Context) error { - return func(ctx context.Context) error { - var errs []error - qbt, err := q.login(ctx) - if err != nil { - errs = append(errs, err) - } else if err := qbt.DeleteTorrentsCtx(ctx, []string{hash}, false); err != nil { - errs = append(errs, err) - } - if err := os.RemoveAll(localPath); err != nil { - errs = append(errs, err) - } - return errors.Join(errs...) - } -} - -func (q QBittorrent) seedSnapshot(hash string) func(context.Context) (SeedSnapshot, error) { - return func(ctx context.Context) (SeedSnapshot, error) { - qbt, err := q.login(ctx) - if err != nil { - return SeedSnapshot{}, err - } - torrents, err := qbt.GetTorrentsCtx(ctx, qbittorrent.TorrentFilterOptions{Hashes: []string{hash}}) - if err != nil { - return SeedSnapshot{}, err - } - if len(torrents) == 0 { - return SeedSnapshot{}, fmt.Errorf("qbittorrent torrent %s not found", hash) - } - torrent := torrents[0] - total := torrent.TotalSize - if total <= 0 { - total = torrent.Size - } - var totalPtr *int64 - if total > 0 { - totalPtr = &total - } - detail := qbittorrentDetail(ctx, qbt, torrent) - detail.Phase = "seeding" - return SeedSnapshot{ - Downloaded: torrent.Completed, - Total: totalPtr, - Bps: torrent.DlSpeed, - Detail: detail, - }, nil - } -} - type progressWriter struct { progress Progress total *int64 @@ -437,439 +67,6 @@ func (p *progressWriter) Write(data []byte) (int, error) { return n, nil } -func (a Aria2) waitAria2(ctx context.Context, aria **arigo.Client, gid string, progress Progress) (arigo.Status, error) { - ticker := time.NewTicker(time.Second) - defer ticker.Stop() - for { - select { - case <-ctx.Done(): - return arigo.Status{}, ctx.Err() - case <-ticker.C: - status, err := (*aria).TellStatus(gid) - if err != nil { - if isAria2RPCDisconnected(err) { - if err := a.reconnect(ctx, aria); err != nil { - return arigo.Status{}, err - } - continue - } - return arigo.Status{}, err - } - total := int64(status.TotalLength) - completed := int64(status.CompletedLength) - bps := int64(status.DownloadSpeed) - var totalPtr *int64 - if total > 0 { - totalPtr = &total - } - peers := a.getAria2Peers(ctx, aria, gid) - if err := progress(completed, totalPtr, bps, aria2Detail(status, peers)); err != nil { - _ = (*aria).ForcePause(gid) - return arigo.Status{}, err - } - switch string(status.Status) { - case "complete", string(arigo.StatusCompleted): - if len(status.FollowedBy) == 0 && !hasAria2LocalFile(status.Files) { - continue - } - return status, nil - case string(arigo.StatusActive): - if total > 0 && completed >= total && hasAria2LocalFile(status.Files) { - return status, nil - } - case string(arigo.StatusError), string(arigo.StatusRemoved): - if status.ErrorMessage != "" { - return arigo.Status{}, errors.New(status.ErrorMessage) - } - return arigo.Status{}, fmt.Errorf("aria2 download ended with status %s", status.Status) - } - } - } -} - -func (a Aria2) getAria2Peers(ctx context.Context, aria **arigo.Client, gid string) []arigo.Peer { - peers, err := (*aria).GetPeers(gid) - if err == nil { - return peers - } - if !isAria2RPCDisconnected(err) { - return nil - } - if err := a.reconnect(ctx, aria); err != nil { - return nil - } - peers, err = (*aria).GetPeers(gid) - if err != nil { - return nil - } - return peers -} - -func (a Aria2) getAria2Files(ctx context.Context, aria **arigo.Client, gid string) ([]arigo.File, error) { - files, err := (*aria).GetFiles(gid) - if err == nil { - return files, nil - } - if !isAria2RPCDisconnected(err) { - return nil, err - } - if err := a.reconnect(ctx, aria); err != nil { - return nil, err - } - return (*aria).GetFiles(gid) -} - -func (a Aria2) reconnect(ctx context.Context, aria **arigo.Client) error { - _ = (*aria).Close() - next, err := a.client(ctx) - if err != nil { - return err - } - *aria = next - return nil -} - -func isAria2RPCDisconnected(err error) bool { - return errors.Is(err, rpc2.ErrShutdown) || errors.Is(err, io.ErrClosedPipe) || strings.Contains(err.Error(), "connection is shut down") -} - -func aria2Detail(status arigo.Status, peers []arigo.Peer) *client.DownloadTaskDetail { - connections := int64(status.Connections) - seeders := int64(status.NumSeeders) - peerCount := int64(len(peers)) - leechers := aria2Leechers(peers) - uploaded := int64(status.UploadLength) - uploadBps := int64(status.UploadSpeed) - detail := &client.DownloadTaskDetail{ - Engine: "aria2", - Phase: aria2Phase(string(status.Status), status.FollowedBy), - EngineState: string(status.Status), - ETASeconds: aria2ETA(status), - Connections: &connections, - InfoHash: status.InfoHash, - TorrentName: status.BitTorrent.Info.Name, - Seeders: &seeders, - Leechers: leechers, - Peers: &peerCount, - PeerUploadedBytes: &uploaded, - PeerUploadBps: &uploadBps, - Trackers: aria2Trackers(status.BitTorrent.AnnounceList), - PeerSamples: aria2Peers(peers), - Files: aria2Files(status.Files), - } - if status.ErrorMessage != "" { - detail.Message = status.ErrorMessage - } - return detail -} - -func aria2ETA(status arigo.Status) *int64 { - total := int64(status.TotalLength) - completed := int64(status.CompletedLength) - bps := int64(status.DownloadSpeed) - if total <= 0 || completed >= total || bps <= 0 { - return nil - } - remaining := total - completed - eta := (remaining + bps - 1) / bps - return &eta -} - -func aria2Phase(state string, followedBy []string) string { - switch state { - case string(arigo.StatusWaiting): - if len(followedBy) > 0 { - return "metadata" - } - return "downloading" - case string(arigo.StatusActive): - return "downloading" - case "complete", string(arigo.StatusCompleted): - return "completed" - case string(arigo.StatusError), string(arigo.StatusRemoved): - return "error" - default: - return "downloading" - } -} - -func aria2Trackers(announceList [][]string) []client.DownloadTaskTracker { - trackers := make([]client.DownloadTaskTracker, 0, 20) - seen := map[string]struct{}{} - for _, tier := range announceList { - for _, url := range tier { - if url == "" { - continue - } - if _, exists := seen[url]; exists { - continue - } - seen[url] = struct{}{} - trackers = append(trackers, client.DownloadTaskTracker{ - URL: url, - Status: "announce", - Message: "aria2 exposes announce URLs only", - }) - if len(trackers) >= 20 { - return trackers - } - } - } - return trackers -} - -func aria2Leechers(peers []arigo.Peer) *int64 { - if len(peers) == 0 { - return nil - } - var count int64 - for _, peer := range peers { - if !peer.Seeder { - count++ - } - } - return &count -} - -func aria2Peers(peers []arigo.Peer) []client.DownloadTaskPeer { - out := make([]client.DownloadTaskPeer, 0, min(len(peers), 20)) - for _, peer := range peers { - if peer.IP == "" { - continue - } - down := int64(peer.DownloadSpeed) - up := int64(peer.UploadSpeed) - out = append(out, client.DownloadTaskPeer{ - Address: fmt.Sprintf("%s:%d", peer.IP, peer.Port), - DownloadBps: &down, - UploadBps: &up, - }) - if len(out) >= 20 { - break - } - } - return out -} - -func aria2Files(files []arigo.File) []client.DownloadTaskFile { - out := make([]client.DownloadTaskFile, 0, min(len(files), 50)) - for _, file := range files { - if file.Path == "" || isAria2MetadataPath(file.Path) { - continue - } - size := int64(file.Length) - completed := int64(file.CompletedLength) - selected := file.Selected - out = append(out, client.DownloadTaskFile{ - Path: file.Path, - Size: size, - CompletedBytes: &completed, - Selected: &selected, - }) - if len(out) >= 50 { - break - } - } - return out -} - -func hasAria2LocalFile(files []arigo.File) bool { - for _, file := range files { - if file.Path != "" && !isAria2MetadataPath(file.Path) { - return true - } - } - return false -} - -func waitQBittorrent( - ctx context.Context, - qbt *qbittorrent.Client, - tag string, - progress Progress, -) (qbittorrent.Torrent, error) { - ticker := time.NewTicker(time.Second) - defer ticker.Stop() - for { - select { - case <-ctx.Done(): - return qbittorrent.Torrent{}, ctx.Err() - case <-ticker.C: - torrents, err := qbt.GetTorrentsCtx(ctx, qbittorrent.TorrentFilterOptions{Tag: tag}) - if err != nil { - return qbittorrent.Torrent{}, err - } - if len(torrents) == 0 { - continue - } - torrent := torrents[0] - total := torrent.TotalSize - if total <= 0 { - total = torrent.Size - } - var totalPtr *int64 - if total > 0 { - totalPtr = &total - } - if err := progress(torrent.Completed, totalPtr, torrent.DlSpeed, qbittorrentDetail(ctx, qbt, torrent)); err != nil { - _ = qbt.StopCtx(ctx, []string{torrent.Hash}) - return qbittorrent.Torrent{}, err - } - if torrent.Progress >= 1 || (torrent.AmountLeft == 0 && total > 0) { - return torrent, nil - } - if isQBittorrentErrorState(torrent.State) { - return qbittorrent.Torrent{}, fmt.Errorf("qbittorrent download ended with state %s", torrent.State) - } - } - } -} - -func qbittorrentDetail(ctx context.Context, qbt *qbittorrent.Client, torrent qbittorrent.Torrent) *client.DownloadTaskDetail { - connections := int64(torrent.NumSeeds + torrent.NumLeechs) - seeders := torrent.NumSeeds - leechers := torrent.NumLeechs - peers := torrent.NumComplete + torrent.NumIncomplete - uploaded := torrent.Uploaded - uploadBps := torrent.UpSpeed - var eta *int64 - if torrent.ETA >= 0 { - eta = &torrent.ETA - } - return &client.DownloadTaskDetail{ - Engine: "qbittorrent", - Phase: qbittorrentPhase(string(torrent.State)), - EngineState: string(torrent.State), - ETASeconds: eta, - Connections: &connections, - InfoHash: torrent.Hash, - TorrentName: torrent.Name, - Seeders: &seeders, - Leechers: &leechers, - Peers: &peers, - PeerUploadedBytes: &uploaded, - PeerUploadBps: &uploadBps, - Trackers: qbittorrentTrackers(ctx, qbt, torrent), - PeerSamples: qbittorrentPeers(ctx, qbt, torrent.Hash), - } -} - -func qbittorrentPhase(state string) string { - normalized := strings.ToLower(state) - switch { - case strings.Contains(normalized, "meta"): - return "metadata" - case strings.Contains(normalized, "up"), strings.Contains(normalized, "seed"): - return "seeding" - case strings.Contains(normalized, "error"), strings.Contains(normalized, "missing"): - return "error" - case strings.Contains(normalized, "paused"): - return "downloading" - case normalized == "uploading": - return "seeding" - default: - return "downloading" - } -} - -func qbittorrentTrackers(ctx context.Context, qbt *qbittorrent.Client, torrent qbittorrent.Torrent) []client.DownloadTaskTracker { - trackers := torrent.Trackers - if len(trackers) == 0 && torrent.Hash != "" { - loaded, err := qbt.GetTorrentTrackersCtx(ctx, torrent.Hash) - if err == nil { - trackers = loaded - } - } - out := make([]client.DownloadTaskTracker, 0, min(len(trackers), 20)) - for _, tracker := range trackers { - peers := int64(tracker.NumPeers) - seeds := int64(tracker.NumSeeds) - leechers := int64(tracker.NumLeechers) - out = append(out, client.DownloadTaskTracker{ - URL: tracker.Url, - Status: fmt.Sprint(tracker.Status), - Peers: &peers, - Seeds: &seeds, - Leechers: &leechers, - Message: tracker.Message, - }) - if len(out) >= 20 { - break - } - } - return out -} - -func qbittorrentPeers(ctx context.Context, qbt *qbittorrent.Client, hash string) []client.DownloadTaskPeer { - if hash == "" { - return nil - } - peers, err := qbt.GetTorrentPeersCtx(ctx, hash, 0) - if err != nil || peers == nil { - return nil - } - out := make([]client.DownloadTaskPeer, 0, min(len(peers.Peers), 20)) - for address, peer := range peers.Peers { - progress := peer.Progress - down := peer.DownSpeed - up := peer.UpSpeed - label := address - if peer.IP != "" && peer.Port > 0 { - label = fmt.Sprintf("%s:%d", peer.IP, peer.Port) - } - out = append(out, client.DownloadTaskPeer{ - Address: label, - Client: peer.Client, - Progress: &progress, - DownloadBps: &down, - UploadBps: &up, - }) - if len(out) >= 20 { - break - } - } - return out -} - -func resultFromAria2Files(task client.DownloadTask, taskDir string, fallbackName string, files []arigo.File) (Result, error) { - downloaded := make([]downloadedFile, 0, len(files)) - for _, file := range files { - if file.Selected && file.Length > 0 && !isDownloadSidecarPath(file.Path) { - abs, rel := downloadedPath(taskDir, file.Path) - downloaded = append(downloaded, downloadedFile{path: abs, relativePath: rel}) - } - } - if len(downloaded) == 0 { - return resultFromPath(task, taskDir, fallbackName) - } - return resultFromDownloadedFiles(task, taskDir, fallbackName, downloaded) -} - -func resultFromQBittorrentFiles( - ctx context.Context, - qbt *qbittorrent.Client, - task client.DownloadTask, - taskDir string, - torrent qbittorrent.Torrent, -) (Result, error) { - files, err := qbt.GetFilesInformationCtx(ctx, torrent.Hash) - if err != nil || files == nil { - return resultFromPath(task, taskDir, torrent.Name) - } - downloaded := make([]downloadedFile, 0, len(*files)) - for _, file := range *files { - if file.Priority == 0 || file.Size <= 0 { - continue - } - abs, rel := downloadedPath(taskDir, file.Name) - downloaded = append(downloaded, downloadedFile{path: abs, relativePath: rel}) - } - if len(downloaded) == 0 { - return resultFromPath(task, taskDir, torrent.Name) - } - return resultFromDownloadedFiles(task, taskDir, torrent.Name, downloaded) -} - func resultFromPath(task client.DownloadTask, path string, fallbackName string) (Result, error) { info, err := os.Stat(path) if err != nil { @@ -911,6 +108,21 @@ func resultFromPath(task client.DownloadTask, path string, fallbackName string) return Result{Path: path, Name: outputName(task, fallbackName), Size: size, IsDir: true}, nil } +func recoverFromTaskDir(task client.DownloadTask, dir string) (Result, bool, error) { + taskDir := filepath.Join(dir, task.ID) + if _, err := os.Stat(taskDir); err != nil { + if os.IsNotExist(err) { + return Result{}, false, nil + } + return Result{}, false, err + } + result, err := resultFromPath(task, taskDir, requestedOutputName(task)) + if err != nil { + return Result{}, false, err + } + return result, true, nil +} + type downloadedFile struct { path string relativePath string @@ -1113,8 +325,3 @@ func filenameFromURL(parsed *url.URL) string { } return name } - -func isQBittorrentErrorState(state qbittorrent.TorrentState) bool { - value := strings.ToLower(string(state)) - return strings.Contains(value, "error") || strings.Contains(value, "missing") -} diff --git a/downloader/internal/engine/engine_test.go b/downloader/internal/engine/engine_test.go index 7cb8212b..6e3b88da 100644 --- a/downloader/internal/engine/engine_test.go +++ b/downloader/internal/engine/engine_test.go @@ -72,6 +72,49 @@ func TestHTTPRejectsMagnet(t *testing.T) { } } +func TestHTTPDownloadResumesExistingFile(t *testing.T) { + var rangeHeader string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + rangeHeader = r.Header.Get("Range") + if rangeHeader != "bytes=5-" { + t.Fatalf("expected resume range bytes=5-, got %q", rangeHeader) + } + w.Header().Set("Content-Length", "6") + w.WriteHeader(http.StatusPartialContent) + _, _ = w.Write([]byte(" world")) + })) + defer server.Close() + + dir := t.TempDir() + taskDir := filepath.Join(dir, "task-1") + if err := os.MkdirAll(taskDir, 0o755); err != nil { + t.Fatal(err) + } + path := filepath.Join(taskDir, "file.txt") + if err := os.WriteFile(path, []byte("hello"), 0o644); err != nil { + t.Fatal(err) + } + + result, err := (HTTP{Dir: dir}).Download( + context.Background(), + client.DownloadTask{ID: "task-1", SourceType: "http", SourceURI: server.URL + "/file.txt"}, + func(downloaded int64, total *int64, bps int64, detail *client.DownloadTaskDetail) error { return nil }, + ) + if err != nil { + t.Fatal(err) + } + data, err := os.ReadFile(result.Path) + if err != nil { + t.Fatal(err) + } + if string(data) != "hello world" { + t.Fatalf("expected resumed content, got %q", string(data)) + } + if result.Size != 11 { + t.Fatalf("expected size 11, got %d", result.Size) + } +} + func TestQBittorrentCheckUsesWebAPIVersion(t *testing.T) { server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.URL.Path != "/api/v2/app/version" { diff --git a/downloader/internal/engine/http.go b/downloader/internal/engine/http.go new file mode 100644 index 00000000..f6ad9ab7 --- /dev/null +++ b/downloader/internal/engine/http.go @@ -0,0 +1,126 @@ +package engine + +import ( + "context" + "errors" + "io" + "net/http" + "os" + "path/filepath" + "strconv" + "time" + + "github.com/saltbo/zpan/downloader/internal/client" +) + +type HTTP struct { + Dir string +} + +func (h HTTP) Name() string { + return "builtin" +} + +func (h HTTP) Capabilities() []string { + return []string{"http"} +} + +func (h HTTP) Check(ctx context.Context) error { + if err := os.MkdirAll(h.Dir, 0o755); err != nil { + return err + } + file, err := os.CreateTemp(h.Dir, ".zpan-check-*") + if err != nil { + return err + } + path := file.Name() + if err := file.Close(); err != nil { + return err + } + return os.Remove(path) +} + +func (h HTTP) Recover(ctx context.Context, task client.DownloadTask) (Result, bool, error) { + return recoverFromTaskDir(task, h.Dir) +} + +func (h HTTP) Download(ctx context.Context, task client.DownloadTask, progress Progress) (Result, error) { + if task.SourceType != "http" { + return Result{}, errors.New("http engine only supports http sources") + } + taskDir := filepath.Join(h.Dir, task.ID) + if err := os.MkdirAll(taskDir, 0o755); err != nil { + return Result{}, err + } + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, task.SourceURI, nil) + if err != nil { + return Result{}, err + } + name := outputName(task, filenameFromURL(req.URL)) + path := filepath.Join(taskDir, name) + existingSize, err := existingFileSize(path) + if err != nil { + return Result{}, err + } + if existingSize > 0 { + req.Header.Set("Range", "bytes="+strconv.FormatInt(existingSize, 10)+"-") + } + + res, err := http.DefaultClient.Do(req) + if err != nil { + return Result{}, err + } + defer res.Body.Close() + if res.StatusCode == http.StatusRequestedRangeNotSatisfiable && existingSize > 0 { + return resultFromFile(task, path) + } + if res.StatusCode < 200 || res.StatusCode >= 300 { + return Result{}, errors.New(res.Status) + } + + appendExisting := existingSize > 0 && res.StatusCode == http.StatusPartialContent + if existingSize > 0 && !appendExisting { + existingSize = 0 + } + file, err := openOutputFile(path, appendExisting) + if err != nil { + return Result{}, err + } + defer file.Close() + + var total *int64 + if res.ContentLength > 0 { + value := res.ContentLength + existingSize + total = &value + } + counter := &progressWriter{progress: progress, total: total, downloaded: existingSize, lastBytes: existingSize, lastAt: time.Now()} + if _, err := io.Copy(file, io.TeeReader(res.Body, counter)); err != nil { + return Result{}, err + } + if err := progress(counter.downloaded, total, 0, &client.DownloadTaskDetail{Engine: "builtin", Phase: "completed"}); err != nil { + return Result{}, err + } + return Result{Path: path, Name: name, Size: counter.downloaded}, nil +} + +func existingFileSize(path string) (int64, error) { + info, err := os.Stat(path) + if err == nil { + if info.IsDir() { + return 0, nil + } + return info.Size(), nil + } + if os.IsNotExist(err) { + return 0, nil + } + return 0, err +} + +func openOutputFile(path string, appendExisting bool) (*os.File, error) { + if appendExisting { + return os.OpenFile(path, os.O_WRONLY|os.O_APPEND, 0o644) + } + return os.Create(path) +} diff --git a/downloader/internal/engine/process.go b/downloader/internal/engine/process.go new file mode 100644 index 00000000..0bfae4b0 --- /dev/null +++ b/downloader/internal/engine/process.go @@ -0,0 +1,64 @@ +package engine + +import ( + "context" + "fmt" + "net/url" + "os/exec" + "strconv" + "strings" +) + +type Starter interface { + Start(ctx context.Context) (*exec.Cmd, error) +} + +type localEngineURL struct { + port string +} + +func lookPathAny(names ...string) (string, error) { + var lastErr error + for _, name := range names { + path, err := exec.LookPath(name) + if err == nil { + return path, nil + } + lastErr = err + } + return "", lastErr +} + +func parseLocalEngineURL(raw string, defaultPort string) (localEngineURL, error) { + normalized := raw + if strings.HasPrefix(normalized, "ws://") { + normalized = "http://" + strings.TrimPrefix(normalized, "ws://") + } + if strings.HasPrefix(normalized, "wss://") { + normalized = "https://" + strings.TrimPrefix(normalized, "wss://") + } + parsed, err := url.Parse(normalized) + if err != nil { + return localEngineURL{}, err + } + host := parsed.Hostname() + if host != "" && host != "127.0.0.1" && host != "localhost" && host != "::1" { + return localEngineURL{}, fmt.Errorf("auto start only supports local engine URLs, got %s", host) + } + port := parsed.Port() + if port == "" { + port = defaultPort + } + if _, err := strconv.Atoi(port); err != nil { + return localEngineURL{}, fmt.Errorf("invalid engine port %q", port) + } + return localEngineURL{port: port}, nil +} + +func filepathBase(path string) string { + parts := strings.FieldsFunc(path, func(r rune) bool { return r == '/' || r == '\\' }) + if len(parts) == 0 { + return path + } + return parts[len(parts)-1] +} diff --git a/downloader/internal/engine/qbittorrent.go b/downloader/internal/engine/qbittorrent.go new file mode 100644 index 00000000..3eaa0dd9 --- /dev/null +++ b/downloader/internal/engine/qbittorrent.go @@ -0,0 +1,455 @@ +package engine + +import ( + "context" + "errors" + "fmt" + "io" + "net/http" + "os" + "os/exec" + "path/filepath" + "strings" + "time" + + qbittorrent "github.com/autobrr/go-qbittorrent" + "github.com/saltbo/zpan/downloader/internal/client" +) + +type QBittorrent struct { + URL string + Username string + Password string + Dir string + RetainSeed bool +} + +func (q QBittorrent) Name() string { + return "qbittorrent" +} + +func (q QBittorrent) Capabilities() []string { + return []string{"http", "magnet", "torrent"} +} + +func (q QBittorrent) Start(ctx context.Context) (*exec.Cmd, error) { + path, err := lookPathAny("qbittorrent-nox", "qbittorrent") + if err != nil { + return nil, err + } + webURL, err := parseLocalEngineURL(q.URL, "8080") + if err != nil { + return nil, err + } + args := []string{} + if strings.Contains(filepathBase(path), "qbittorrent-nox") { + args = append(args, "--webui-port="+webURL.port) + } + cmd := exec.CommandContext(ctx, path, args...) + if err := cmd.Start(); err != nil { + return nil, err + } + go func() { _ = cmd.Wait() }() + return cmd, nil +} + +func (q QBittorrent) Check(ctx context.Context) error { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, strings.TrimRight(q.URL, "/")+"/api/v2/app/version", nil) + if err != nil { + return err + } + res, err := (&http.Client{Timeout: 2 * time.Second}).Do(req) + if err != nil { + return err + } + defer res.Body.Close() + if res.StatusCode < 200 || res.StatusCode >= 300 { + return fmt.Errorf("qbittorrent web api returned %s", res.Status) + } + version, err := io.ReadAll(io.LimitReader(res.Body, 256)) + if err != nil { + return err + } + if strings.TrimSpace(string(version)) == "" { + return errors.New("qbittorrent web api did not return a version") + } + return nil +} + +func (q QBittorrent) Recover(ctx context.Context, task client.DownloadTask) (Result, bool, error) { + if task.SourceType == "http" { + return HTTP{Dir: q.Dir}.Recover(ctx, task) + } + qbt, err := q.login(ctx) + if err != nil { + return Result{}, false, err + } + torrent, ok, err := q.findTask(ctx, qbt, task) + if err != nil { + return Result{}, false, err + } + if ok && (torrent.Progress >= 1 || (torrent.AmountLeft == 0 && torrent.TotalSize > 0)) { + result, err := resultFromQBittorrentFiles(ctx, qbt, task, filepath.Join(q.Dir, task.ID), torrent) + return result, err == nil, err + } + return recoverFromTaskDir(task, q.Dir) +} + +func (q QBittorrent) login(ctx context.Context) (*qbittorrent.Client, error) { + qbt := qbittorrent.NewClient(qbittorrent.Config{ + Host: q.URL, + Username: q.Username, + Password: q.Password, + Timeout: 10, + }) + if err := qbt.LoginCtx(ctx); err != nil { + return nil, err + } + return qbt, nil +} + +func (q QBittorrent) Download(ctx context.Context, task client.DownloadTask, progress Progress) (Result, error) { + if task.SourceType == "http" { + return HTTP{Dir: q.Dir}.Download(ctx, task, progress) + } + taskDir := filepath.Join(q.Dir, task.ID) + if err := os.MkdirAll(taskDir, 0o755); err != nil { + return Result{}, err + } + + qbt, err := q.login(ctx) + if err != nil { + return Result{}, err + } + + tag := qbittorrentTrackingTag(task.ID) + torrent, ok, err := q.findTask(ctx, qbt, task) + if err != nil { + return Result{}, err + } + if ok { + _ = qbt.StartCtx(ctx, []string{torrent.Hash}) + torrent, err = waitQBittorrent(ctx, qbt, tag, progress) + if err != nil { + return Result{}, err + } + return q.resultFromTorrent(ctx, qbt, task, taskDir, torrent) + } + + options := qbittorrentAddOptions(task, taskDir, tag) + if _, err := qbt.AddTorrentFromUrlCtx(ctx, task.SourceURI, options); err != nil { + torrent, ok, findErr := q.findTask(ctx, qbt, task) + if findErr != nil { + return Result{}, findErr + } + if !ok { + return Result{}, err + } + _ = qbt.StartCtx(ctx, []string{torrent.Hash}) + } + + torrent, err = waitQBittorrent(ctx, qbt, tag, progress) + if err != nil { + return Result{}, err + } + return q.resultFromTorrent(ctx, qbt, task, taskDir, torrent) +} + +func (q QBittorrent) resultFromTorrent( + ctx context.Context, + qbt *qbittorrent.Client, + task client.DownloadTask, + taskDir string, + torrent qbittorrent.Torrent, +) (Result, error) { + result, err := resultFromQBittorrentFiles(ctx, qbt, task, taskDir, torrent) + if err != nil { + return Result{}, err + } + if q.RetainSeed { + result.Seed = &Seed{ + Engine: "qbittorrent", + ID: torrent.Hash, + Path: taskDir, + Snapshot: q.seedSnapshot(torrent.Hash), + Cleanup: q.cleanupSeed(torrent.Hash, taskDir), + } + return result, nil + } + _ = qbt.DeleteTorrentsCtx(ctx, []string{torrent.Hash}, false) + return result, nil +} + +func (q QBittorrent) findTask(ctx context.Context, qbt *qbittorrent.Client, task client.DownloadTask) (qbittorrent.Torrent, bool, error) { + tag := qbittorrentTrackingTag(task.ID) + torrents, err := qbt.GetTorrentsCtx(ctx, qbittorrent.TorrentFilterOptions{Tag: tag}) + if err != nil { + return qbittorrent.Torrent{}, false, err + } + if len(torrents) > 0 { + return torrents[0], true, nil + } + taskDir := filepath.Clean(filepath.Join(q.Dir, task.ID)) + torrents, err = qbt.GetTorrentsCtx(ctx, qbittorrent.TorrentFilterOptions{}) + if err != nil { + return qbittorrent.Torrent{}, false, err + } + for _, torrent := range torrents { + if filepath.Clean(torrent.SavePath) == taskDir { + return torrent, true, nil + } + } + return qbittorrent.Torrent{}, false, nil +} + +func qbittorrentAddOptions(task client.DownloadTask, taskDir string, trackingTag string) map[string]string { + category := "zpan" + if task.Category != "" { + category = task.Category + } + tags := append([]string{trackingTag}, task.Tags...) + options := (&qbittorrent.TorrentAddOptions{ + SavePath: taskDir, + Category: category, + Tags: strings.Join(tags, ","), + LimitRatio: 0, + LimitSeedTime: 0, + SequentialDownload: false, + }).Prepare() + if name := requestedOutputName(task); name != "" { + options["rename"] = name + } + return options +} + +func qbittorrentTrackingTag(taskID string) string { + return "ztid=" + taskID +} + +func (q QBittorrent) cleanupSeed(hash string, localPath string) func(context.Context) error { + return func(ctx context.Context) error { + var errs []error + qbt, err := q.login(ctx) + if err != nil { + errs = append(errs, err) + } else if err := qbt.DeleteTorrentsCtx(ctx, []string{hash}, false); err != nil { + errs = append(errs, err) + } + if err := os.RemoveAll(localPath); err != nil { + errs = append(errs, err) + } + return errors.Join(errs...) + } +} + +func (q QBittorrent) seedSnapshot(hash string) func(context.Context) (SeedSnapshot, error) { + return func(ctx context.Context) (SeedSnapshot, error) { + qbt, err := q.login(ctx) + if err != nil { + return SeedSnapshot{}, err + } + torrents, err := qbt.GetTorrentsCtx(ctx, qbittorrent.TorrentFilterOptions{Hashes: []string{hash}}) + if err != nil { + return SeedSnapshot{}, err + } + if len(torrents) == 0 { + return SeedSnapshot{}, fmt.Errorf("qbittorrent torrent %s not found", hash) + } + torrent := torrents[0] + total := torrent.TotalSize + if total <= 0 { + total = torrent.Size + } + var totalPtr *int64 + if total > 0 { + totalPtr = &total + } + detail := qbittorrentDetail(ctx, qbt, torrent) + detail.Phase = "seeding" + return SeedSnapshot{ + Downloaded: torrent.Completed, + Total: totalPtr, + Bps: torrent.DlSpeed, + Detail: detail, + }, nil + } +} + +func waitQBittorrent( + ctx context.Context, + qbt *qbittorrent.Client, + tag string, + progress Progress, +) (qbittorrent.Torrent, error) { + ticker := time.NewTicker(time.Second) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return qbittorrent.Torrent{}, ctx.Err() + case <-ticker.C: + torrents, err := qbt.GetTorrentsCtx(ctx, qbittorrent.TorrentFilterOptions{Tag: tag}) + if err != nil { + return qbittorrent.Torrent{}, err + } + if len(torrents) == 0 { + continue + } + torrent := torrents[0] + total := torrent.TotalSize + if total <= 0 { + total = torrent.Size + } + var totalPtr *int64 + if total > 0 { + totalPtr = &total + } + if err := progress(torrent.Completed, totalPtr, torrent.DlSpeed, qbittorrentDetail(ctx, qbt, torrent)); err != nil { + _ = qbt.StopCtx(ctx, []string{torrent.Hash}) + return qbittorrent.Torrent{}, err + } + if torrent.Progress >= 1 || (torrent.AmountLeft == 0 && total > 0) { + return torrent, nil + } + if isQBittorrentErrorState(torrent.State) { + return qbittorrent.Torrent{}, fmt.Errorf("qbittorrent download ended with state %s", torrent.State) + } + } + } +} + +func qbittorrentDetail(ctx context.Context, qbt *qbittorrent.Client, torrent qbittorrent.Torrent) *client.DownloadTaskDetail { + connections := int64(torrent.NumSeeds + torrent.NumLeechs) + seeders := torrent.NumSeeds + leechers := torrent.NumLeechs + peers := torrent.NumComplete + torrent.NumIncomplete + uploaded := torrent.Uploaded + uploadBps := torrent.UpSpeed + var eta *int64 + if torrent.ETA >= 0 { + eta = &torrent.ETA + } + return &client.DownloadTaskDetail{ + Engine: "qbittorrent", + Phase: qbittorrentPhase(string(torrent.State)), + EngineState: string(torrent.State), + ETASeconds: eta, + Connections: &connections, + InfoHash: torrent.Hash, + TorrentName: torrent.Name, + Seeders: &seeders, + Leechers: &leechers, + Peers: &peers, + PeerUploadedBytes: &uploaded, + PeerUploadBps: &uploadBps, + Trackers: qbittorrentTrackers(ctx, qbt, torrent), + PeerSamples: qbittorrentPeers(ctx, qbt, torrent.Hash), + } +} + +func qbittorrentPhase(state string) string { + normalized := strings.ToLower(state) + switch { + case strings.Contains(normalized, "meta"): + return "metadata" + case strings.Contains(normalized, "up"), strings.Contains(normalized, "seed"): + return "seeding" + case strings.Contains(normalized, "error"), strings.Contains(normalized, "missing"): + return "error" + case strings.Contains(normalized, "paused"): + return "downloading" + case normalized == "uploading": + return "seeding" + default: + return "downloading" + } +} + +func qbittorrentTrackers(ctx context.Context, qbt *qbittorrent.Client, torrent qbittorrent.Torrent) []client.DownloadTaskTracker { + trackers := torrent.Trackers + if len(trackers) == 0 && torrent.Hash != "" { + loaded, err := qbt.GetTorrentTrackersCtx(ctx, torrent.Hash) + if err == nil { + trackers = loaded + } + } + out := make([]client.DownloadTaskTracker, 0, min(len(trackers), 20)) + for _, tracker := range trackers { + peers := int64(tracker.NumPeers) + seeds := int64(tracker.NumSeeds) + leechers := int64(tracker.NumLeechers) + out = append(out, client.DownloadTaskTracker{ + URL: tracker.Url, + Status: fmt.Sprint(tracker.Status), + Peers: &peers, + Seeds: &seeds, + Leechers: &leechers, + Message: tracker.Message, + }) + if len(out) >= 20 { + break + } + } + return out +} + +func qbittorrentPeers(ctx context.Context, qbt *qbittorrent.Client, hash string) []client.DownloadTaskPeer { + if hash == "" { + return nil + } + peers, err := qbt.GetTorrentPeersCtx(ctx, hash, 0) + if err != nil || peers == nil { + return nil + } + out := make([]client.DownloadTaskPeer, 0, min(len(peers.Peers), 20)) + for address, peer := range peers.Peers { + progress := peer.Progress + down := peer.DownSpeed + up := peer.UpSpeed + label := address + if peer.IP != "" && peer.Port > 0 { + label = fmt.Sprintf("%s:%d", peer.IP, peer.Port) + } + out = append(out, client.DownloadTaskPeer{ + Address: label, + Client: peer.Client, + Progress: &progress, + DownloadBps: &down, + UploadBps: &up, + }) + if len(out) >= 20 { + break + } + } + return out +} + +func resultFromQBittorrentFiles( + ctx context.Context, + qbt *qbittorrent.Client, + task client.DownloadTask, + taskDir string, + torrent qbittorrent.Torrent, +) (Result, error) { + files, err := qbt.GetFilesInformationCtx(ctx, torrent.Hash) + if err != nil || files == nil { + return resultFromPath(task, taskDir, torrent.Name) + } + downloaded := make([]downloadedFile, 0, len(*files)) + for _, file := range *files { + if file.Priority == 0 || file.Size <= 0 { + continue + } + abs, rel := downloadedPath(taskDir, file.Name) + downloaded = append(downloaded, downloadedFile{path: abs, relativePath: rel}) + } + if len(downloaded) == 0 { + return resultFromPath(task, taskDir, torrent.Name) + } + return resultFromDownloadedFiles(task, taskDir, torrent.Name, downloaded) +} + +func isQBittorrentErrorState(state qbittorrent.TorrentState) bool { + value := strings.ToLower(string(state)) + return strings.Contains(value, "error") || strings.Contains(value, "missing") +} diff --git a/downloader/internal/openapi/client.gen.go b/downloader/internal/openapi/client.gen.go index 5c31050f..3cf05f9c 100644 --- a/downloader/internal/openapi/client.gen.go +++ b/downloader/internal/openapi/client.gen.go @@ -16,90 +16,6 @@ import ( "github.com/oapi-codegen/runtime" ) -// Defines values for AbortObjectUploadSessionRequestAction. -const ( - Abort AbortObjectUploadSessionRequestAction = "abort" -) - -// Valid indicates whether the value is a known member of the AbortObjectUploadSessionRequestAction enum. -func (e AbortObjectUploadSessionRequestAction) Valid() bool { - switch e { - case Abort: - return true - default: - return false - } -} - -// Defines values for CompleteObjectUploadSessionRequestAction. -const ( - Complete CompleteObjectUploadSessionRequestAction = "complete" -) - -// Valid indicates whether the value is a known member of the CompleteObjectUploadSessionRequestAction enum. -func (e CompleteObjectUploadSessionRequestAction) Valid() bool { - switch e { - case Complete: - return true - default: - return false - } -} - -// Defines values for ConfirmObjectRequestAction. -const ( - Confirm ConfirmObjectRequestAction = "confirm" -) - -// Valid indicates whether the value is a known member of the ConfirmObjectRequestAction enum. -func (e ConfirmObjectRequestAction) Valid() bool { - switch e { - case Confirm: - return true - default: - return false - } -} - -// Defines values for ConfirmObjectRequestOnConflict. -const ( - ConfirmObjectRequestOnConflictFail ConfirmObjectRequestOnConflict = "fail" - ConfirmObjectRequestOnConflictRename ConfirmObjectRequestOnConflict = "rename" -) - -// Valid indicates whether the value is a known member of the ConfirmObjectRequestOnConflict enum. -func (e ConfirmObjectRequestOnConflict) Valid() bool { - switch e { - case ConfirmObjectRequestOnConflictFail: - return true - case ConfirmObjectRequestOnConflictRename: - return true - default: - return false - } -} - -// Defines values for ObjectUploadSessionStatus. -const ( - ObjectUploadSessionStatusAborted ObjectUploadSessionStatus = "aborted" - ObjectUploadSessionStatusActive ObjectUploadSessionStatus = "active" - ObjectUploadSessionStatusCompleted ObjectUploadSessionStatus = "completed" -) - -// Valid indicates whether the value is a known member of the ObjectUploadSessionStatus enum. -func (e ObjectUploadSessionStatus) Valid() bool { - switch e { - case ObjectUploadSessionStatusAborted: - return true - case ObjectUploadSessionStatusActive: - return true - case ObjectUploadSessionStatusCompleted: - return true - default: - return false - } -} - // Defines values for GetApiAdminDownloaders200JSONResponseBodyItemsHeartbeatEngine. const ( GetApiAdminDownloaders200JSONResponseBodyItemsHeartbeatEngineAria2 GetApiAdminDownloaders200JSONResponseBodyItemsHeartbeatEngine = "aria2" @@ -1155,43 +1071,43 @@ func (e PostApiDownloadTasksIdActions200JSONResponseBody0SourceType) Valid() boo // Defines values for PostApiDownloadTasksIdActions200JSONResponseBody0Status. const ( - Assigned PostApiDownloadTasksIdActions200JSONResponseBody0Status = "assigned" - BillingPaused PostApiDownloadTasksIdActions200JSONResponseBody0Status = "billing_paused" - Canceled PostApiDownloadTasksIdActions200JSONResponseBody0Status = "canceled" - Canceling PostApiDownloadTasksIdActions200JSONResponseBody0Status = "canceling" - Completed PostApiDownloadTasksIdActions200JSONResponseBody0Status = "completed" - Failed PostApiDownloadTasksIdActions200JSONResponseBody0Status = "failed" - Paused PostApiDownloadTasksIdActions200JSONResponseBody0Status = "paused" - Pausing PostApiDownloadTasksIdActions200JSONResponseBody0Status = "pausing" - Queued PostApiDownloadTasksIdActions200JSONResponseBody0Status = "queued" - Running PostApiDownloadTasksIdActions200JSONResponseBody0Status = "running" - Uploading PostApiDownloadTasksIdActions200JSONResponseBody0Status = "uploading" + PostApiDownloadTasksIdActions200JSONResponseBody0StatusAssigned PostApiDownloadTasksIdActions200JSONResponseBody0Status = "assigned" + PostApiDownloadTasksIdActions200JSONResponseBody0StatusBillingPaused PostApiDownloadTasksIdActions200JSONResponseBody0Status = "billing_paused" + PostApiDownloadTasksIdActions200JSONResponseBody0StatusCanceled PostApiDownloadTasksIdActions200JSONResponseBody0Status = "canceled" + PostApiDownloadTasksIdActions200JSONResponseBody0StatusCanceling PostApiDownloadTasksIdActions200JSONResponseBody0Status = "canceling" + PostApiDownloadTasksIdActions200JSONResponseBody0StatusCompleted PostApiDownloadTasksIdActions200JSONResponseBody0Status = "completed" + PostApiDownloadTasksIdActions200JSONResponseBody0StatusFailed PostApiDownloadTasksIdActions200JSONResponseBody0Status = "failed" + PostApiDownloadTasksIdActions200JSONResponseBody0StatusPaused PostApiDownloadTasksIdActions200JSONResponseBody0Status = "paused" + PostApiDownloadTasksIdActions200JSONResponseBody0StatusPausing PostApiDownloadTasksIdActions200JSONResponseBody0Status = "pausing" + PostApiDownloadTasksIdActions200JSONResponseBody0StatusQueued PostApiDownloadTasksIdActions200JSONResponseBody0Status = "queued" + PostApiDownloadTasksIdActions200JSONResponseBody0StatusRunning PostApiDownloadTasksIdActions200JSONResponseBody0Status = "running" + PostApiDownloadTasksIdActions200JSONResponseBody0StatusUploading PostApiDownloadTasksIdActions200JSONResponseBody0Status = "uploading" ) // Valid indicates whether the value is a known member of the PostApiDownloadTasksIdActions200JSONResponseBody0Status enum. func (e PostApiDownloadTasksIdActions200JSONResponseBody0Status) Valid() bool { switch e { - case Assigned: + case PostApiDownloadTasksIdActions200JSONResponseBody0StatusAssigned: return true - case BillingPaused: + case PostApiDownloadTasksIdActions200JSONResponseBody0StatusBillingPaused: return true - case Canceled: + case PostApiDownloadTasksIdActions200JSONResponseBody0StatusCanceled: return true - case Canceling: + case PostApiDownloadTasksIdActions200JSONResponseBody0StatusCanceling: return true - case Completed: + case PostApiDownloadTasksIdActions200JSONResponseBody0StatusCompleted: return true - case Failed: + case PostApiDownloadTasksIdActions200JSONResponseBody0StatusFailed: return true - case Paused: + case PostApiDownloadTasksIdActions200JSONResponseBody0StatusPaused: return true - case Pausing: + case PostApiDownloadTasksIdActions200JSONResponseBody0StatusPausing: return true - case Queued: + case PostApiDownloadTasksIdActions200JSONResponseBody0StatusQueued: return true - case Running: + case PostApiDownloadTasksIdActions200JSONResponseBody0StatusRunning: return true - case Uploading: + case PostApiDownloadTasksIdActions200JSONResponseBody0StatusUploading: return true default: return false @@ -1297,41 +1213,112 @@ func (e PostApiObjectsJSONBodyOnConflict) Valid() bool { } } -// AbortObjectUploadSessionRequest defines model for AbortObjectUploadSessionRequest. -type AbortObjectUploadSessionRequest struct { - Action AbortObjectUploadSessionRequestAction `json:"action"` +// Defines values for PatchApiObjectsIdJSONBodyAction. +const ( + Confirm PatchApiObjectsIdJSONBodyAction = "confirm" +) + +// Valid indicates whether the value is a known member of the PatchApiObjectsIdJSONBodyAction enum. +func (e PatchApiObjectsIdJSONBodyAction) Valid() bool { + switch e { + case Confirm: + return true + default: + return false + } } -// AbortObjectUploadSessionRequestAction defines model for AbortObjectUploadSessionRequest.Action. -type AbortObjectUploadSessionRequestAction string +// Defines values for PatchApiObjectsIdJSONBodyOnConflict. +const ( + PatchApiObjectsIdJSONBodyOnConflictFail PatchApiObjectsIdJSONBodyOnConflict = "fail" + PatchApiObjectsIdJSONBodyOnConflictRename PatchApiObjectsIdJSONBodyOnConflict = "rename" + PatchApiObjectsIdJSONBodyOnConflictReplace PatchApiObjectsIdJSONBodyOnConflict = "replace" +) -// CompleteObjectUploadSessionRequest defines model for CompleteObjectUploadSessionRequest. -type CompleteObjectUploadSessionRequest struct { - Action CompleteObjectUploadSessionRequestAction `json:"action"` - Parts []struct { - Etag string `json:"etag"` - PartNumber int `json:"partNumber"` - } `json:"parts"` +// Valid indicates whether the value is a known member of the PatchApiObjectsIdJSONBodyOnConflict enum. +func (e PatchApiObjectsIdJSONBodyOnConflict) Valid() bool { + switch e { + case PatchApiObjectsIdJSONBodyOnConflictFail: + return true + case PatchApiObjectsIdJSONBodyOnConflictRename: + return true + case PatchApiObjectsIdJSONBodyOnConflictReplace: + return true + default: + return false + } } -// CompleteObjectUploadSessionRequestAction defines model for CompleteObjectUploadSessionRequest.Action. -type CompleteObjectUploadSessionRequestAction string +// Defines values for PostApiObjectsIdUploads201JSONResponseBodyStatus. +const ( + PostApiObjectsIdUploads201JSONResponseBodyStatusAborted PostApiObjectsIdUploads201JSONResponseBodyStatus = "aborted" + PostApiObjectsIdUploads201JSONResponseBodyStatusActive PostApiObjectsIdUploads201JSONResponseBodyStatus = "active" + PostApiObjectsIdUploads201JSONResponseBodyStatusCompleted PostApiObjectsIdUploads201JSONResponseBodyStatus = "completed" +) -// ConfirmObjectRequest defines model for ConfirmObjectRequest. -type ConfirmObjectRequest struct { - Action ConfirmObjectRequestAction `json:"action"` - OnConflict *ConfirmObjectRequestOnConflict `json:"onConflict,omitempty"` +// Valid indicates whether the value is a known member of the PostApiObjectsIdUploads201JSONResponseBodyStatus enum. +func (e PostApiObjectsIdUploads201JSONResponseBodyStatus) Valid() bool { + switch e { + case PostApiObjectsIdUploads201JSONResponseBodyStatusAborted: + return true + case PostApiObjectsIdUploads201JSONResponseBodyStatusActive: + return true + case PostApiObjectsIdUploads201JSONResponseBodyStatusCompleted: + return true + default: + return false + } } -// ConfirmObjectRequestAction defines model for ConfirmObjectRequest.Action. -type ConfirmObjectRequestAction string +// Defines values for PatchApiObjectsIdUploadsUploadSessionIdJSONBody0Action. +const ( + Complete PatchApiObjectsIdUploadsUploadSessionIdJSONBody0Action = "complete" +) -// ConfirmObjectRequestOnConflict defines model for ConfirmObjectRequest.OnConflict. -type ConfirmObjectRequestOnConflict string +// Valid indicates whether the value is a known member of the PatchApiObjectsIdUploadsUploadSessionIdJSONBody0Action enum. +func (e PatchApiObjectsIdUploadsUploadSessionIdJSONBody0Action) Valid() bool { + switch e { + case Complete: + return true + default: + return false + } +} -// CreateObjectUploadSessionRequest defines model for CreateObjectUploadSessionRequest. -type CreateObjectUploadSessionRequest struct { - PartSize *int `json:"partSize,omitempty"` +// Defines values for PatchApiObjectsIdUploadsUploadSessionIdJSONBody1Action. +const ( + Abort PatchApiObjectsIdUploadsUploadSessionIdJSONBody1Action = "abort" +) + +// Valid indicates whether the value is a known member of the PatchApiObjectsIdUploadsUploadSessionIdJSONBody1Action enum. +func (e PatchApiObjectsIdUploadsUploadSessionIdJSONBody1Action) Valid() bool { + switch e { + case Abort: + return true + default: + return false + } +} + +// Defines values for PatchApiObjectsIdUploadsUploadSessionId200JSONResponseBodyStatus. +const ( + PatchApiObjectsIdUploadsUploadSessionId200JSONResponseBodyStatusAborted PatchApiObjectsIdUploadsUploadSessionId200JSONResponseBodyStatus = "aborted" + PatchApiObjectsIdUploadsUploadSessionId200JSONResponseBodyStatusActive PatchApiObjectsIdUploadsUploadSessionId200JSONResponseBodyStatus = "active" + PatchApiObjectsIdUploadsUploadSessionId200JSONResponseBodyStatusCompleted PatchApiObjectsIdUploadsUploadSessionId200JSONResponseBodyStatus = "completed" +) + +// Valid indicates whether the value is a known member of the PatchApiObjectsIdUploadsUploadSessionId200JSONResponseBodyStatus enum. +func (e PatchApiObjectsIdUploadsUploadSessionId200JSONResponseBodyStatus) Valid() bool { + switch e { + case PatchApiObjectsIdUploadsUploadSessionId200JSONResponseBodyStatusAborted: + return true + case PatchApiObjectsIdUploadsUploadSessionId200JSONResponseBodyStatusActive: + return true + case PatchApiObjectsIdUploadsUploadSessionId200JSONResponseBodyStatusCompleted: + return true + default: + return false + } } // DeviceCode defines model for DeviceCode. @@ -1370,52 +1357,6 @@ type ErrorResponse struct { Error string `json:"error"` } -// ObjectDraft defines model for ObjectDraft. -type ObjectDraft struct { - ContentDisposition *string `json:"contentDisposition,omitempty"` - Id string `json:"id"` - Name string `json:"name"` - UploadUrl *string `json:"uploadUrl,omitempty"` -} - -// ObjectUploadSession defines model for ObjectUploadSession. -type ObjectUploadSession struct { - CreatedAt string `json:"createdAt"` - ExpiresAt string `json:"expiresAt"` - Id string `json:"id"` - ObjectId string `json:"objectId"` - PartSize int `json:"partSize"` - Status ObjectUploadSessionStatus `json:"status"` - UpdatedAt string `json:"updatedAt"` - UploadId string `json:"uploadId"` -} - -// ObjectUploadSessionStatus defines model for ObjectUploadSession.Status. -type ObjectUploadSessionStatus string - -// PatchObjectUploadSessionRequest defines model for PatchObjectUploadSessionRequest. -type PatchObjectUploadSessionRequest struct { - union json.RawMessage -} - -// PresignObjectUploadPartsRequest defines model for PresignObjectUploadPartsRequest. -type PresignObjectUploadPartsRequest struct { - PartNumbers []int `json:"partNumbers"` -} - -// PresignObjectUploadPartsResponse defines model for PresignObjectUploadPartsResponse. -type PresignObjectUploadPartsResponse struct { - PartSize int `json:"partSize"` - Parts []PresignedObjectUploadPart `json:"parts"` - UploadId string `json:"uploadId"` -} - -// PresignedObjectUploadPart defines model for PresignedObjectUploadPart. -type PresignedObjectUploadPart struct { - PartNumber int `json:"partNumber"` - Url string `json:"url"` -} - // GetApiAdminDownloaders200JSONResponseBodyItemsHeartbeatEngine defines parameters for GetApiAdminDownloaders. type GetApiAdminDownloaders200JSONResponseBodyItemsHeartbeatEngine string @@ -1428,13 +1369,13 @@ type PostApiAdminDownloadersJSONBody struct { Arch string `json:"arch"` Capabilities []string `json:"capabilities"` CurrentTasks int `json:"currentTasks"` - DownloadBps *int `json:"downloadBps,omitempty"` + DownloadBps *int64 `json:"downloadBps,omitempty"` Engine PostApiAdminDownloadersJSONBodyHeartbeatEngine `json:"engine"` - FreeDiskBytes *int `json:"freeDiskBytes,omitempty"` + FreeDiskBytes *int64 `json:"freeDiskBytes,omitempty"` Hostname string `json:"hostname"` MaxConcurrentTasks int `json:"maxConcurrentTasks"` Platform string `json:"platform"` - UploadBps *int `json:"uploadBps,omitempty"` + UploadBps *int64 `json:"uploadBps,omitempty"` Version string `json:"version"` } `json:"heartbeat"` Name string `json:"name"` @@ -1571,10 +1512,10 @@ type PatchApiDownloadTasksIdJSONBody struct { EngineState *string `json:"engineState,omitempty"` EtaSeconds *int `json:"etaSeconds,omitempty"` Files *[]struct { - CompletedBytes *int `json:"completedBytes,omitempty"` + CompletedBytes *int64 `json:"completedBytes,omitempty"` Path string `json:"path"` Selected *bool `json:"selected,omitempty"` - Size int `json:"size"` + Size int64 `json:"size"` } `json:"files,omitempty"` InfoHash *string `json:"infoHash,omitempty"` Leechers *int `json:"leechers,omitempty"` @@ -1582,12 +1523,12 @@ type PatchApiDownloadTasksIdJSONBody struct { PeerSamples *[]struct { Address string `json:"address"` Client *string `json:"client,omitempty"` - DownloadBps *int `json:"downloadBps,omitempty"` + DownloadBps *int64 `json:"downloadBps,omitempty"` Progress *float32 `json:"progress,omitempty"` - UploadBps *int `json:"uploadBps,omitempty"` + UploadBps *int64 `json:"uploadBps,omitempty"` } `json:"peerSamples,omitempty"` - PeerUploadBps *int `json:"peerUploadBps,omitempty"` - PeerUploadedBytes *int `json:"peerUploadedBytes,omitempty"` + PeerUploadBps *int64 `json:"peerUploadBps,omitempty"` + PeerUploadedBytes *int64 `json:"peerUploadedBytes,omitempty"` Peers *int `json:"peers,omitempty"` Phase *PatchApiDownloadTasksIdJSONBodyDetailPhase `json:"phase,omitempty"` Seeders *int `json:"seeders,omitempty"` @@ -1601,14 +1542,14 @@ type PatchApiDownloadTasksIdJSONBody struct { Url string `json:"url"` } `json:"trackers,omitempty"` } `json:"detail,omitempty"` - DownloadBps *int `json:"downloadBps,omitempty"` - DownloadedBytes *int `json:"downloadedBytes,omitempty"` + DownloadBps *int64 `json:"downloadBps,omitempty"` + DownloadedBytes *int64 `json:"downloadedBytes,omitempty"` ErrorMessage *string `json:"errorMessage,omitempty"` ResultObjectId *string `json:"resultObjectId,omitempty"` Status *PatchApiDownloadTasksIdJSONBodyStatus `json:"status,omitempty"` - StorageUploadBps *int `json:"storageUploadBps,omitempty"` - StorageUploadedBytes *int `json:"storageUploadedBytes,omitempty"` - TotalBytes *int `json:"totalBytes,omitempty"` + StorageUploadBps *int64 `json:"storageUploadBps,omitempty"` + StorageUploadedBytes *int64 `json:"storageUploadedBytes,omitempty"` + TotalBytes *int64 `json:"totalBytes,omitempty"` } // PatchApiDownloadTasksIdJSONBodyDetailEngine defines parameters for PatchApiDownloadTasksId. @@ -1650,10 +1591,10 @@ type PostApiDownloadTasksIdActions200JSONResponseBody0 struct { EngineState *string `json:"engineState,omitempty"` EtaSeconds *int `json:"etaSeconds,omitempty"` Files *[]struct { - CompletedBytes *int `json:"completedBytes,omitempty"` + CompletedBytes *int64 `json:"completedBytes,omitempty"` Path string `json:"path"` Selected *bool `json:"selected,omitempty"` - Size int `json:"size"` + Size int64 `json:"size"` } `json:"files,omitempty"` InfoHash *string `json:"infoHash,omitempty"` Leechers *int `json:"leechers,omitempty"` @@ -1661,12 +1602,12 @@ type PostApiDownloadTasksIdActions200JSONResponseBody0 struct { PeerSamples *[]struct { Address string `json:"address"` Client *string `json:"client,omitempty"` - DownloadBps *int `json:"downloadBps,omitempty"` + DownloadBps *int64 `json:"downloadBps,omitempty"` Progress *float32 `json:"progress,omitempty"` - UploadBps *int `json:"uploadBps,omitempty"` + UploadBps *int64 `json:"uploadBps,omitempty"` } `json:"peerSamples,omitempty"` - PeerUploadBps *int `json:"peerUploadBps,omitempty"` - PeerUploadedBytes *int `json:"peerUploadedBytes,omitempty"` + PeerUploadBps *int64 `json:"peerUploadBps,omitempty"` + PeerUploadedBytes *int64 `json:"peerUploadedBytes,omitempty"` Peers *int `json:"peers,omitempty"` Phase *PostApiDownloadTasksIdActions200JSONResponseBody0DetailPhase `json:"phase,omitempty"` Seeders *int `json:"seeders,omitempty"` @@ -1728,13 +1669,13 @@ type PostApiDownloaderHeartbeatJSONBody struct { Arch string `json:"arch"` Capabilities []string `json:"capabilities"` CurrentTasks int `json:"currentTasks"` - DownloadBps *int `json:"downloadBps,omitempty"` + DownloadBps *int64 `json:"downloadBps,omitempty"` Engine PostApiDownloaderHeartbeatJSONBodyEngine `json:"engine"` - FreeDiskBytes *int `json:"freeDiskBytes,omitempty"` + FreeDiskBytes *int64 `json:"freeDiskBytes,omitempty"` Hostname string `json:"hostname"` MaxConcurrentTasks int `json:"maxConcurrentTasks"` Platform string `json:"platform"` - UploadBps *int `json:"uploadBps,omitempty"` + UploadBps *int64 `json:"uploadBps,omitempty"` Version string `json:"version"` } @@ -1749,17 +1690,70 @@ type PostApiDownloaderHeartbeat200JSONResponseBodyStatus string // PostApiObjectsJSONBody defines parameters for PostApiObjects. type PostApiObjectsJSONBody struct { - Dirtype *float32 `json:"dirtype,omitempty"` + Dirtype *int `json:"dirtype,omitempty"` Name string `json:"name"` OnConflict *PostApiObjectsJSONBodyOnConflict `json:"onConflict,omitempty"` Parent *string `json:"parent,omitempty"` - Size *float32 `json:"size,omitempty"` + Size *int `json:"size,omitempty"` Type string `json:"type"` } // PostApiObjectsJSONBodyOnConflict defines parameters for PostApiObjects. type PostApiObjectsJSONBodyOnConflict string +// PatchApiObjectsIdJSONBody defines parameters for PatchApiObjectsId. +type PatchApiObjectsIdJSONBody struct { + Action PatchApiObjectsIdJSONBodyAction `json:"action"` + OnConflict *PatchApiObjectsIdJSONBodyOnConflict `json:"onConflict,omitempty"` +} + +// PatchApiObjectsIdJSONBodyAction defines parameters for PatchApiObjectsId. +type PatchApiObjectsIdJSONBodyAction string + +// PatchApiObjectsIdJSONBodyOnConflict defines parameters for PatchApiObjectsId. +type PatchApiObjectsIdJSONBodyOnConflict string + +// PostApiObjectsIdUploadsJSONBody defines parameters for PostApiObjectsIdUploads. +type PostApiObjectsIdUploadsJSONBody struct { + PartSize *int `json:"partSize,omitempty"` +} + +// PostApiObjectsIdUploads201JSONResponseBodyStatus defines parameters for PostApiObjectsIdUploads. +type PostApiObjectsIdUploads201JSONResponseBodyStatus string + +// PatchApiObjectsIdUploadsUploadSessionIdJSONBody defines parameters for PatchApiObjectsIdUploadsUploadSessionId. +type PatchApiObjectsIdUploadsUploadSessionIdJSONBody struct { + union json.RawMessage +} + +// PatchApiObjectsIdUploadsUploadSessionIdJSONBody0 defines parameters for PatchApiObjectsIdUploadsUploadSessionId. +type PatchApiObjectsIdUploadsUploadSessionIdJSONBody0 struct { + Action PatchApiObjectsIdUploadsUploadSessionIdJSONBody0Action `json:"action"` + Parts []struct { + Etag string `json:"etag"` + PartNumber int `json:"partNumber"` + } `json:"parts"` +} + +// PatchApiObjectsIdUploadsUploadSessionIdJSONBody0Action defines parameters for PatchApiObjectsIdUploadsUploadSessionId. +type PatchApiObjectsIdUploadsUploadSessionIdJSONBody0Action string + +// PatchApiObjectsIdUploadsUploadSessionIdJSONBody1 defines parameters for PatchApiObjectsIdUploadsUploadSessionId. +type PatchApiObjectsIdUploadsUploadSessionIdJSONBody1 struct { + Action PatchApiObjectsIdUploadsUploadSessionIdJSONBody1Action `json:"action"` +} + +// PatchApiObjectsIdUploadsUploadSessionIdJSONBody1Action defines parameters for PatchApiObjectsIdUploadsUploadSessionId. +type PatchApiObjectsIdUploadsUploadSessionIdJSONBody1Action string + +// PatchApiObjectsIdUploadsUploadSessionId200JSONResponseBodyStatus defines parameters for PatchApiObjectsIdUploadsUploadSessionId. +type PatchApiObjectsIdUploadsUploadSessionId200JSONResponseBodyStatus string + +// PostApiObjectsIdUploadsUploadSessionIdPartsJSONBody defines parameters for PostApiObjectsIdUploadsUploadSessionIdParts. +type PostApiObjectsIdUploadsUploadSessionIdPartsJSONBody struct { + PartNumbers []int `json:"partNumbers"` +} + // PostApiAdminDownloadersJSONRequestBody defines body for PostApiAdminDownloaders for application/json ContentType. type PostApiAdminDownloadersJSONRequestBody PostApiAdminDownloadersJSONBody @@ -1788,78 +1782,16 @@ type PostApiDownloaderHeartbeatJSONRequestBody PostApiDownloaderHeartbeatJSONBod type PostApiObjectsJSONRequestBody PostApiObjectsJSONBody // PatchApiObjectsIdJSONRequestBody defines body for PatchApiObjectsId for application/json ContentType. -type PatchApiObjectsIdJSONRequestBody = ConfirmObjectRequest +type PatchApiObjectsIdJSONRequestBody PatchApiObjectsIdJSONBody // PostApiObjectsIdUploadsJSONRequestBody defines body for PostApiObjectsIdUploads for application/json ContentType. -type PostApiObjectsIdUploadsJSONRequestBody = CreateObjectUploadSessionRequest +type PostApiObjectsIdUploadsJSONRequestBody PostApiObjectsIdUploadsJSONBody // PatchApiObjectsIdUploadsUploadSessionIdJSONRequestBody defines body for PatchApiObjectsIdUploadsUploadSessionId for application/json ContentType. -type PatchApiObjectsIdUploadsUploadSessionIdJSONRequestBody = PatchObjectUploadSessionRequest +type PatchApiObjectsIdUploadsUploadSessionIdJSONRequestBody PatchApiObjectsIdUploadsUploadSessionIdJSONBody // PostApiObjectsIdUploadsUploadSessionIdPartsJSONRequestBody defines body for PostApiObjectsIdUploadsUploadSessionIdParts for application/json ContentType. -type PostApiObjectsIdUploadsUploadSessionIdPartsJSONRequestBody = PresignObjectUploadPartsRequest - -// AsCompleteObjectUploadSessionRequest returns the union data inside the PatchObjectUploadSessionRequest as a CompleteObjectUploadSessionRequest -func (t PatchObjectUploadSessionRequest) AsCompleteObjectUploadSessionRequest() (CompleteObjectUploadSessionRequest, error) { - var body CompleteObjectUploadSessionRequest - err := json.Unmarshal(t.union, &body) - return body, err -} - -// FromCompleteObjectUploadSessionRequest overwrites any union data inside the PatchObjectUploadSessionRequest as the provided CompleteObjectUploadSessionRequest -func (t *PatchObjectUploadSessionRequest) FromCompleteObjectUploadSessionRequest(v CompleteObjectUploadSessionRequest) error { - b, err := json.Marshal(v) - t.union = b - return err -} - -// MergeCompleteObjectUploadSessionRequest performs a merge with any union data inside the PatchObjectUploadSessionRequest, using the provided CompleteObjectUploadSessionRequest -func (t *PatchObjectUploadSessionRequest) MergeCompleteObjectUploadSessionRequest(v CompleteObjectUploadSessionRequest) error { - b, err := json.Marshal(v) - if err != nil { - return err - } - - merged, err := runtime.JSONMerge(t.union, b) - t.union = merged - return err -} - -// AsAbortObjectUploadSessionRequest returns the union data inside the PatchObjectUploadSessionRequest as a AbortObjectUploadSessionRequest -func (t PatchObjectUploadSessionRequest) AsAbortObjectUploadSessionRequest() (AbortObjectUploadSessionRequest, error) { - var body AbortObjectUploadSessionRequest - err := json.Unmarshal(t.union, &body) - return body, err -} - -// FromAbortObjectUploadSessionRequest overwrites any union data inside the PatchObjectUploadSessionRequest as the provided AbortObjectUploadSessionRequest -func (t *PatchObjectUploadSessionRequest) FromAbortObjectUploadSessionRequest(v AbortObjectUploadSessionRequest) error { - b, err := json.Marshal(v) - t.union = b - return err -} - -// MergeAbortObjectUploadSessionRequest performs a merge with any union data inside the PatchObjectUploadSessionRequest, using the provided AbortObjectUploadSessionRequest -func (t *PatchObjectUploadSessionRequest) MergeAbortObjectUploadSessionRequest(v AbortObjectUploadSessionRequest) error { - b, err := json.Marshal(v) - if err != nil { - return err - } - - merged, err := runtime.JSONMerge(t.union, b) - t.union = merged - return err -} - -func (t PatchObjectUploadSessionRequest) MarshalJSON() ([]byte, error) { - b, err := t.union.MarshalJSON() - return b, err -} - -func (t *PatchObjectUploadSessionRequest) UnmarshalJSON(b []byte) error { - err := t.union.UnmarshalJSON(b) - return err -} +type PostApiObjectsIdUploadsUploadSessionIdPartsJSONRequestBody PostApiObjectsIdUploadsUploadSessionIdPartsJSONBody // AsPostApiDownloadTasksIdActions200JSONResponseBody0 returns the union data inside the PostApiDownloadTasksIdActions200JSONResponseBody as a PostApiDownloadTasksIdActions200JSONResponseBody0 func (t PostApiDownloadTasksIdActions200JSONResponseBody) AsPostApiDownloadTasksIdActions200JSONResponseBody0() (PostApiDownloadTasksIdActions200JSONResponseBody0, error) { @@ -1923,6 +1855,68 @@ func (t *PostApiDownloadTasksIdActions200JSONResponseBody) UnmarshalJSON(b []byt return err } +// AsPatchApiObjectsIdUploadsUploadSessionIdJSONBody0 returns the union data inside the PatchApiObjectsIdUploadsUploadSessionIdJSONBody as a PatchApiObjectsIdUploadsUploadSessionIdJSONBody0 +func (t PatchApiObjectsIdUploadsUploadSessionIdJSONBody) AsPatchApiObjectsIdUploadsUploadSessionIdJSONBody0() (PatchApiObjectsIdUploadsUploadSessionIdJSONBody0, error) { + var body PatchApiObjectsIdUploadsUploadSessionIdJSONBody0 + err := json.Unmarshal(t.union, &body) + return body, err +} + +// FromPatchApiObjectsIdUploadsUploadSessionIdJSONBody0 overwrites any union data inside the PatchApiObjectsIdUploadsUploadSessionIdJSONBody as the provided PatchApiObjectsIdUploadsUploadSessionIdJSONBody0 +func (t *PatchApiObjectsIdUploadsUploadSessionIdJSONBody) FromPatchApiObjectsIdUploadsUploadSessionIdJSONBody0(v PatchApiObjectsIdUploadsUploadSessionIdJSONBody0) error { + b, err := json.Marshal(v) + t.union = b + return err +} + +// MergePatchApiObjectsIdUploadsUploadSessionIdJSONBody0 performs a merge with any union data inside the PatchApiObjectsIdUploadsUploadSessionIdJSONBody, using the provided PatchApiObjectsIdUploadsUploadSessionIdJSONBody0 +func (t *PatchApiObjectsIdUploadsUploadSessionIdJSONBody) MergePatchApiObjectsIdUploadsUploadSessionIdJSONBody0(v PatchApiObjectsIdUploadsUploadSessionIdJSONBody0) error { + b, err := json.Marshal(v) + if err != nil { + return err + } + + merged, err := runtime.JSONMerge(t.union, b) + t.union = merged + return err +} + +// AsPatchApiObjectsIdUploadsUploadSessionIdJSONBody1 returns the union data inside the PatchApiObjectsIdUploadsUploadSessionIdJSONBody as a PatchApiObjectsIdUploadsUploadSessionIdJSONBody1 +func (t PatchApiObjectsIdUploadsUploadSessionIdJSONBody) AsPatchApiObjectsIdUploadsUploadSessionIdJSONBody1() (PatchApiObjectsIdUploadsUploadSessionIdJSONBody1, error) { + var body PatchApiObjectsIdUploadsUploadSessionIdJSONBody1 + err := json.Unmarshal(t.union, &body) + return body, err +} + +// FromPatchApiObjectsIdUploadsUploadSessionIdJSONBody1 overwrites any union data inside the PatchApiObjectsIdUploadsUploadSessionIdJSONBody as the provided PatchApiObjectsIdUploadsUploadSessionIdJSONBody1 +func (t *PatchApiObjectsIdUploadsUploadSessionIdJSONBody) FromPatchApiObjectsIdUploadsUploadSessionIdJSONBody1(v PatchApiObjectsIdUploadsUploadSessionIdJSONBody1) error { + b, err := json.Marshal(v) + t.union = b + return err +} + +// MergePatchApiObjectsIdUploadsUploadSessionIdJSONBody1 performs a merge with any union data inside the PatchApiObjectsIdUploadsUploadSessionIdJSONBody, using the provided PatchApiObjectsIdUploadsUploadSessionIdJSONBody1 +func (t *PatchApiObjectsIdUploadsUploadSessionIdJSONBody) MergePatchApiObjectsIdUploadsUploadSessionIdJSONBody1(v PatchApiObjectsIdUploadsUploadSessionIdJSONBody1) error { + b, err := json.Marshal(v) + if err != nil { + return err + } + + merged, err := runtime.JSONMerge(t.union, b) + t.union = merged + return err +} + +func (t PatchApiObjectsIdUploadsUploadSessionIdJSONBody) MarshalJSON() ([]byte, error) { + b, err := t.union.MarshalJSON() + return b, err +} + +func (t *PatchApiObjectsIdUploadsUploadSessionIdJSONBody) UnmarshalJSON(b []byte) error { + err := t.union.UnmarshalJSON(b) + return err +} + // RequestEditorFn is the function signature for the RequestEditor callback function type RequestEditorFn func(ctx context.Context, req *http.Request) error @@ -3796,10 +3790,10 @@ type GetApiDownloadTasksResponse struct { EngineState *string `json:"engineState,omitempty"` EtaSeconds *int `json:"etaSeconds,omitempty"` Files *[]struct { - CompletedBytes *int `json:"completedBytes,omitempty"` + CompletedBytes *int64 `json:"completedBytes,omitempty"` Path string `json:"path"` Selected *bool `json:"selected,omitempty"` - Size int `json:"size"` + Size int64 `json:"size"` } `json:"files,omitempty"` InfoHash *string `json:"infoHash,omitempty"` Leechers *int `json:"leechers,omitempty"` @@ -3807,12 +3801,12 @@ type GetApiDownloadTasksResponse struct { PeerSamples *[]struct { Address string `json:"address"` Client *string `json:"client,omitempty"` - DownloadBps *int `json:"downloadBps,omitempty"` + DownloadBps *int64 `json:"downloadBps,omitempty"` Progress *float32 `json:"progress,omitempty"` - UploadBps *int `json:"uploadBps,omitempty"` + UploadBps *int64 `json:"uploadBps,omitempty"` } `json:"peerSamples,omitempty"` - PeerUploadBps *int `json:"peerUploadBps,omitempty"` - PeerUploadedBytes *int `json:"peerUploadedBytes,omitempty"` + PeerUploadBps *int64 `json:"peerUploadBps,omitempty"` + PeerUploadedBytes *int64 `json:"peerUploadedBytes,omitempty"` Peers *int `json:"peers,omitempty"` Phase *GetApiDownloadTasks200JSONResponseBodyItemsDetailPhase `json:"phase,omitempty"` Seeders *int `json:"seeders,omitempty"` @@ -3887,10 +3881,10 @@ type PostApiDownloadTasksResponse struct { EngineState *string `json:"engineState,omitempty"` EtaSeconds *int `json:"etaSeconds,omitempty"` Files *[]struct { - CompletedBytes *int `json:"completedBytes,omitempty"` + CompletedBytes *int64 `json:"completedBytes,omitempty"` Path string `json:"path"` Selected *bool `json:"selected,omitempty"` - Size int `json:"size"` + Size int64 `json:"size"` } `json:"files,omitempty"` InfoHash *string `json:"infoHash,omitempty"` Leechers *int `json:"leechers,omitempty"` @@ -3898,12 +3892,12 @@ type PostApiDownloadTasksResponse struct { PeerSamples *[]struct { Address string `json:"address"` Client *string `json:"client,omitempty"` - DownloadBps *int `json:"downloadBps,omitempty"` + DownloadBps *int64 `json:"downloadBps,omitempty"` Progress *float32 `json:"progress,omitempty"` - UploadBps *int `json:"uploadBps,omitempty"` + UploadBps *int64 `json:"uploadBps,omitempty"` } `json:"peerSamples,omitempty"` - PeerUploadBps *int `json:"peerUploadBps,omitempty"` - PeerUploadedBytes *int `json:"peerUploadedBytes,omitempty"` + PeerUploadBps *int64 `json:"peerUploadBps,omitempty"` + PeerUploadedBytes *int64 `json:"peerUploadedBytes,omitempty"` Peers *int `json:"peers,omitempty"` Phase *PostApiDownloadTasks201JSONResponseBodyDetailPhase `json:"phase,omitempty"` Seeders *int `json:"seeders,omitempty"` @@ -4012,10 +4006,10 @@ type GetApiDownloadTasksIdResponse struct { EngineState *string `json:"engineState,omitempty"` EtaSeconds *int `json:"etaSeconds,omitempty"` Files *[]struct { - CompletedBytes *int `json:"completedBytes,omitempty"` + CompletedBytes *int64 `json:"completedBytes,omitempty"` Path string `json:"path"` Selected *bool `json:"selected,omitempty"` - Size int `json:"size"` + Size int64 `json:"size"` } `json:"files,omitempty"` InfoHash *string `json:"infoHash,omitempty"` Leechers *int `json:"leechers,omitempty"` @@ -4023,12 +4017,12 @@ type GetApiDownloadTasksIdResponse struct { PeerSamples *[]struct { Address string `json:"address"` Client *string `json:"client,omitempty"` - DownloadBps *int `json:"downloadBps,omitempty"` + DownloadBps *int64 `json:"downloadBps,omitempty"` Progress *float32 `json:"progress,omitempty"` - UploadBps *int `json:"uploadBps,omitempty"` + UploadBps *int64 `json:"uploadBps,omitempty"` } `json:"peerSamples,omitempty"` - PeerUploadBps *int `json:"peerUploadBps,omitempty"` - PeerUploadedBytes *int `json:"peerUploadedBytes,omitempty"` + PeerUploadBps *int64 `json:"peerUploadBps,omitempty"` + PeerUploadedBytes *int64 `json:"peerUploadedBytes,omitempty"` Peers *int `json:"peers,omitempty"` Phase *GetApiDownloadTasksId200JSONResponseBodyDetailPhase `json:"phase,omitempty"` Seeders *int `json:"seeders,omitempty"` @@ -4099,10 +4093,10 @@ type PatchApiDownloadTasksIdResponse struct { EngineState *string `json:"engineState,omitempty"` EtaSeconds *int `json:"etaSeconds,omitempty"` Files *[]struct { - CompletedBytes *int `json:"completedBytes,omitempty"` + CompletedBytes *int64 `json:"completedBytes,omitempty"` Path string `json:"path"` Selected *bool `json:"selected,omitempty"` - Size int `json:"size"` + Size int64 `json:"size"` } `json:"files,omitempty"` InfoHash *string `json:"infoHash,omitempty"` Leechers *int `json:"leechers,omitempty"` @@ -4110,12 +4104,12 @@ type PatchApiDownloadTasksIdResponse struct { PeerSamples *[]struct { Address string `json:"address"` Client *string `json:"client,omitempty"` - DownloadBps *int `json:"downloadBps,omitempty"` + DownloadBps *int64 `json:"downloadBps,omitempty"` Progress *float32 `json:"progress,omitempty"` - UploadBps *int `json:"uploadBps,omitempty"` + UploadBps *int64 `json:"uploadBps,omitempty"` } `json:"peerSamples,omitempty"` - PeerUploadBps *int `json:"peerUploadBps,omitempty"` - PeerUploadedBytes *int `json:"peerUploadedBytes,omitempty"` + PeerUploadBps *int64 `json:"peerUploadBps,omitempty"` + PeerUploadedBytes *int64 `json:"peerUploadedBytes,omitempty"` Peers *int `json:"peers,omitempty"` Phase *PatchApiDownloadTasksId200JSONResponseBodyDetailPhase `json:"phase,omitempty"` Seeders *int `json:"seeders,omitempty"` @@ -4279,10 +4273,20 @@ func (r PostApiDownloaderHeartbeatResponse) ContentType() string { type PostApiObjectsResponse struct { Body []byte HTTPResponse *http.Response - JSON200 *ObjectDraft - JSON201 *ObjectDraft - JSON403 *ErrorResponse - JSON409 *ErrorResponse + JSON200 *struct { + ContentDisposition *string `json:"contentDisposition,omitempty"` + Id string `json:"id"` + Name string `json:"name"` + UploadUrl *string `json:"uploadUrl,omitempty"` + } + JSON201 *struct { + ContentDisposition *string `json:"contentDisposition,omitempty"` + Id string `json:"id"` + Name string `json:"name"` + UploadUrl *string `json:"uploadUrl,omitempty"` + } + JSON403 *ErrorResponse + JSON409 *ErrorResponse } // Status returns HTTPResponse.Status @@ -4312,9 +4316,14 @@ func (r PostApiObjectsResponse) ContentType() string { type PatchApiObjectsIdResponse struct { Body []byte HTTPResponse *http.Response - JSON200 *ObjectDraft - JSON403 *ErrorResponse - JSON404 *ErrorResponse + JSON200 *struct { + ContentDisposition *string `json:"contentDisposition,omitempty"` + Id string `json:"id"` + Name string `json:"name"` + UploadUrl *string `json:"uploadUrl,omitempty"` + } + JSON403 *ErrorResponse + JSON404 *ErrorResponse } // Status returns HTTPResponse.Status @@ -4344,10 +4353,19 @@ func (r PatchApiObjectsIdResponse) ContentType() string { type PostApiObjectsIdUploadsResponse struct { Body []byte HTTPResponse *http.Response - JSON201 *ObjectUploadSession - JSON400 *ErrorResponse - JSON403 *ErrorResponse - JSON404 *ErrorResponse + JSON201 *struct { + CreatedAt string `json:"createdAt"` + ExpiresAt string `json:"expiresAt"` + Id string `json:"id"` + ObjectId string `json:"objectId"` + PartSize int `json:"partSize"` + Status PostApiObjectsIdUploads201JSONResponseBodyStatus `json:"status"` + UpdatedAt string `json:"updatedAt"` + UploadId string `json:"uploadId"` + } + JSON400 *ErrorResponse + JSON403 *ErrorResponse + JSON404 *ErrorResponse } // Status returns HTTPResponse.Status @@ -4377,10 +4395,19 @@ func (r PostApiObjectsIdUploadsResponse) ContentType() string { type PatchApiObjectsIdUploadsUploadSessionIdResponse struct { Body []byte HTTPResponse *http.Response - JSON200 *ObjectUploadSession - JSON400 *ErrorResponse - JSON403 *ErrorResponse - JSON404 *ErrorResponse + JSON200 *struct { + CreatedAt string `json:"createdAt"` + ExpiresAt string `json:"expiresAt"` + Id string `json:"id"` + ObjectId string `json:"objectId"` + PartSize int `json:"partSize"` + Status PatchApiObjectsIdUploadsUploadSessionId200JSONResponseBodyStatus `json:"status"` + UpdatedAt string `json:"updatedAt"` + UploadId string `json:"uploadId"` + } + JSON400 *ErrorResponse + JSON403 *ErrorResponse + JSON404 *ErrorResponse } // Status returns HTTPResponse.Status @@ -4410,10 +4437,17 @@ func (r PatchApiObjectsIdUploadsUploadSessionIdResponse) ContentType() string { type PostApiObjectsIdUploadsUploadSessionIdPartsResponse struct { Body []byte HTTPResponse *http.Response - JSON200 *PresignObjectUploadPartsResponse - JSON400 *ErrorResponse - JSON403 *ErrorResponse - JSON404 *ErrorResponse + JSON200 *struct { + PartSize int `json:"partSize"` + Parts []struct { + PartNumber int `json:"partNumber"` + Url string `json:"url"` + } `json:"parts"` + UploadId string `json:"uploadId"` + } + JSON400 *ErrorResponse + JSON403 *ErrorResponse + JSON404 *ErrorResponse } // Status returns HTTPResponse.Status @@ -4993,10 +5027,10 @@ func ParseGetApiDownloadTasksResponse(rsp *http.Response) (*GetApiDownloadTasksR EngineState *string `json:"engineState,omitempty"` EtaSeconds *int `json:"etaSeconds,omitempty"` Files *[]struct { - CompletedBytes *int `json:"completedBytes,omitempty"` + CompletedBytes *int64 `json:"completedBytes,omitempty"` Path string `json:"path"` Selected *bool `json:"selected,omitempty"` - Size int `json:"size"` + Size int64 `json:"size"` } `json:"files,omitempty"` InfoHash *string `json:"infoHash,omitempty"` Leechers *int `json:"leechers,omitempty"` @@ -5004,12 +5038,12 @@ func ParseGetApiDownloadTasksResponse(rsp *http.Response) (*GetApiDownloadTasksR PeerSamples *[]struct { Address string `json:"address"` Client *string `json:"client,omitempty"` - DownloadBps *int `json:"downloadBps,omitempty"` + DownloadBps *int64 `json:"downloadBps,omitempty"` Progress *float32 `json:"progress,omitempty"` - UploadBps *int `json:"uploadBps,omitempty"` + UploadBps *int64 `json:"uploadBps,omitempty"` } `json:"peerSamples,omitempty"` - PeerUploadBps *int `json:"peerUploadBps,omitempty"` - PeerUploadedBytes *int `json:"peerUploadedBytes,omitempty"` + PeerUploadBps *int64 `json:"peerUploadBps,omitempty"` + PeerUploadedBytes *int64 `json:"peerUploadedBytes,omitempty"` Peers *int `json:"peers,omitempty"` Phase *GetApiDownloadTasks200JSONResponseBodyItemsDetailPhase `json:"phase,omitempty"` Seeders *int `json:"seeders,omitempty"` @@ -5086,10 +5120,10 @@ func ParsePostApiDownloadTasksResponse(rsp *http.Response) (*PostApiDownloadTask EngineState *string `json:"engineState,omitempty"` EtaSeconds *int `json:"etaSeconds,omitempty"` Files *[]struct { - CompletedBytes *int `json:"completedBytes,omitempty"` + CompletedBytes *int64 `json:"completedBytes,omitempty"` Path string `json:"path"` Selected *bool `json:"selected,omitempty"` - Size int `json:"size"` + Size int64 `json:"size"` } `json:"files,omitempty"` InfoHash *string `json:"infoHash,omitempty"` Leechers *int `json:"leechers,omitempty"` @@ -5097,12 +5131,12 @@ func ParsePostApiDownloadTasksResponse(rsp *http.Response) (*PostApiDownloadTask PeerSamples *[]struct { Address string `json:"address"` Client *string `json:"client,omitempty"` - DownloadBps *int `json:"downloadBps,omitempty"` + DownloadBps *int64 `json:"downloadBps,omitempty"` Progress *float32 `json:"progress,omitempty"` - UploadBps *int `json:"uploadBps,omitempty"` + UploadBps *int64 `json:"uploadBps,omitempty"` } `json:"peerSamples,omitempty"` - PeerUploadBps *int `json:"peerUploadBps,omitempty"` - PeerUploadedBytes *int `json:"peerUploadedBytes,omitempty"` + PeerUploadBps *int64 `json:"peerUploadBps,omitempty"` + PeerUploadedBytes *int64 `json:"peerUploadedBytes,omitempty"` Peers *int `json:"peers,omitempty"` Phase *PostApiDownloadTasks201JSONResponseBodyDetailPhase `json:"phase,omitempty"` Seeders *int `json:"seeders,omitempty"` @@ -5221,10 +5255,10 @@ func ParseGetApiDownloadTasksIdResponse(rsp *http.Response) (*GetApiDownloadTask EngineState *string `json:"engineState,omitempty"` EtaSeconds *int `json:"etaSeconds,omitempty"` Files *[]struct { - CompletedBytes *int `json:"completedBytes,omitempty"` + CompletedBytes *int64 `json:"completedBytes,omitempty"` Path string `json:"path"` Selected *bool `json:"selected,omitempty"` - Size int `json:"size"` + Size int64 `json:"size"` } `json:"files,omitempty"` InfoHash *string `json:"infoHash,omitempty"` Leechers *int `json:"leechers,omitempty"` @@ -5232,12 +5266,12 @@ func ParseGetApiDownloadTasksIdResponse(rsp *http.Response) (*GetApiDownloadTask PeerSamples *[]struct { Address string `json:"address"` Client *string `json:"client,omitempty"` - DownloadBps *int `json:"downloadBps,omitempty"` + DownloadBps *int64 `json:"downloadBps,omitempty"` Progress *float32 `json:"progress,omitempty"` - UploadBps *int `json:"uploadBps,omitempty"` + UploadBps *int64 `json:"uploadBps,omitempty"` } `json:"peerSamples,omitempty"` - PeerUploadBps *int `json:"peerUploadBps,omitempty"` - PeerUploadedBytes *int `json:"peerUploadedBytes,omitempty"` + PeerUploadBps *int64 `json:"peerUploadBps,omitempty"` + PeerUploadedBytes *int64 `json:"peerUploadedBytes,omitempty"` Peers *int `json:"peers,omitempty"` Phase *GetApiDownloadTasksId200JSONResponseBodyDetailPhase `json:"phase,omitempty"` Seeders *int `json:"seeders,omitempty"` @@ -5310,10 +5344,10 @@ func ParsePatchApiDownloadTasksIdResponse(rsp *http.Response) (*PatchApiDownload EngineState *string `json:"engineState,omitempty"` EtaSeconds *int `json:"etaSeconds,omitempty"` Files *[]struct { - CompletedBytes *int `json:"completedBytes,omitempty"` + CompletedBytes *int64 `json:"completedBytes,omitempty"` Path string `json:"path"` Selected *bool `json:"selected,omitempty"` - Size int `json:"size"` + Size int64 `json:"size"` } `json:"files,omitempty"` InfoHash *string `json:"infoHash,omitempty"` Leechers *int `json:"leechers,omitempty"` @@ -5321,12 +5355,12 @@ func ParsePatchApiDownloadTasksIdResponse(rsp *http.Response) (*PatchApiDownload PeerSamples *[]struct { Address string `json:"address"` Client *string `json:"client,omitempty"` - DownloadBps *int `json:"downloadBps,omitempty"` + DownloadBps *int64 `json:"downloadBps,omitempty"` Progress *float32 `json:"progress,omitempty"` - UploadBps *int `json:"uploadBps,omitempty"` + UploadBps *int64 `json:"uploadBps,omitempty"` } `json:"peerSamples,omitempty"` - PeerUploadBps *int `json:"peerUploadBps,omitempty"` - PeerUploadedBytes *int `json:"peerUploadedBytes,omitempty"` + PeerUploadBps *int64 `json:"peerUploadBps,omitempty"` + PeerUploadedBytes *int64 `json:"peerUploadedBytes,omitempty"` Peers *int `json:"peers,omitempty"` Phase *PatchApiDownloadTasksId200JSONResponseBodyDetailPhase `json:"phase,omitempty"` Seeders *int `json:"seeders,omitempty"` @@ -5532,14 +5566,24 @@ func ParsePostApiObjectsResponse(rsp *http.Response) (*PostApiObjectsResponse, e switch { case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 200: - var dest ObjectDraft + var dest struct { + ContentDisposition *string `json:"contentDisposition,omitempty"` + Id string `json:"id"` + Name string `json:"name"` + UploadUrl *string `json:"uploadUrl,omitempty"` + } if err := json.Unmarshal(bodyBytes, &dest); err != nil { return nil, err } response.JSON200 = &dest case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 201: - var dest ObjectDraft + var dest struct { + ContentDisposition *string `json:"contentDisposition,omitempty"` + Id string `json:"id"` + Name string `json:"name"` + UploadUrl *string `json:"uploadUrl,omitempty"` + } if err := json.Unmarshal(bodyBytes, &dest); err != nil { return nil, err } @@ -5579,7 +5623,12 @@ func ParsePatchApiObjectsIdResponse(rsp *http.Response) (*PatchApiObjectsIdRespo switch { case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 200: - var dest ObjectDraft + var dest struct { + ContentDisposition *string `json:"contentDisposition,omitempty"` + Id string `json:"id"` + Name string `json:"name"` + UploadUrl *string `json:"uploadUrl,omitempty"` + } if err := json.Unmarshal(bodyBytes, &dest); err != nil { return nil, err } @@ -5619,7 +5668,16 @@ func ParsePostApiObjectsIdUploadsResponse(rsp *http.Response) (*PostApiObjectsId switch { case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 201: - var dest ObjectUploadSession + var dest struct { + CreatedAt string `json:"createdAt"` + ExpiresAt string `json:"expiresAt"` + Id string `json:"id"` + ObjectId string `json:"objectId"` + PartSize int `json:"partSize"` + Status PostApiObjectsIdUploads201JSONResponseBodyStatus `json:"status"` + UpdatedAt string `json:"updatedAt"` + UploadId string `json:"uploadId"` + } if err := json.Unmarshal(bodyBytes, &dest); err != nil { return nil, err } @@ -5666,7 +5724,16 @@ func ParsePatchApiObjectsIdUploadsUploadSessionIdResponse(rsp *http.Response) (* switch { case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 200: - var dest ObjectUploadSession + var dest struct { + CreatedAt string `json:"createdAt"` + ExpiresAt string `json:"expiresAt"` + Id string `json:"id"` + ObjectId string `json:"objectId"` + PartSize int `json:"partSize"` + Status PatchApiObjectsIdUploadsUploadSessionId200JSONResponseBodyStatus `json:"status"` + UpdatedAt string `json:"updatedAt"` + UploadId string `json:"uploadId"` + } if err := json.Unmarshal(bodyBytes, &dest); err != nil { return nil, err } @@ -5713,7 +5780,14 @@ func ParsePostApiObjectsIdUploadsUploadSessionIdPartsResponse(rsp *http.Response switch { case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 200: - var dest PresignObjectUploadPartsResponse + var dest struct { + PartSize int `json:"partSize"` + Parts []struct { + PartNumber int `json:"partNumber"` + Url string `json:"url"` + } `json:"parts"` + UploadId string `json:"uploadId"` + } if err := json.Unmarshal(bodyBytes, &dest); err != nil { return nil, err } diff --git a/downloader/internal/worker/api.go b/downloader/internal/worker/api.go new file mode 100644 index 00000000..472f02ff --- /dev/null +++ b/downloader/internal/worker/api.go @@ -0,0 +1,143 @@ +package worker + +import ( + "context" + "fmt" + "time" + + "github.com/saltbo/zpan/downloader/internal/client" +) + +const apiRetryAttempts = 3 + +func (w *Worker) updateTask(ctx context.Context, id string, patch client.TaskPatch) (client.DownloadTask, error) { + var task client.DownloadTask + err := w.callAPI(ctx, "update task", func(ctx context.Context) error { + var err error + task, err = w.api.UpdateTask(ctx, id, patch) + return err + }) + return task, err +} + +func (w *Worker) createFolder(ctx context.Context, token string, name string, parent string) (client.ObjectDraft, error) { + var draft client.ObjectDraft + err := w.callAPI(ctx, "create folder", func(ctx context.Context) error { + var err error + draft, err = w.api.CreateFolder(ctx, token, name, parent) + return err + }) + return draft, err +} + +func (w *Worker) createObject(ctx context.Context, token string, name string, size int64, parent string) (client.ObjectDraft, error) { + var draft client.ObjectDraft + err := w.callAPI(ctx, "create object", func(ctx context.Context) error { + var err error + draft, err = w.api.CreateObject(ctx, token, name, size, parent) + return err + }) + return draft, err +} + +func (w *Worker) confirmObject(ctx context.Context, token string, id string) error { + return w.callAPI(ctx, "confirm object", func(ctx context.Context) error { + return w.api.ConfirmObject(ctx, token, id) + }) +} + +func (w *Worker) createObjectUploadSession(ctx context.Context, token string, id string, partSize int64) (client.ObjectUploadSession, error) { + var session client.ObjectUploadSession + err := w.callAPI(ctx, "create multipart upload session", func(ctx context.Context) error { + var err error + session, err = w.api.CreateObjectUploadSession(ctx, token, id, partSize) + return err + }) + return session, err +} + +func (w *Worker) presignObjectUploadParts(ctx context.Context, token string, id string, sessionID string, partNumbers []int) ([]client.PresignedObjectUploadPart, error) { + var parts []client.PresignedObjectUploadPart + err := w.callAPI(ctx, "presign multipart upload parts", func(ctx context.Context) error { + var err error + parts, err = w.api.PresignObjectUploadParts(ctx, token, id, sessionID, partNumbers) + return err + }) + return parts, err +} + +func (w *Worker) completeObjectUploadSession(ctx context.Context, token string, id string, sessionID string, parts []client.CompletedObjectUploadPart) error { + return w.callAPI(ctx, "complete multipart upload session", func(ctx context.Context) error { + return w.api.CompleteObjectUploadSession(ctx, token, id, sessionID, parts) + }) +} + +func (w *Worker) abortObjectUploadSession(ctx context.Context, token string, id string, sessionID string) error { + return w.callAPI(ctx, "abort multipart upload session", func(ctx context.Context) error { + return w.api.AbortObjectUploadSession(ctx, token, id, sessionID) + }) +} + +func (w *Worker) callAPI(ctx context.Context, operation string, call func(context.Context) error) error { + var last error + for attempt := 1; attempt <= apiRetryAttempts; attempt++ { + if err := ctx.Err(); err != nil { + return err + } + if err := call(ctx); err != nil { + last = err + if attempt == apiRetryAttempts || !isRetryableAPIError(err) { + return err + } + delay := time.Duration(attempt) * 500 * time.Millisecond + w.logger.Warn("retrying downloader api call", "operation", operation, "attempt", attempt, "delay", delay.String(), "error", err) + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(delay): + } + continue + } + return nil + } + return fmt.Errorf("%s failed: %w", operation, last) +} + +func isRetryableAPIError(err error) bool { + if err == nil { + return false + } + message := err.Error() + return containsAny(message, + "connection refused", + "connection reset", + "connection is shut down", + "timeout", + "temporary failure", + "Too Many Requests", + "Bad Gateway", + "Service Unavailable", + "Gateway Timeout", + ) +} + +func containsAny(value string, needles ...string) bool { + for _, needle := range needles { + if needle != "" && contains(value, needle) { + return true + } + } + return false +} + +func contains(value string, needle string) bool { + if len(needle) > len(value) { + return false + } + for i := 0; i <= len(value)-len(needle); i++ { + if value[i:i+len(needle)] == needle { + return true + } + } + return false +} diff --git a/downloader/internal/worker/api_test.go b/downloader/internal/worker/api_test.go new file mode 100644 index 00000000..ecf720cf --- /dev/null +++ b/downloader/internal/worker/api_test.go @@ -0,0 +1,46 @@ +package worker + +import ( + "context" + "errors" + "testing" + + "github.com/saltbo/zpan/downloader/internal/config" +) + +func TestCallAPIRetriesTransientErrors(t *testing.T) { + w := NewWithAPI(config.Config{}, nil) + attempts := 0 + + err := w.callAPI(context.Background(), "test", func(context.Context) error { + attempts++ + if attempts < 3 { + return errors.New("503 Service Unavailable") + } + return nil + }) + + if err != nil { + t.Fatalf("expected retry to succeed, got %v", err) + } + if attempts != 3 { + t.Fatalf("expected 3 attempts, got %d", attempts) + } +} + +func TestCallAPIDoesNotRetryApplicationErrors(t *testing.T) { + w := NewWithAPI(config.Config{}, nil) + attempts := 0 + + err := w.callAPI(context.Background(), "test", func(context.Context) error { + attempts++ + return errors.New("Task is paused") + }) + + if err == nil { + t.Fatal("expected application error") + } + if attempts != 1 { + t.Fatalf("expected 1 attempt, got %d", attempts) + } +} diff --git a/downloader/internal/worker/engines.go b/downloader/internal/worker/engines.go new file mode 100644 index 00000000..82c3f37a --- /dev/null +++ b/downloader/internal/worker/engines.go @@ -0,0 +1,153 @@ +package worker + +import ( + "context" + "fmt" + "strings" + "time" + + "github.com/saltbo/zpan/downloader/internal/config" + "github.com/saltbo/zpan/downloader/internal/engine" +) + +func (w *Worker) resolveEngine(ctx context.Context) error { + if w.cfg.Engine == "" || w.cfg.Engine == "auto" { + return w.resolveAutoEngine(ctx) + } + downloader, err := configuredEngine(w.cfg) + if err != nil { + return err + } + w.logger.Info("checking configured downloader engine", "engine", downloader.Name()) + if w.checkEngine(ctx, downloader) == nil { + w.useEngine(downloader, "configured engine is already running") + return nil + } + if _, ok := downloader.(engine.Starter); !ok { + w.useEngine(downloader, "configured built-in engine") + return nil + } + if err := w.startEngine(ctx, downloader); err != nil { + w.logger.Warn("configured downloader engine could not be started", "engine", downloader.Name(), "error", err) + w.useEngine(downloader, "configured engine selected despite failed auto start") + return nil + } + w.useEngine(downloader, "configured engine started") + return nil +} + +func (w *Worker) resolveAutoEngine(ctx context.Context) error { + candidates := externalEngines(w.cfg) + w.logger.Info("auto selecting downloader engine", "priority", engineNames(candidates)) + for _, downloader := range candidates { + w.logger.Info("checking downloader engine availability", "engine", downloader.Name()) + if err := w.checkEngine(ctx, downloader); err == nil { + w.useEngine(downloader, "engine is already running") + return nil + } else { + w.logger.Info("downloader engine is not running", "engine", downloader.Name(), "error", err) + } + } + for _, downloader := range candidates { + w.logger.Info("checking downloader engine binary", "engine", downloader.Name()) + if err := w.startEngine(ctx, downloader); err != nil { + w.logger.Info("downloader engine is not available for auto start", "engine", downloader.Name(), "error", err) + continue + } + w.useEngine(downloader, "engine binary found and started") + return nil + } + downloader := engine.HTTP{Dir: w.cfg.DownloadDir} + w.useEngine(downloader, "no external downloader engine is running or installed") + return nil +} + +func (w *Worker) useEngine(downloader engine.Engine, reason string) { + w.cfg.Engine = downloader.Name() + w.engine = downloader + w.logger.Info("selected downloader engine", "engine", downloader.Name(), "reason", reason) +} + +func (w *Worker) checkEngine(ctx context.Context, downloader engine.Engine) error { + checkCtx, cancel := context.WithTimeout(ctx, 2*time.Second) + defer cancel() + return downloader.Check(checkCtx) +} + +func (w *Worker) startEngine(ctx context.Context, downloader engine.Engine) error { + starter, ok := downloader.(engine.Starter) + if !ok { + return fmt.Errorf("%s cannot be auto started", downloader.Name()) + } + w.logger.Info("starting downloader engine", "engine", downloader.Name()) + cmd, err := starter.Start(ctx) + if err != nil { + return err + } + if cmd.Process != nil { + w.logger.Info("downloader engine process started", "engine", downloader.Name(), "pid", cmd.Process.Pid) + } + w.started = append(w.started, cmd) + if err := waitForEngine(ctx, downloader); err != nil { + if cmd.Process != nil { + _ = cmd.Process.Kill() + } + return err + } + return nil +} + +func (w *Worker) stopStartedEngines() { + for _, cmd := range w.started { + if cmd.Process == nil { + continue + } + w.logger.Info("stopping auto-started downloader engine", "pid", cmd.Process.Pid) + _ = cmd.Process.Kill() + } +} + +func configuredEngine(cfg config.Config) (engine.Engine, error) { + for _, downloader := range append(externalEngines(cfg), engine.HTTP{Dir: cfg.DownloadDir}) { + if downloader.Name() == cfg.Engine { + return downloader, nil + } + } + return nil, fmt.Errorf("unsupported downloader engine %q; expected auto, builtin, aria2, or qbittorrent", cfg.Engine) +} + +func externalEngines(cfg config.Config) []engine.Engine { + return []engine.Engine{ + engine.Aria2{URL: cfg.Aria2URL, Secret: cfg.Aria2Secret, Dir: cfg.DownloadDir, RetainSeed: cfg.SeedEnabled}, + engine.QBittorrent{URL: cfg.QBittorrentURL, Username: cfg.QBittorrentUser, Password: cfg.QBittorrentPass, Dir: cfg.DownloadDir, RetainSeed: cfg.SeedEnabled}, + } +} + +func waitForEngine(ctx context.Context, downloader engine.Engine) error { + deadline := time.Now().Add(8 * time.Second) + var lastErr error + for time.Now().Before(deadline) { + checkCtx, cancel := context.WithTimeout(ctx, time.Second) + err := downloader.Check(checkCtx) + cancel() + if err == nil { + return nil + } + lastErr = err + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(500 * time.Millisecond): + } + } + return lastErr +} + +func engineNames(engines []engine.Engine) string { + names := make([]string, 0, len(engines)+1) + for _, downloader := range engines { + names = append(names, downloader.Name()) + } + names = append(names, "builtin") + return strings.Join(names, ",") +} diff --git a/downloader/internal/worker/seeds.go b/downloader/internal/worker/seeds.go new file mode 100644 index 00000000..73b6cc59 --- /dev/null +++ b/downloader/internal/worker/seeds.go @@ -0,0 +1,181 @@ +package worker + +import ( + "context" + "log/slog" + "os" + "sort" + "time" + + "github.com/saltbo/zpan/downloader/internal/client" + "github.com/saltbo/zpan/downloader/internal/engine" +) + +const retainedSeedReportInterval = 5 * time.Second + +type retainedSeed struct { + taskID string + engine string + seedID string + path string + retainedAt time.Time + expiresAt time.Time + snapshot func(context.Context) (engine.SeedSnapshot, error) + cleanup func(context.Context) error +} + +func cleanupDownloadedResult(ctx context.Context, result engine.Result) error { + if result.Seed != nil && result.Seed.Cleanup != nil { + return result.Seed.Cleanup(ctx) + } + return os.RemoveAll(result.Path) +} + +func (w *Worker) retainSeed(task client.DownloadTask, result engine.Result, log *slog.Logger) bool { + if !w.cfg.SeedEnabled || result.Seed == nil || result.Seed.Cleanup == nil || result.Seed.Snapshot == nil { + return false + } + now := time.Now() + seed := retainedSeed{ + taskID: task.ID, + engine: result.Seed.Engine, + seedID: result.Seed.ID, + path: result.Seed.Path, + retainedAt: now, + snapshot: result.Seed.Snapshot, + cleanup: result.Seed.Cleanup, + } + if w.cfg.SeedDuration > 0 { + seed.expiresAt = now.Add(w.cfg.SeedDuration) + } + w.mu.Lock() + w.retainedSeeds = append(w.retainedSeeds, seed) + count := len(w.retainedSeeds) + w.mu.Unlock() + + log.Info("retaining completed bt task for seeding", + "engine", seed.engine, + "seed_id", seed.seedID, + "path", seed.path, + "expires_at", optionalTime(seed.expiresAt), + "retained_seeds", count, + ) + return true +} + +func (w *Worker) reportRetainedSeeds(ctx context.Context) { + for _, seed := range w.retainedSeedSnapshot() { + log := w.logger.With("task_id", seed.taskID, "engine", seed.engine, "seed_id", seed.seedID) + snapshot, err := seed.snapshot(ctx) + if err != nil { + log.Warn("failed to inspect retained bt seed", "error", err) + continue + } + if snapshot.Detail == nil { + continue + } + snapshot.Detail.Phase = "seeding" + zero := int64(0) + _, err = w.updateTask(ctx, seed.taskID, client.TaskPatch{ + DownloadedBytes: &snapshot.Downloaded, + TotalBytes: snapshot.Total, + DownloadBps: &snapshot.Bps, + StorageUploadBps: &zero, + Detail: snapshot.Detail, + }) + if err != nil { + log.Warn("failed to report retained bt seed", "error", err) + continue + } + log.Debug("reported retained bt seed", "downloaded_bytes", snapshot.Downloaded, "bps", snapshot.Bps) + } +} + +func (w *Worker) cleanupRetainedSeeds(ctx context.Context) { + seeds := w.retainedSeedSnapshot() + if len(seeds) == 0 { + return + } + + reasons := map[string]string{} + now := time.Now() + for _, seed := range seeds { + if !seed.expiresAt.IsZero() && !now.Before(seed.expiresAt) { + reasons[seed.taskID] = "expired" + } + } + + if w.cfg.SeedCacheLimit > 0 { + type seedSize struct { + seed retainedSeed + size int64 + } + sized := make([]seedSize, 0, len(seeds)) + var total int64 + for _, seed := range seeds { + if reasons[seed.taskID] != "" { + continue + } + size, err := directorySize(seed.path) + if err != nil { + w.logger.Warn("failed to inspect retained seed size", "task_id", seed.taskID, "path", seed.path, "error", err) + continue + } + total += size + sized = append(sized, seedSize{seed: seed, size: size}) + } + sort.Slice(sized, func(i, j int) bool { + return sized[i].seed.retainedAt.Before(sized[j].seed.retainedAt) + }) + for _, item := range sized { + if total <= w.cfg.SeedCacheLimit { + break + } + reasons[item.seed.taskID] = "cache_limit" + total -= item.size + } + } + + for _, seed := range seeds { + reason := reasons[seed.taskID] + if reason == "" { + continue + } + w.cleanupRetainedSeed(ctx, seed, reason) + } +} + +func (w *Worker) retainedSeedSnapshot() []retainedSeed { + w.mu.Lock() + defer w.mu.Unlock() + return append([]retainedSeed(nil), w.retainedSeeds...) +} + +func (w *Worker) cleanupRetainedSeed(ctx context.Context, seed retainedSeed, reason string) { + w.logger.Info("cleaning retained bt seed", + "task_id", seed.taskID, + "engine", seed.engine, + "seed_id", seed.seedID, + "path", seed.path, + "reason", reason, + ) + if err := seed.cleanup(ctx); err != nil { + w.logger.Warn("failed to clean retained bt seed", + "task_id", seed.taskID, + "engine", seed.engine, + "seed_id", seed.seedID, + "path", seed.path, + "error", err, + ) + return + } + w.mu.Lock() + next := w.retainedSeeds[:0] + for _, retained := range w.retainedSeeds { + if retained.taskID != seed.taskID { + next = append(next, retained) + } + } + w.retainedSeeds = next + w.mu.Unlock() +} diff --git a/downloader/internal/worker/uploader.go b/downloader/internal/worker/uploader.go new file mode 100644 index 00000000..5ccddbdd --- /dev/null +++ b/downloader/internal/worker/uploader.go @@ -0,0 +1,306 @@ +package worker + +import ( + "context" + "fmt" + "io" + "log/slog" + "net/http" + "os" + "path" + "strings" + "time" + + "github.com/saltbo/zpan/downloader/internal/client" + "github.com/saltbo/zpan/downloader/internal/engine" +) + +const maxSingleUploadSize = 5 * 1024 * 1024 * 1024 +const defaultMultipartPartSize = 64 * 1024 * 1024 +const maxMultipartPartSize = 512 * 1024 * 1024 +const maxMultipartParts = 10_000 +const presignMultipartPartsBatchSize = 100 + +type uploadProgress struct { + totalBytes int64 + uploaded int64 + lastAt time.Time + lastBytes int64 +} + +func (w *Worker) uploadResult( + ctx context.Context, + log *slog.Logger, + task client.DownloadTask, + result engine.Result, +) (string, error) { + if !result.IsDir { + progress := &uploadProgress{totalBytes: result.Size, lastAt: time.Now()} + return w.uploadSingleFile(ctx, log, task, result.Path, result.Name, result.Size, task.TargetFolder, progress) + } + + progress := &uploadProgress{totalBytes: result.Size, lastAt: time.Now()} + log.Info("creating remote folder", "name", result.Name, "size", result.Size, "target_folder", task.TargetFolder) + root, err := w.createFolder(ctx, task.UploadToken, result.Name, task.TargetFolder) + if err != nil { + return "", fmt.Errorf("create remote folder: %w", err) + } + rootPath := joinObjectPath(task.TargetFolder, root.Name) + entries, err := collectDirectoryEntries(result.Path) + if err != nil { + return "", err + } + for _, entry := range entries { + parent := joinObjectPath(rootPath, path.Dir(entry.relativePath)) + if entry.isDir { + log.Debug("creating remote subfolder", "name", entry.name, "parent", parent) + if _, err := w.createFolder(ctx, task.UploadToken, entry.name, parent); err != nil { + return "", fmt.Errorf("create remote subfolder %s: %w", entry.relativePath, err) + } + continue + } + if _, err := w.uploadSingleFile(ctx, log, task, entry.path, entry.name, entry.size, parent, progress); err != nil { + return "", err + } + } + return root.ID, nil +} + +func (w *Worker) uploadSingleFile( + ctx context.Context, + log *slog.Logger, + task client.DownloadTask, + path string, + name string, + size int64, + parent string, + progress *uploadProgress, +) (string, error) { + log.Info("creating remote object", "name", name, "size", size, "target_folder", parent) + draft, err := w.createObject(ctx, task.UploadToken, name, size, parent) + if err != nil { + return "", fmt.Errorf("create remote object: %w", err) + } + log.Info("uploading file to object storage", "object_id", draft.ID, "path", path) + if size > maxSingleUploadSize { + if err := w.uploadMultipartFile(ctx, log, task, draft.ID, path, size, progress); err != nil { + return "", fmt.Errorf("upload object %s: %w", draft.ID, err) + } + } else { + if err := uploadFile(ctx, draft.UploadURL, path, draft.ContentDisposition, func(written int64) error { + return w.reportUploadProgress(ctx, log, task, progress, written) + }); err != nil { + return "", fmt.Errorf("upload object %s: %w", draft.ID, err) + } + } + log.Info("confirming uploaded object", "object_id", draft.ID) + if err := w.confirmObject(ctx, task.UploadToken, draft.ID); err != nil { + return "", fmt.Errorf("confirm object %s: %w", draft.ID, err) + } + return draft.ID, nil +} + +func (w *Worker) uploadMultipartFile( + ctx context.Context, + log *slog.Logger, + task client.DownloadTask, + objectID string, + filePath string, + size int64, + progress *uploadProgress, +) error { + partSize := multipartPartSize(size) + log.Info("creating multipart upload session", "object_id", objectID, "part_size", partSize) + session, err := w.createObjectUploadSession(ctx, task.UploadToken, objectID, partSize) + if err != nil { + return fmt.Errorf("create multipart upload session: %w", err) + } + if session.PartSize <= 0 { + return fmt.Errorf("create multipart upload session: invalid part size %d", session.PartSize) + } + completed := false + defer func() { + if completed { + return + } + abortCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second) + defer cancel() + if abortErr := w.abortObjectUploadSession(abortCtx, task.UploadToken, objectID, session.ID); abortErr != nil { + log.Warn("failed to abort multipart upload session", "object_id", objectID, "upload_session_id", session.ID, "error", abortErr) + } + }() + + file, err := os.Open(filePath) + if err != nil { + return err + } + defer file.Close() + + totalParts := int((size + session.PartSize - 1) / session.PartSize) + parts := make([]client.CompletedObjectUploadPart, 0, totalParts) + for firstPart := 1; firstPart <= totalParts; firstPart += presignMultipartPartsBatchSize { + lastPart := firstPart + presignMultipartPartsBatchSize - 1 + if lastPart > totalParts { + lastPart = totalParts + } + partNumbers := make([]int, 0, lastPart-firstPart+1) + for partNumber := firstPart; partNumber <= lastPart; partNumber++ { + partNumbers = append(partNumbers, partNumber) + } + presignedParts, err := w.presignObjectUploadParts(ctx, task.UploadToken, objectID, session.ID, partNumbers) + if err != nil { + return fmt.Errorf("presign multipart upload parts: %w", err) + } + byNumber := make(map[int]string, len(presignedParts)) + for _, part := range presignedParts { + byNumber[part.PartNumber] = part.URL + } + for _, partNumber := range partNumbers { + url, ok := byNumber[partNumber] + if !ok { + return fmt.Errorf("presign multipart upload part %d: missing upload URL", partNumber) + } + offset := int64(partNumber-1) * session.PartSize + length := session.PartSize + if remaining := size - offset; remaining < length { + length = remaining + } + etag, err := uploadFilePart(ctx, url, file, offset, length, func(written int64) error { + return w.reportUploadProgress(ctx, log, task, progress, written) + }) + if err != nil { + return fmt.Errorf("upload part %d: %w", partNumber, err) + } + if etag == "" { + return fmt.Errorf("upload part %d: missing ETag", partNumber) + } + parts = append(parts, client.CompletedObjectUploadPart{PartNumber: partNumber, ETag: etag}) + } + } + if err := w.completeObjectUploadSession(ctx, task.UploadToken, objectID, session.ID, parts); err != nil { + return fmt.Errorf("complete multipart upload session: %w", err) + } + completed = true + return nil +} + +func (w *Worker) reportUploadProgress( + ctx context.Context, + log *slog.Logger, + task client.DownloadTask, + progress *uploadProgress, + written int64, +) error { + progress.uploaded += written + now := time.Now() + if progress.uploaded < progress.totalBytes && now.Sub(progress.lastAt) < time.Second { + return nil + } + elapsed := now.Sub(progress.lastAt).Seconds() + var bps int64 + if elapsed > 0 { + bps = int64(float64(progress.uploaded-progress.lastBytes) / elapsed) + } + detail := task.Detail + if detail == nil { + detail = &client.DownloadTaskDetail{} + } + detail.Phase = "uploading" + detail.ETASeconds = uploadETA(progress, bps) + detail.PeerUploadBps = nil + zero := int64(0) + _, err := w.updateTask(ctx, task.ID, client.TaskPatch{ + Status: "uploading", + StorageUploadedBytes: &progress.uploaded, + DownloadBps: &zero, + StorageUploadBps: &bps, + Detail: detail, + }) + if err != nil { + log.Error("failed to report upload progress", "uploaded_bytes", progress.uploaded, "total_bytes", progress.totalBytes, "bps", bps, "error", err) + return err + } + log.Debug("task upload progress", "uploaded_bytes", progress.uploaded, "total_bytes", progress.totalBytes, "bps", bps) + progress.lastAt = now + progress.lastBytes = progress.uploaded + return nil +} + +func uploadFile(ctx context.Context, url, path string, contentDisposition string, progress func(written int64) error) error { + file, err := os.Open(path) + if err != nil { + return err + } + defer file.Close() + stat, err := file.Stat() + if err != nil { + return err + } + reader := io.Reader(file) + if progress != nil { + reader = &uploadProgressReader{reader: file, progress: progress} + } + req, err := http.NewRequestWithContext(ctx, http.MethodPut, url, reader) + if err != nil { + return err + } + req.Header.Set("Content-Type", "application/octet-stream") + if contentDisposition != "" { + req.Header.Set("Content-Disposition", contentDisposition) + } + req.ContentLength = stat.Size() + res, err := http.DefaultClient.Do(req) + if err != nil { + return err + } + defer res.Body.Close() + if res.StatusCode < 200 || res.StatusCode >= 300 { + body, _ := io.ReadAll(io.LimitReader(res.Body, 4096)) + if len(body) > 0 { + return fmt.Errorf("upload failed: %s: %s", res.Status, strings.TrimSpace(string(body))) + } + return fmt.Errorf("upload failed: %s", res.Status) + } + return nil +} + +func uploadFilePart(ctx context.Context, url string, file *os.File, offset int64, length int64, progress func(written int64) error) (string, error) { + reader := io.NewSectionReader(file, offset, length) + var body io.Reader = reader + if progress != nil { + body = &uploadProgressReader{reader: reader, progress: progress} + } + req, err := http.NewRequestWithContext(ctx, http.MethodPut, url, body) + if err != nil { + return "", err + } + req.ContentLength = length + res, err := http.DefaultClient.Do(req) + if err != nil { + return "", err + } + defer res.Body.Close() + if res.StatusCode < 200 || res.StatusCode >= 300 { + body, _ := io.ReadAll(io.LimitReader(res.Body, 4096)) + if len(body) > 0 { + return "", fmt.Errorf("upload failed: %s: %s", res.Status, strings.TrimSpace(string(body))) + } + return "", fmt.Errorf("upload failed: %s", res.Status) + } + return res.Header.Get("ETag"), nil +} + +type uploadProgressReader struct { + reader io.Reader + progress func(written int64) error +} + +func (r *uploadProgressReader) Read(p []byte) (int, error) { + n, err := r.reader.Read(p) + if n > 0 { + if progressErr := r.progress(int64(n)); progressErr != nil { + return n, progressErr + } + } + return n, err +} diff --git a/downloader/internal/worker/worker.go b/downloader/internal/worker/worker.go index 649340fd..72a3e47d 100644 --- a/downloader/internal/worker/worker.go +++ b/downloader/internal/worker/worker.go @@ -4,17 +4,12 @@ import ( "context" "errors" "fmt" - "io" "log/slog" - "net/http" - "net/url" "os" "os/exec" - "path" "path/filepath" "runtime" "sort" - "strconv" "strings" "sync" "time" @@ -26,12 +21,6 @@ import ( const Version = "0.1.0" const maxTaskErrorMessageLength = 1000 -const retainedSeedReportInterval = 5 * time.Second -const maxSingleUploadSize = 5 * 1024 * 1024 * 1024 -const defaultMultipartPartSize = 64 * 1024 * 1024 -const maxMultipartPartSize = 512 * 1024 * 1024 -const maxMultipartParts = 10_000 -const presignMultipartPartsBatchSize = 100 var errBillingPaused = errors.New("billing paused") var errTaskPausing = errors.New("task pausing") @@ -39,7 +28,7 @@ var errTaskCanceling = errors.New("task canceling") type Worker struct { cfg config.Config - api *client.Client + api apiClient engine engine.Engine logger *slog.Logger running map[string]context.CancelCauseFunc @@ -48,21 +37,32 @@ type Worker struct { mu sync.Mutex } -type retainedSeed struct { - taskID string - engine string - seedID string - path string - retainedAt time.Time - expiresAt time.Time - snapshot func(context.Context) (engine.SeedSnapshot, error) - cleanup func(context.Context) error +type apiClient interface { + Heartbeat(context.Context, client.Heartbeat) error + AssignedControlTasks(context.Context) ([]client.DownloadTask, error) + AssignedTasks(context.Context) ([]client.DownloadTask, error) + UpdateTask(context.Context, string, client.TaskPatch) (client.DownloadTask, error) + CreateFolder(context.Context, string, string, string) (client.ObjectDraft, error) + CreateObject(context.Context, string, string, int64, string) (client.ObjectDraft, error) + ConfirmObject(context.Context, string, string) error + CreateObjectUploadSession(context.Context, string, string, int64) (client.ObjectUploadSession, error) + PresignObjectUploadParts(context.Context, string, string, string, []int) ([]client.PresignedObjectUploadPart, error) + CompleteObjectUploadSession(context.Context, string, string, string, []client.CompletedObjectUploadPart) error + AbortObjectUploadSession(context.Context, string, string, string) error } -func New(cfg config.Config) *Worker { +func New(cfg config.Config) (*Worker, error) { + api, err := client.New(cfg.ServerURL, cfg.Token) + if err != nil { + return nil, err + } + return NewWithAPI(cfg, api), nil +} + +func NewWithAPI(cfg config.Config, api apiClient) *Worker { return &Worker{ cfg: cfg, - api: client.New(cfg.ServerURL, cfg.Token), + api: api, logger: slog.Default(), running: map[string]context.CancelCauseFunc{}, } @@ -124,10 +124,17 @@ func (w *Worker) Run(ctx context.Context) error { } func (w *Worker) tick(ctx context.Context) error { - if err := w.api.Heartbeat(ctx, w.heartbeat()); err != nil { + if err := w.callAPI(ctx, "heartbeat", func(ctx context.Context) error { + return w.api.Heartbeat(ctx, w.heartbeat()) + }); err != nil { return err } - controlTasks, err := w.api.AssignedControlTasks(ctx) + var controlTasks []client.DownloadTask + err := w.callAPI(ctx, "assigned control tasks", func(ctx context.Context) error { + var err error + controlTasks, err = w.api.AssignedControlTasks(ctx) + return err + }) if err != nil { return err } @@ -137,7 +144,12 @@ func (w *Worker) tick(ctx context.Context) error { } w.ackStoppedControlTask(ctx, task) } - tasks, err := w.api.AssignedTasks(ctx) + var tasks []client.DownloadTask + err = w.callAPI(ctx, "assigned tasks", func(ctx context.Context) error { + var err error + tasks, err = w.api.AssignedTasks(ctx) + return err + }) if err != nil { return err } @@ -162,7 +174,26 @@ func (w *Worker) process(ctx context.Context, task client.DownloadTask) { defer w.finish(task.ID) log := w.taskLogger(task) log.Info("task started", "source_uri", task.SourceURI, "target_folder", task.TargetFolder) - if _, err := w.api.UpdateTask(ctx, task.ID, client.TaskPatch{Status: "running"}); err != nil { + currentDetail := task.Detail + if task.Status == "uploading" { + result, recovered, err := w.engine.Recover(ctx, task) + if err != nil { + msg := taskErrorMessage(err) + log.Error("failed to recover uploading task", "error", err) + if _, updateErr := w.updateTask(ctx, task.ID, client.TaskPatch{Status: "failed", ErrorMessage: &msg}); updateErr != nil { + log.Error("failed to mark task failed", "error", updateErr) + } + return + } + if recovered { + log.Info("recovered completed download result", "path", result.Path, "name", result.Name, "size", result.Size) + w.uploadAndComplete(ctx, log, task, result, currentDetail) + return + } + log.Warn("uploading task has no recoverable local result; restarting download") + } + + if _, err := w.updateTask(ctx, task.ID, client.TaskPatch{Status: "running"}); err != nil { log.Error("failed to mark task running", "error", err) if w.resolveControlledTaskUpdate(ctx, task.ID, err, log) { return @@ -170,13 +201,12 @@ func (w *Worker) process(ctx context.Context, task client.DownloadTask) { } var lastProgressLog time.Time - currentDetail := task.Detail result, err := w.engine.Download(ctx, task, func(downloaded int64, total *int64, bps int64, detail *client.DownloadTaskDetail) error { detail = withDownloadETA(detail, downloaded, total, bps) if detail != nil { currentDetail = detail } - _, updateErr := w.api.UpdateTask(ctx, task.ID, client.TaskPatch{ + _, updateErr := w.updateTask(ctx, task.ID, client.TaskPatch{ DownloadedBytes: &downloaded, TotalBytes: total, DownloadBps: &bps, @@ -199,14 +229,14 @@ func (w *Worker) process(ctx context.Context, task client.DownloadTask) { if err != nil { if errors.Is(err, context.Canceled) { if errors.Is(context.Cause(ctx), errTaskPausing) { - if _, updateErr := w.api.UpdateTask(context.WithoutCancel(ctx), task.ID, client.TaskPatch{Status: "paused"}); updateErr != nil { + if _, updateErr := w.updateTask(context.WithoutCancel(ctx), task.ID, client.TaskPatch{Status: "paused"}); updateErr != nil { log.Error("failed to mark task paused", "error", updateErr) } log.Info("task paused by control action") return } if errors.Is(context.Cause(ctx), errTaskCanceling) { - if _, updateErr := w.api.UpdateTask(context.WithoutCancel(ctx), task.ID, client.TaskPatch{Status: "canceled"}); updateErr != nil { + if _, updateErr := w.updateTask(context.WithoutCancel(ctx), task.ID, client.TaskPatch{Status: "canceled"}); updateErr != nil { log.Error("failed to mark task canceled", "error", updateErr) } log.Info("task canceled by control action") @@ -228,15 +258,25 @@ func (w *Worker) process(ctx context.Context, task client.DownloadTask) { } msg := taskErrorMessage(err) log.Error("task download failed", "error", err) - if _, updateErr := w.api.UpdateTask(ctx, task.ID, client.TaskPatch{Status: "failed", ErrorMessage: &msg}); updateErr != nil { + if _, updateErr := w.updateTask(ctx, task.ID, client.TaskPatch{Status: "failed", ErrorMessage: &msg}); updateErr != nil { log.Error("failed to mark task failed", "error", updateErr) } return } log.Debug("task download completed", "path", result.Path, "name", result.Name, "size", result.Size) + w.uploadAndComplete(ctx, log, task, result, currentDetail) +} + +func (w *Worker) uploadAndComplete( + ctx context.Context, + log *slog.Logger, + task client.DownloadTask, + result engine.Result, + currentDetail *client.DownloadTaskDetail, +) { zero := int64(0) - if _, err := w.api.UpdateTask(ctx, task.ID, client.TaskPatch{Status: "uploading", DownloadBps: &zero}); err != nil { + if _, err := w.updateTask(ctx, task.ID, client.TaskPatch{Status: "uploading", DownloadBps: &zero}); err != nil { log.Error("failed to mark task uploading", "error", err) } task.Detail = currentDetail @@ -244,7 +284,7 @@ func (w *Worker) process(ctx context.Context, task client.DownloadTask) { if err != nil { msg := taskErrorMessage(err) log.Error("failed to upload result", "error", err) - if _, updateErr := w.api.UpdateTask(ctx, task.ID, client.TaskPatch{Status: "failed", ErrorMessage: &msg}); updateErr != nil { + if _, updateErr := w.updateTask(ctx, task.ID, client.TaskPatch{Status: "failed", ErrorMessage: &msg}); updateErr != nil { log.Error("failed to mark task failed", "error", updateErr) } return @@ -256,7 +296,7 @@ func (w *Worker) process(ctx context.Context, task client.DownloadTask) { } completedDetail.Phase = "completed" completedDetail.PeerUploadBps = nil - if _, err := w.api.UpdateTask(ctx, task.ID, client.TaskPatch{ + if _, err := w.updateTask(ctx, task.ID, client.TaskPatch{ Status: "completed", ResultObjectID: &resultObjectID, StorageUploadedBytes: &uploadedBytes, @@ -278,367 +318,6 @@ func (w *Worker) process(ctx context.Context, task client.DownloadTask) { } } -func cleanupDownloadedResult(ctx context.Context, result engine.Result) error { - if result.Seed != nil && result.Seed.Cleanup != nil { - return result.Seed.Cleanup(ctx) - } - return os.RemoveAll(result.Path) -} - -func (w *Worker) retainSeed(task client.DownloadTask, result engine.Result, log *slog.Logger) bool { - if !w.cfg.SeedEnabled || result.Seed == nil || result.Seed.Cleanup == nil || result.Seed.Snapshot == nil { - return false - } - now := time.Now() - seed := retainedSeed{ - taskID: task.ID, - engine: result.Seed.Engine, - seedID: result.Seed.ID, - path: result.Seed.Path, - retainedAt: now, - snapshot: result.Seed.Snapshot, - cleanup: result.Seed.Cleanup, - } - if w.cfg.SeedDuration > 0 { - seed.expiresAt = now.Add(w.cfg.SeedDuration) - } - w.mu.Lock() - w.retainedSeeds = append(w.retainedSeeds, seed) - count := len(w.retainedSeeds) - w.mu.Unlock() - - log.Info("retaining completed bt task for seeding", - "engine", seed.engine, - "seed_id", seed.seedID, - "path", seed.path, - "expires_at", optionalTime(seed.expiresAt), - "retained_seeds", count, - ) - return true -} - -func (w *Worker) reportRetainedSeeds(ctx context.Context) { - for _, seed := range w.retainedSeedSnapshot() { - log := w.logger.With("task_id", seed.taskID, "engine", seed.engine, "seed_id", seed.seedID) - snapshot, err := seed.snapshot(ctx) - if err != nil { - log.Warn("failed to inspect retained bt seed", "error", err) - continue - } - if snapshot.Detail == nil { - continue - } - snapshot.Detail.Phase = "seeding" - zero := int64(0) - _, err = w.api.UpdateTask(ctx, seed.taskID, client.TaskPatch{ - DownloadedBytes: &snapshot.Downloaded, - TotalBytes: snapshot.Total, - DownloadBps: &snapshot.Bps, - StorageUploadBps: &zero, - Detail: snapshot.Detail, - }) - if err != nil { - log.Warn("failed to report retained bt seed", "error", err) - continue - } - log.Debug("reported retained bt seed", "downloaded_bytes", snapshot.Downloaded, "bps", snapshot.Bps) - } -} - -func (w *Worker) cleanupRetainedSeeds(ctx context.Context) { - seeds := w.retainedSeedSnapshot() - if len(seeds) == 0 { - return - } - - reasons := map[string]string{} - now := time.Now() - for _, seed := range seeds { - if !seed.expiresAt.IsZero() && !now.Before(seed.expiresAt) { - reasons[seed.taskID] = "expired" - } - } - - if w.cfg.SeedCacheLimit > 0 { - type seedSize struct { - seed retainedSeed - size int64 - } - sized := make([]seedSize, 0, len(seeds)) - var total int64 - for _, seed := range seeds { - if reasons[seed.taskID] != "" { - continue - } - size, err := directorySize(seed.path) - if err != nil { - w.logger.Warn("failed to inspect retained seed size", "task_id", seed.taskID, "path", seed.path, "error", err) - continue - } - total += size - sized = append(sized, seedSize{seed: seed, size: size}) - } - sort.Slice(sized, func(i, j int) bool { - return sized[i].seed.retainedAt.Before(sized[j].seed.retainedAt) - }) - for _, item := range sized { - if total <= w.cfg.SeedCacheLimit { - break - } - reasons[item.seed.taskID] = "cache_limit" - total -= item.size - } - } - - for _, seed := range seeds { - reason := reasons[seed.taskID] - if reason == "" { - continue - } - w.cleanupRetainedSeed(ctx, seed, reason) - } -} - -func (w *Worker) retainedSeedSnapshot() []retainedSeed { - w.mu.Lock() - defer w.mu.Unlock() - return append([]retainedSeed(nil), w.retainedSeeds...) -} - -func (w *Worker) cleanupRetainedSeed(ctx context.Context, seed retainedSeed, reason string) { - w.logger.Info("cleaning retained bt seed", - "task_id", seed.taskID, - "engine", seed.engine, - "seed_id", seed.seedID, - "path", seed.path, - "reason", reason, - ) - if err := seed.cleanup(ctx); err != nil { - w.logger.Warn("failed to clean retained bt seed", - "task_id", seed.taskID, - "engine", seed.engine, - "seed_id", seed.seedID, - "path", seed.path, - "error", err, - ) - return - } - w.mu.Lock() - next := w.retainedSeeds[:0] - for _, retained := range w.retainedSeeds { - if retained.taskID != seed.taskID { - next = append(next, retained) - } - } - w.retainedSeeds = next - w.mu.Unlock() -} - -type uploadProgress struct { - totalBytes int64 - uploaded int64 - lastAt time.Time - lastBytes int64 -} - -func (w *Worker) uploadResult( - ctx context.Context, - log *slog.Logger, - task client.DownloadTask, - result engine.Result, -) (string, error) { - if !result.IsDir { - progress := &uploadProgress{totalBytes: result.Size, lastAt: time.Now()} - return w.uploadSingleFile(ctx, log, task, result.Path, result.Name, result.Size, task.TargetFolder, progress) - } - - progress := &uploadProgress{totalBytes: result.Size, lastAt: time.Now()} - log.Info("creating remote folder", "name", result.Name, "size", result.Size, "target_folder", task.TargetFolder) - root, err := w.api.CreateFolder(ctx, task.UploadToken, result.Name, task.TargetFolder) - if err != nil { - return "", fmt.Errorf("create remote folder: %w", err) - } - rootPath := joinObjectPath(task.TargetFolder, root.Name) - entries, err := collectDirectoryEntries(result.Path) - if err != nil { - return "", err - } - for _, entry := range entries { - parent := joinObjectPath(rootPath, path.Dir(entry.relativePath)) - if entry.isDir { - log.Debug("creating remote subfolder", "name", entry.name, "parent", parent) - if _, err := w.api.CreateFolder(ctx, task.UploadToken, entry.name, parent); err != nil { - return "", fmt.Errorf("create remote subfolder %s: %w", entry.relativePath, err) - } - continue - } - if _, err := w.uploadSingleFile(ctx, log, task, entry.path, entry.name, entry.size, parent, progress); err != nil { - return "", err - } - } - return root.ID, nil -} - -func (w *Worker) uploadSingleFile( - ctx context.Context, - log *slog.Logger, - task client.DownloadTask, - path string, - name string, - size int64, - parent string, - progress *uploadProgress, -) (string, error) { - log.Info("creating remote object", "name", name, "size", size, "target_folder", parent) - draft, err := w.api.CreateObject(ctx, task.UploadToken, name, size, parent) - if err != nil { - return "", fmt.Errorf("create remote object: %w", err) - } - log.Info("uploading file to object storage", "object_id", draft.ID, "path", path) - if size > maxSingleUploadSize { - if err := w.uploadMultipartFile(ctx, log, task, draft.ID, path, size, progress); err != nil { - return "", fmt.Errorf("upload object %s: %w", draft.ID, err) - } - } else { - if err := uploadFile(ctx, draft.UploadURL, path, draft.ContentDisposition, func(written int64) error { - return w.reportUploadProgress(ctx, log, task, progress, written) - }); err != nil { - return "", fmt.Errorf("upload object %s: %w", draft.ID, err) - } - } - log.Info("confirming uploaded object", "object_id", draft.ID) - if err := w.api.ConfirmObject(ctx, task.UploadToken, draft.ID); err != nil { - return "", fmt.Errorf("confirm object %s: %w", draft.ID, err) - } - return draft.ID, nil -} - -func (w *Worker) uploadMultipartFile( - ctx context.Context, - log *slog.Logger, - task client.DownloadTask, - objectID string, - filePath string, - size int64, - progress *uploadProgress, -) error { - partSize := multipartPartSize(size) - log.Info("creating multipart upload session", "object_id", objectID, "part_size", partSize) - session, err := w.api.CreateObjectUploadSession(ctx, task.UploadToken, objectID, partSize) - if err != nil { - return fmt.Errorf("create multipart upload session: %w", err) - } - if session.PartSize <= 0 { - return fmt.Errorf("create multipart upload session: invalid part size %d", session.PartSize) - } - completed := false - defer func() { - if completed { - return - } - abortCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second) - defer cancel() - if abortErr := w.api.AbortObjectUploadSession(abortCtx, task.UploadToken, objectID, session.ID); abortErr != nil { - log.Warn("failed to abort multipart upload session", "object_id", objectID, "upload_session_id", session.ID, "error", abortErr) - } - }() - - file, err := os.Open(filePath) - if err != nil { - return err - } - defer file.Close() - - totalParts := int((size + session.PartSize - 1) / session.PartSize) - parts := make([]client.CompletedObjectUploadPart, 0, totalParts) - for firstPart := 1; firstPart <= totalParts; firstPart += presignMultipartPartsBatchSize { - lastPart := firstPart + presignMultipartPartsBatchSize - 1 - if lastPart > totalParts { - lastPart = totalParts - } - partNumbers := make([]int, 0, lastPart-firstPart+1) - for partNumber := firstPart; partNumber <= lastPart; partNumber++ { - partNumbers = append(partNumbers, partNumber) - } - presignedParts, err := w.api.PresignObjectUploadParts(ctx, task.UploadToken, objectID, session.ID, partNumbers) - if err != nil { - return fmt.Errorf("presign multipart upload parts: %w", err) - } - byNumber := make(map[int]string, len(presignedParts)) - for _, part := range presignedParts { - byNumber[part.PartNumber] = part.URL - } - for _, partNumber := range partNumbers { - url, ok := byNumber[partNumber] - if !ok { - return fmt.Errorf("presign multipart upload part %d: missing upload URL", partNumber) - } - offset := int64(partNumber-1) * session.PartSize - length := session.PartSize - if remaining := size - offset; remaining < length { - length = remaining - } - etag, err := uploadFilePart(ctx, url, file, offset, length, func(written int64) error { - return w.reportUploadProgress(ctx, log, task, progress, written) - }) - if err != nil { - return fmt.Errorf("upload part %d: %w", partNumber, err) - } - if etag == "" { - return fmt.Errorf("upload part %d: missing ETag", partNumber) - } - parts = append(parts, client.CompletedObjectUploadPart{PartNumber: partNumber, ETag: etag}) - } - } - if err := w.api.CompleteObjectUploadSession(ctx, task.UploadToken, objectID, session.ID, parts); err != nil { - return fmt.Errorf("complete multipart upload session: %w", err) - } - completed = true - return nil -} - -func (w *Worker) reportUploadProgress( - ctx context.Context, - log *slog.Logger, - task client.DownloadTask, - progress *uploadProgress, - written int64, -) error { - progress.uploaded += written - now := time.Now() - if progress.uploaded < progress.totalBytes && now.Sub(progress.lastAt) < time.Second { - return nil - } - elapsed := now.Sub(progress.lastAt).Seconds() - var bps int64 - if elapsed > 0 { - bps = int64(float64(progress.uploaded-progress.lastBytes) / elapsed) - } - detail := task.Detail - if detail == nil { - detail = &client.DownloadTaskDetail{} - } - detail.Phase = "uploading" - detail.ETASeconds = uploadETA(progress, bps) - detail.PeerUploadBps = nil - zero := int64(0) - _, err := w.api.UpdateTask(ctx, task.ID, client.TaskPatch{ - Status: "uploading", - StorageUploadedBytes: &progress.uploaded, - DownloadBps: &zero, - StorageUploadBps: &bps, - Detail: detail, - }) - if err != nil { - log.Error("failed to report upload progress", "uploaded_bytes", progress.uploaded, "total_bytes", progress.totalBytes, "bps", bps, "error", err) - return err - } - log.Debug("task upload progress", "uploaded_bytes", progress.uploaded, "total_bytes", progress.totalBytes, "bps", bps) - progress.lastAt = now - progress.lastBytes = progress.uploaded - return nil -} - func withDownloadETA(detail *client.DownloadTaskDetail, downloaded int64, total *int64, bps int64) *client.DownloadTaskDetail { eta := downloadETA(downloaded, total, bps) if eta == nil { @@ -804,7 +483,7 @@ func (w *Worker) cancelRunning(task client.DownloadTask) bool { func (w *Worker) ackStoppedControlTask(ctx context.Context, task client.DownloadTask) { log := w.taskLogger(task) if task.Status == "pausing" { - if _, err := w.api.UpdateTask(ctx, task.ID, client.TaskPatch{Status: "paused"}); err != nil { + if _, err := w.updateTask(ctx, task.ID, client.TaskPatch{Status: "paused"}); err != nil { log.Error("failed to acknowledge paused task without local process", "error", err) return } @@ -812,7 +491,7 @@ func (w *Worker) ackStoppedControlTask(ctx context.Context, task client.Download return } if task.Status == "canceling" { - if _, err := w.api.UpdateTask(ctx, task.ID, client.TaskPatch{Status: "canceled"}); err != nil { + if _, err := w.updateTask(ctx, task.ID, client.TaskPatch{Status: "canceled"}); err != nil { log.Error("failed to acknowledge canceled task without local process", "error", err) return } @@ -822,13 +501,19 @@ func (w *Worker) ackStoppedControlTask(ctx context.Context, task client.Download func (w *Worker) heartbeat() client.Heartbeat { hostname, _ := os.Hostname() + engineName := w.cfg.Engine + capabilities := []string{"http"} + if w.engine != nil { + engineName = w.engine.Name() + capabilities = w.engine.Capabilities() + } return client.Heartbeat{ Version: Version, Hostname: hostname, Platform: runtime.GOOS, Arch: runtime.GOARCH, - Engine: w.cfg.Engine, - Capabilities: capabilities(w.cfg.Engine), + Engine: engineName, + Capabilities: capabilities, MaxConcurrentTasks: w.cfg.MaxConcurrentTasks, CurrentTasks: w.currentTasks(), DownloadBps: 0, @@ -840,13 +525,13 @@ func (w *Worker) heartbeat() client.Heartbeat { func (w *Worker) resolveControlledTaskUpdate(ctx context.Context, taskID string, err error, log *slog.Logger) bool { message := err.Error() if strings.Contains(message, "Task is pausing") { - if _, updateErr := w.api.UpdateTask(ctx, taskID, client.TaskPatch{Status: "paused"}); updateErr != nil { + if _, updateErr := w.updateTask(ctx, taskID, client.TaskPatch{Status: "paused"}); updateErr != nil { log.Error("failed to mark task paused", "error", updateErr) } return true } if strings.Contains(message, "Task is canceling") { - if _, updateErr := w.api.UpdateTask(ctx, taskID, client.TaskPatch{Status: "canceled"}); updateErr != nil { + if _, updateErr := w.updateTask(ctx, taskID, client.TaskPatch{Status: "canceled"}); updateErr != nil { log.Error("failed to mark task canceled", "error", updateErr) } return true @@ -868,376 +553,6 @@ func (w *Worker) currentTasks() int { return len(w.running) } -type engineCandidate struct { - name string - binary []string - engine func(config.Config) engine.Engine - start func(context.Context, config.Config) (*exec.Cmd, error) - canAuto bool -} - -func (w *Worker) resolveEngine(ctx context.Context) error { - if w.cfg.Engine == "" || w.cfg.Engine == "auto" { - return w.resolveAutoEngine(ctx) - } - candidate, ok := engineCandidateByName(w.cfg.Engine) - if !ok { - return fmt.Errorf("unsupported downloader engine %q; expected auto, builtin, aria2, or qbittorrent", w.cfg.Engine) - } - w.logger.Info("checking configured downloader engine", "engine", candidate.name) - if w.checkCandidate(ctx, candidate) == nil { - w.useCandidate(candidate, "configured engine is already running") - return nil - } - if !candidate.canAuto { - w.useCandidate(candidate, "configured built-in engine") - return nil - } - if err := w.startCandidate(ctx, candidate); err != nil { - w.logger.Warn("configured downloader engine could not be started", "engine", candidate.name, "error", err) - w.engine = candidate.engine(w.cfg) - return nil - } - w.useCandidate(candidate, "configured engine started") - return nil -} - -func (w *Worker) resolveAutoEngine(ctx context.Context) error { - candidates := externalEngineCandidates() - w.logger.Info("auto selecting downloader engine", "priority", "aria2,qbittorrent,builtin") - for _, candidate := range candidates { - w.logger.Info("checking downloader engine availability", "engine", candidate.name) - if err := w.checkCandidate(ctx, candidate); err == nil { - w.useCandidate(candidate, "engine is already running") - return nil - } else { - w.logger.Info("downloader engine is not running", "engine", candidate.name, "error", err) - } - } - for _, candidate := range candidates { - w.logger.Info("checking downloader engine binary", "engine", candidate.name, "binaries", strings.Join(candidate.binary, ",")) - if err := w.startCandidate(ctx, candidate); err != nil { - w.logger.Info("downloader engine is not available for auto start", "engine", candidate.name, "error", err) - continue - } - w.useCandidate(candidate, "engine binary found and started") - return nil - } - w.cfg.Engine = "builtin" - w.engine = engine.HTTP{Dir: w.cfg.DownloadDir} - w.logger.Info("using built-in downloader engine", "reason", "no external downloader engine is running or installed") - return nil -} - -func (w *Worker) useCandidate(candidate engineCandidate, reason string) { - w.cfg.Engine = candidate.name - w.engine = candidate.engine(w.cfg) - w.logger.Info("selected downloader engine", "engine", candidate.name, "reason", reason) -} - -func (w *Worker) checkCandidate(ctx context.Context, candidate engineCandidate) error { - checkCtx, cancel := context.WithTimeout(ctx, 2*time.Second) - defer cancel() - return candidate.engine(w.cfg).Check(checkCtx) -} - -func (w *Worker) startCandidate(ctx context.Context, candidate engineCandidate) error { - if candidate.start == nil { - return fmt.Errorf("%s cannot be auto started", candidate.name) - } - w.logger.Info("starting downloader engine", "engine", candidate.name) - cmd, err := candidate.start(ctx, w.cfg) - if err != nil { - return err - } - if cmd.Process != nil { - w.logger.Info("downloader engine process started", "engine", candidate.name, "pid", cmd.Process.Pid) - } - w.started = append(w.started, cmd) - if err := waitForEngine(ctx, candidate.engine(w.cfg)); err != nil { - if cmd.Process != nil { - _ = cmd.Process.Kill() - } - return err - } - return nil -} - -func (w *Worker) stopStartedEngines() { - for _, cmd := range w.started { - if cmd.Process == nil { - continue - } - w.logger.Info("stopping auto-started downloader engine", "pid", cmd.Process.Pid) - _ = cmd.Process.Kill() - } -} - -func engineCandidateByName(name string) (engineCandidate, bool) { - candidates := append(externalEngineCandidates(), builtinEngineCandidate()) - for _, candidate := range candidates { - if candidate.name == name { - return candidate, true - } - } - return engineCandidate{}, false -} - -func externalEngineCandidates() []engineCandidate { - return []engineCandidate{ - { - name: "aria2", - binary: []string{"aria2c"}, - engine: func(cfg config.Config) engine.Engine { - return engine.Aria2{URL: cfg.Aria2URL, Secret: cfg.Aria2Secret, Dir: cfg.DownloadDir, RetainSeed: cfg.SeedEnabled} - }, - start: startAria2, - canAuto: true, - }, - { - name: "qbittorrent", - binary: []string{"qbittorrent-nox", "qbittorrent"}, - engine: func(cfg config.Config) engine.Engine { - return engine.QBittorrent{URL: cfg.QBittorrentURL, Username: cfg.QBittorrentUser, Password: cfg.QBittorrentPass, Dir: cfg.DownloadDir, RetainSeed: cfg.SeedEnabled} - }, - start: startQBittorrent, - canAuto: true, - }, - } -} - -func builtinEngineCandidate() engineCandidate { - return engineCandidate{ - name: "builtin", - engine: func(cfg config.Config) engine.Engine { return engine.HTTP{Dir: cfg.DownloadDir} }, - } -} - -func selectEngine(cfg config.Config) engine.Engine { - switch cfg.Engine { - case "aria2": - return engine.Aria2{URL: cfg.Aria2URL, Secret: cfg.Aria2Secret, Dir: cfg.DownloadDir, RetainSeed: cfg.SeedEnabled} - case "qbittorrent": - return engine.QBittorrent{ - URL: cfg.QBittorrentURL, - Username: cfg.QBittorrentUser, - Password: cfg.QBittorrentPass, - Dir: cfg.DownloadDir, - RetainSeed: cfg.SeedEnabled, - } - default: - return engine.HTTP{Dir: cfg.DownloadDir} - } -} - -func waitForEngine(ctx context.Context, downloader engine.Engine) error { - deadline := time.Now().Add(8 * time.Second) - var lastErr error - for time.Now().Before(deadline) { - checkCtx, cancel := context.WithTimeout(ctx, time.Second) - err := downloader.Check(checkCtx) - cancel() - if err == nil { - return nil - } - lastErr = err - select { - case <-ctx.Done(): - return ctx.Err() - case <-time.After(500 * time.Millisecond): - } - } - return lastErr -} - -func startAria2(ctx context.Context, cfg config.Config) (*exec.Cmd, error) { - path, err := exec.LookPath("aria2c") - if err != nil { - return nil, err - } - rpcURL, err := parseLocalEngineURL(cfg.Aria2URL, "6800") - if err != nil { - return nil, err - } - args := []string{ - "--enable-rpc=true", - "--rpc-listen-all=false", - "--rpc-listen-port=" + rpcURL.port, - "--dir=" + cfg.DownloadDir, - "--continue=true", - "--allow-overwrite=true", - "--auto-file-renaming=false", - } - if cfg.Aria2Secret != "" { - args = append(args, "--rpc-secret="+cfg.Aria2Secret) - } - cmd := exec.CommandContext(ctx, path, args...) - if err := cmd.Start(); err != nil { - return nil, err - } - go func() { _ = cmd.Wait() }() - return cmd, nil -} - -func startQBittorrent(ctx context.Context, cfg config.Config) (*exec.Cmd, error) { - path, err := lookPathAny("qbittorrent-nox", "qbittorrent") - if err != nil { - return nil, err - } - webURL, err := parseLocalEngineURL(cfg.QBittorrentURL, "8080") - if err != nil { - return nil, err - } - args := []string{} - if strings.Contains(filepathBase(path), "qbittorrent-nox") { - args = append(args, "--webui-port="+webURL.port) - } - cmd := exec.CommandContext(ctx, path, args...) - if err := cmd.Start(); err != nil { - return nil, err - } - go func() { _ = cmd.Wait() }() - return cmd, nil -} - -func lookPathAny(names ...string) (string, error) { - var lastErr error - for _, name := range names { - path, err := exec.LookPath(name) - if err == nil { - return path, nil - } - lastErr = err - } - return "", lastErr -} - -type localEngineURL struct { - port string -} - -func parseLocalEngineURL(raw string, defaultPort string) (localEngineURL, error) { - normalized := raw - if strings.HasPrefix(normalized, "ws://") { - normalized = "http://" + strings.TrimPrefix(normalized, "ws://") - } - if strings.HasPrefix(normalized, "wss://") { - normalized = "https://" + strings.TrimPrefix(normalized, "wss://") - } - parsed, err := url.Parse(normalized) - if err != nil { - return localEngineURL{}, err - } - host := parsed.Hostname() - if host != "" && host != "127.0.0.1" && host != "localhost" && host != "::1" { - return localEngineURL{}, fmt.Errorf("auto start only supports local engine URLs, got %s", host) - } - port := parsed.Port() - if port == "" { - port = defaultPort - } - if _, err := strconv.Atoi(port); err != nil { - return localEngineURL{}, fmt.Errorf("invalid engine port %q", port) - } - return localEngineURL{port: port}, nil -} - -func filepathBase(path string) string { - parts := strings.FieldsFunc(path, func(r rune) bool { return r == '/' || r == '\\' }) - if len(parts) == 0 { - return path - } - return parts[len(parts)-1] -} - -func capabilities(name string) []string { - switch name { - case "aria2", "qbittorrent": - return []string{"http", "magnet", "torrent"} - default: - return []string{"http"} - } -} - -func uploadFile(ctx context.Context, url, path string, contentDisposition string, progress func(written int64) error) error { - file, err := os.Open(path) - if err != nil { - return err - } - defer file.Close() - stat, err := file.Stat() - if err != nil { - return err - } - reader := io.Reader(file) - if progress != nil { - reader = &uploadProgressReader{reader: file, progress: progress} - } - req, err := http.NewRequestWithContext(ctx, http.MethodPut, url, reader) - if err != nil { - return err - } - req.Header.Set("Content-Type", "application/octet-stream") - if contentDisposition != "" { - req.Header.Set("Content-Disposition", contentDisposition) - } - req.ContentLength = stat.Size() - res, err := http.DefaultClient.Do(req) - if err != nil { - return err - } - defer res.Body.Close() - if res.StatusCode < 200 || res.StatusCode >= 300 { - body, _ := io.ReadAll(io.LimitReader(res.Body, 4096)) - if len(body) > 0 { - return fmt.Errorf("upload failed: %s: %s", res.Status, strings.TrimSpace(string(body))) - } - return fmt.Errorf("upload failed: %s", res.Status) - } - return nil -} - -func uploadFilePart(ctx context.Context, url string, file *os.File, offset int64, length int64, progress func(written int64) error) (string, error) { - reader := io.NewSectionReader(file, offset, length) - var body io.Reader = reader - if progress != nil { - body = &uploadProgressReader{reader: reader, progress: progress} - } - req, err := http.NewRequestWithContext(ctx, http.MethodPut, url, body) - if err != nil { - return "", err - } - req.ContentLength = length - res, err := http.DefaultClient.Do(req) - if err != nil { - return "", err - } - defer res.Body.Close() - if res.StatusCode < 200 || res.StatusCode >= 300 { - body, _ := io.ReadAll(io.LimitReader(res.Body, 4096)) - if len(body) > 0 { - return "", fmt.Errorf("upload failed: %s: %s", res.Status, strings.TrimSpace(string(body))) - } - return "", fmt.Errorf("upload failed: %s", res.Status) - } - return res.Header.Get("ETag"), nil -} - -type uploadProgressReader struct { - reader io.Reader - progress func(written int64) error -} - -func (r *uploadProgressReader) Read(p []byte) (int, error) { - n, err := r.reader.Read(p) - if n > 0 { - if progressErr := r.progress(int64(n)); progressErr != nil { - return n, progressErr - } - } - return n, err -} - func taskErrorMessage(err error) string { msg := err.Error() if len(msg) <= maxTaskErrorMessageLength { diff --git a/downloader/internal/worker/worker_test.go b/downloader/internal/worker/worker_test.go index 1e9248b3..58b453f5 100644 --- a/downloader/internal/worker/worker_test.go +++ b/downloader/internal/worker/worker_test.go @@ -18,7 +18,7 @@ import ( ) func TestResolveEngineRejectsUnknownConfiguredEngine(t *testing.T) { - w := New(config.Config{Engine: "bad-engine"}) + w := NewWithAPI(config.Config{Engine: "bad-engine"}, nil) err := w.resolveEngine(context.Background()) if err == nil { @@ -215,7 +215,7 @@ func TestTaskErrorMessageTruncatesToSchemaLimit(t *testing.T) { func TestRetainSeedKeepsDownloadedResult(t *testing.T) { dir := t.TempDir() cleaned := false - w := New(config.Config{SeedEnabled: true, SeedDuration: time.Hour}) + w := NewWithAPI(config.Config{SeedEnabled: true, SeedDuration: time.Hour}, nil) retained := w.retainSeed( clientTask("task-1"), @@ -250,7 +250,7 @@ func TestRetainSeedKeepsDownloadedResult(t *testing.T) { func TestCleanupRetainedSeedsRemovesExpiredSeed(t *testing.T) { dir := t.TempDir() - w := New(config.Config{SeedEnabled: true, SeedDuration: time.Hour}) + w := NewWithAPI(config.Config{SeedEnabled: true, SeedDuration: time.Hour}, nil) cleaned := false w.retainedSeeds = []retainedSeed{{ taskID: "task-1", @@ -295,7 +295,7 @@ func TestCleanupRetainedSeedsRemovesOldestWhenCacheLimitExceeded(t *testing.T) { } var cleaned []string - w := New(config.Config{SeedEnabled: true, SeedCacheLimit: 3}) + w := NewWithAPI(config.Config{SeedEnabled: true, SeedCacheLimit: 3}, nil) w.retainedSeeds = []retainedSeed{ { taskID: "old", diff --git a/downloader/oapi-codegen.yaml b/downloader/oapi-codegen.yaml new file mode 100644 index 00000000..b72312c4 --- /dev/null +++ b/downloader/oapi-codegen.yaml @@ -0,0 +1,5 @@ +package: openapi +generate: + models: true + client: true +output: internal/openapi/client.gen.go diff --git a/package.json b/package.json index 4853ab8c..961b704d 100644 --- a/package.json +++ b/package.json @@ -36,7 +36,10 @@ "e2e:cloud:cf": "node scripts/run-cloud-e2e.mjs --runtime cf", "e2e:archive": "node scripts/run-cloud-e2e.mjs --local --with-s3-mock --spec archive.spec.ts", "e2e:archive:cf": "node scripts/run-cloud-e2e.mjs --runtime cf --local --with-s3-mock --spec archive.spec.ts", - "openapi:downloader": "tsx scripts/generate-downloader-openapi.ts" + "openapi:downloader": "tsx scripts/generate-downloader-openapi.ts", + "openapi:downloader:go": "pnpm openapi:downloader && cd downloader && go run github.com/oapi-codegen/oapi-codegen/v2/cmd/oapi-codegen@v2.7.0 -config oapi-codegen.yaml ../docs/openapi/downloader.json", + "openapi:downloader:check": "tsx scripts/check-downloader-openapi.ts", + "openapi:downloader:all": "pnpm openapi:downloader:go" }, "engines": { "node": ">=24" diff --git a/scripts/check-downloader-openapi.ts b/scripts/check-downloader-openapi.ts new file mode 100644 index 00000000..93e33d26 --- /dev/null +++ b/scripts/check-downloader-openapi.ts @@ -0,0 +1,77 @@ +import { execFile as execFileCallback } from 'node:child_process' +import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { promisify } from 'node:util' +import { downloaderOpenAPIDocument } from '../server/openapi/downloader' + +const execFile = promisify(execFileCallback) +const root = process.cwd() + +async function main() { + const tempDir = await mkdtemp(join(tmpdir(), 'zpan-downloader-openapi-')) + try { + const generatedDocPath = join(tempDir, 'downloader.json') + const generatedClientPath = join(tempDir, 'client.gen.go') + const configPath = join(tempDir, 'oapi-codegen.yaml') + + await mkdir(join(root, 'docs/openapi'), { recursive: true }) + await writeFile( + generatedDocPath, + `${JSON.stringify(downloaderOpenAPIDocument(), null, 2)}\n`, + 'utf8', + ) + await writeFile( + configPath, + [ + 'package: openapi', + 'generate:', + ' models: true', + ' client: true', + `output: ${JSON.stringify(generatedClientPath)}`, + '', + ].join('\n'), + 'utf8', + ) + + await execFile( + 'go', + [ + 'run', + 'github.com/oapi-codegen/oapi-codegen/v2/cmd/oapi-codegen@v2.7.0', + '-config', + configPath, + generatedDocPath, + ], + { cwd: root }, + ) + + await assertSame( + 'docs/openapi/downloader.json', + generatedDocPath, + 'Downloader OpenAPI document is stale.', + ) + await assertSame( + 'downloader/internal/openapi/client.gen.go', + generatedClientPath, + 'Downloader Go OpenAPI client is stale.', + ) + } finally { + await rm(tempDir, { force: true, recursive: true }) + } +} + +async function assertSame(path: string, generatedPath: string, message: string) { + const actual = await readFile(join(root, path), 'utf8') + const generated = await readFile(generatedPath, 'utf8') + if (actual !== generated) { + console.error(message) + console.error(`Run: pnpm openapi:downloader:go`) + process.exitCode = 1 + } +} + +main().catch((err) => { + console.error(err) + process.exit(1) +}) diff --git a/server/openapi/downloader.ts b/server/openapi/downloader.ts index 7c6c457f..5c8d0b4d 100644 --- a/server/openapi/downloader.ts +++ b/server/openapi/downloader.ts @@ -1,5 +1,14 @@ import { createRoute, OpenAPIHono, z } from '@hono/zod-openapi' -import { createMatterSchema } from '@shared/schemas' +import { + confirmMatterSchema, + createMatterSchema, + createObjectUploadSessionSchema, + objectDraftSchema, + objectUploadSessionSchema, + patchObjectUploadSessionSchema, + presignObjectUploadPartsResponseSchema, + presignObjectUploadPartsSchema, +} from '@shared/schemas' import downloadTasks from '../routes/download-tasks' import downloaders, { downloaderSelfRoute } from '../routes/downloaders' @@ -40,84 +49,6 @@ const deviceTokenSchema = z }) .openapi('DeviceToken') -const objectDraftSchema = z - .object({ - id: z.string(), - name: z.string(), - uploadUrl: z.string().optional(), - contentDisposition: z.string().optional(), - }) - .openapi('ObjectDraft') - -const confirmObjectRequestSchema = z - .object({ - action: z.enum(['confirm']), - onConflict: z.enum(['fail', 'rename']).optional(), - }) - .openapi('ConfirmObjectRequest') - -const createObjectUploadSessionRequestSchema = z - .object({ - partSize: z - .number() - .int() - .min(5 * 1024 * 1024) - .max(512 * 1024 * 1024) - .optional(), - }) - .openapi('CreateObjectUploadSessionRequest') - -const objectUploadSessionSchema = z - .object({ - id: z.string(), - objectId: z.string(), - uploadId: z.string(), - partSize: z.number().int(), - status: z.enum(['active', 'completed', 'aborted']), - expiresAt: z.string(), - createdAt: z.string(), - updatedAt: z.string(), - }) - .openapi('ObjectUploadSession') - -const presignObjectUploadPartsRequestSchema = z - .object({ - partNumbers: z.array(z.number().int().min(1).max(10_000)).min(1).max(100), - }) - .openapi('PresignObjectUploadPartsRequest') - -const presignedObjectUploadPartSchema = z - .object({ - partNumber: z.number().int(), - url: z.string(), - }) - .openapi('PresignedObjectUploadPart') - -const presignObjectUploadPartsResponseSchema = z - .object({ - uploadId: z.string(), - partSize: z.number().int(), - parts: z.array(presignedObjectUploadPartSchema), - }) - .openapi('PresignObjectUploadPartsResponse') - -const completeObjectUploadSessionRequestSchema = z - .object({ - action: z.literal('complete'), - parts: z.array(z.object({ partNumber: z.number().int(), etag: z.string() })).min(1), - }) - .openapi('CompleteObjectUploadSessionRequest') - -const abortObjectUploadSessionRequestSchema = z - .object({ - action: z.literal('abort'), - }) - .openapi('AbortObjectUploadSessionRequest') - -const patchObjectUploadSessionRequestSchema = z - .union([completeObjectUploadSessionRequestSchema, abortObjectUploadSessionRequestSchema]) - .openapi('PatchObjectUploadSessionRequest') - function jsonResponse(schema: z.ZodType, description: string) { return { content: { @@ -194,7 +125,7 @@ function mountObjectUploadRoutes(app: OpenAPIHono) { request: { params: z.object({ id: z.string() }), body: { - content: { 'application/json': { schema: confirmObjectRequestSchema } }, + content: { 'application/json': { schema: confirmMatterSchema } }, required: true, }, }, @@ -214,7 +145,7 @@ function mountObjectUploadRoutes(app: OpenAPIHono) { request: { params: z.object({ id: z.string() }), body: { - content: { 'application/json': { schema: createObjectUploadSessionRequestSchema } }, + content: { 'application/json': { schema: createObjectUploadSessionSchema } }, required: true, }, }, @@ -235,7 +166,7 @@ function mountObjectUploadRoutes(app: OpenAPIHono) { request: { params: z.object({ id: z.string(), uploadSessionId: z.string() }), body: { - content: { 'application/json': { schema: presignObjectUploadPartsRequestSchema } }, + content: { 'application/json': { schema: presignObjectUploadPartsSchema } }, required: true, }, }, @@ -256,7 +187,7 @@ function mountObjectUploadRoutes(app: OpenAPIHono) { request: { params: z.object({ id: z.string(), uploadSessionId: z.string() }), body: { - content: { 'application/json': { schema: patchObjectUploadSessionRequestSchema } }, + content: { 'application/json': { schema: patchObjectUploadSessionSchema } }, required: true, }, }, diff --git a/server/routes/download-tasks.integration.test.ts b/server/routes/download-tasks.integration.test.ts index cc376845..ce689b53 100644 --- a/server/routes/download-tasks.integration.test.ts +++ b/server/routes/download-tasks.integration.test.ts @@ -235,6 +235,18 @@ describe('Download tasks API integration', () => { expect(runningTask.detail.infoHash).toBe('abc123') expect(runningTask.detail.trackers[0].url).toBe('udp://tracker.example/announce') + const recoverRunningRes = await app.request('/api/download-tasks?assignedTo=me&status=running', { + headers: { Authorization: `Bearer ${createdDownloader.token}` }, + }) + expect(recoverRunningRes.status).toBe(200) + const recoverRunning = (await recoverRunningRes.json()) as { + items: Array<{ id: string; uploadToken?: string; status: string }> + } + const recoverRunningTask = recoverRunning.items.find((item) => item.id === createdTask.id) + expect(recoverRunningTask?.status).toBe('running') + expect(recoverRunningTask?.uploadToken).toBeTruthy() + uploadHeaders.Authorization = `Bearer ${recoverRunningTask?.uploadToken}` + const createFolderRes = await app.request('/api/objects', { method: 'POST', headers: uploadHeaders, @@ -335,6 +347,23 @@ describe('Download tasks API integration', () => { const confirmed = (await confirmRes.json()) as { id: string; status: string } expect(confirmed.status).toBe('active') + const uploadingRes = await app.request(`/api/download-tasks/${createdTask.id}`, { + method: 'PATCH', + headers: downloaderHeaders, + body: JSON.stringify({ status: 'uploading', downloadedBytes: 10 * 1024 * 1024 }), + }) + expect(uploadingRes.status).toBe(200) + const recoverUploadingRes = await app.request('/api/download-tasks?assignedTo=me&status=uploading', { + headers: { Authorization: `Bearer ${createdDownloader.token}` }, + }) + expect(recoverUploadingRes.status).toBe(200) + const recoverUploading = (await recoverUploadingRes.json()) as { + items: Array<{ id: string; uploadToken?: string; status: string }> + } + const recoverUploadingTask = recoverUploading.items.find((item) => item.id === createdTask.id) + expect(recoverUploadingTask?.status).toBe('uploading') + expect(recoverUploadingTask?.uploadToken).toBeTruthy() + const completeTaskRes = await app.request(`/api/download-tasks/${createdTask.id}`, { method: 'PATCH', headers: downloaderHeaders, diff --git a/server/routes/download-tasks.ts b/server/routes/download-tasks.ts index ab8e1e29..de0ce816 100644 --- a/server/routes/download-tasks.ts +++ b/server/routes/download-tasks.ts @@ -2,7 +2,8 @@ import { createRoute, OpenAPIHono, z } from '@hono/zod-openapi' import { createDownloadTaskSchema, downloadTaskActionInputSchema, - downloadTaskDetailSchema, + downloadTaskPageSchema, + downloadTaskSchema, listDownloadTasksQuerySchema, updateDownloadTaskSchema, } from '@shared/schemas' @@ -19,13 +20,6 @@ import { } from '../services/downloads' const errorSchema = z.object({ error: z.string() }) -const int64Schema = () => z.number().int().openapi({ type: 'integer', format: 'int64' }) -const nullableInt64Schema = () => - z - .number() - .int() - .nullable() - .openapi({ type: 'integer', format: 'int64', nullable: true } as never) type OpenAPIContext = Context & { req: Context['req'] & { @@ -34,46 +28,6 @@ type OpenAPIContext = Context & { } } -const downloadTaskSchema = z.object({ - id: z.string(), - sourceType: z.enum(['http', 'magnet', 'torrent_url']), - sourceUri: z.string(), - name: z.string(), - targetFolder: z.string(), - category: z.string().nullable(), - tags: z.array(z.string()), - status: z.enum([ - 'queued', - 'assigned', - 'running', - 'billing_paused', - 'pausing', - 'paused', - 'uploading', - 'canceling', - 'completed', - 'failed', - 'canceled', - ]), - downloadedBytes: int64Schema(), - storageUploadedBytes: int64Schema(), - totalBytes: nullableInt64Schema(), - downloadBps: int64Schema(), - storageUploadBps: int64Schema(), - errorMessage: z.string().nullable().optional(), - resultObjectId: z.string().nullable().optional(), - detail: downloadTaskDetailSchema.nullable().optional(), - uploadToken: z.string().optional(), - assignedDownloaderId: z.string().nullable().optional(), -}) - -const downloadTaskPageSchema = z.object({ - items: z.array(downloadTaskSchema), - total: z.number().int(), - page: z.number().int(), - pageSize: z.number().int(), -}) - const sseEncoder = new TextEncoder() const downloadTaskEventIntervalMs = 2000 diff --git a/server/routes/downloaders.ts b/server/routes/downloaders.ts index 99c44a80..2997d9c0 100644 --- a/server/routes/downloaders.ts +++ b/server/routes/downloaders.ts @@ -1,5 +1,13 @@ import { createRoute, OpenAPIHono, z } from '@hono/zod-openapi' -import { createDownloaderSchema, downloaderHeartbeatSchema, updateDownloaderSchema } from '@shared/schemas' +import { + createDownloaderResponseSchema, + createDownloaderSchema, + deleteDownloaderResponseSchema, + downloaderHeartbeatSchema, + downloaderListSchema, + downloaderSchema, + updateDownloaderSchema, +} from '@shared/schemas' import type { Context } from 'hono' import { requireAdmin, requireDownloader } from '../middleware/auth' import type { Env } from '../middleware/platform' @@ -13,7 +21,6 @@ import { } from '../services/downloads' const errorSchema = z.object({ error: z.string() }) -const int64Schema = () => z.number().int().openapi({ type: 'integer', format: 'int64' }) type OpenAPIContext = Context & { req: Context['req'] & { @@ -22,43 +29,6 @@ type OpenAPIContext = Context & { } } -const downloaderHeartbeatResponseSchema = z.object({ - version: z.string(), - hostname: z.string(), - platform: z.string(), - arch: z.string(), - engine: z.enum(['builtin', 'aria2', 'qbittorrent']), - capabilities: z.array(z.string()), - maxConcurrentTasks: z.number().int(), - currentTasks: z.number().int(), - downloadBps: int64Schema(), - uploadBps: int64Schema(), - freeDiskBytes: int64Schema(), -}) - -const downloaderSchema = z.object({ - id: z.string(), - name: z.string().optional(), - status: z.enum(['online', 'offline', 'disabled']).optional(), - enabled: z.boolean().optional(), - heartbeat: downloaderHeartbeatResponseSchema.optional(), -}) - -const downloaderListSchema = z.object({ - items: z.array(downloaderSchema), - total: z.number().int(), -}) - -const createDownloaderResponseSchema = z.object({ - downloader: downloaderSchema, - token: z.string(), -}) - -const deleteDownloaderResponseSchema = z.object({ - id: z.string(), - deleted: z.boolean(), -}) - function jsonResponse(schema: z.ZodType, description: string) { return { content: { 'application/json': { schema } }, description } } diff --git a/server/services/downloads.ts b/server/services/downloads.ts index 92ea53be..7cea96b4 100644 --- a/server/services/downloads.ts +++ b/server/services/downloads.ts @@ -1,750 +1,2 @@ -import type { - CreateDownloaderInput, - CreateDownloadTaskInput, - DownloaderHeartbeatInput, - DownloadTaskActionInput, - UpdateDownloaderInput, - UpdateDownloadTaskInput, -} from '@shared/schemas' -import type { Downloader, DownloadTask } from '@shared/types' -import { and, asc, count, desc, eq, inArray, like, sql } from 'drizzle-orm' -import { nanoid } from 'nanoid' -import { downloaders, downloadTasks } from '../db/schema' -import type { Platform } from '../platform/interface' -import { hashDownloadToken, signDownloadToken } from './download-tokens' -import { RemoteDownloadBillingBlockedError, reportRemoteDownloadUnit } from './remote-download-usage' - -const DEFAULT_REMOTE_DOWNLOAD_UNIT_BYTES = 100 * 1024 * 1024 -const UPLOAD_TOKEN_TTL_SECONDS = 24 * 60 * 60 -const PAUSABLE_TASK_STATUSES = ['queued', 'assigned', 'running'] as const -const CANCELABLE_TASK_STATUSES = [ - 'queued', - 'assigned', - 'running', - 'billing_paused', - 'paused', - 'uploading', - 'pausing', -] as const -const TERMINAL_TASK_STATUSES = ['completed', 'failed', 'canceled'] as const - -export class DownloadError extends Error { - constructor( - readonly code: - | 'not_found' - | 'forbidden' - | 'no_downloader' - | 'invalid_state' - | 'billing_paused' - | 'unsupported_source', - message: string = code, - ) { - super(message) - this.name = 'DownloadError' - } -} - -type DownloaderRow = typeof downloaders.$inferSelect -type DownloadTaskRow = typeof downloadTasks.$inferSelect - -export async function createDownloader( - platform: Platform, - input: CreateDownloaderInput, - userId: string, -): Promise<{ downloader: Downloader; token: string }> { - const now = new Date() - const id = nanoid() - const jti = nanoid() - const token = await signDownloadToken(platform, { - v: 1, - typ: 'downloader', - downloaderId: id, - jti, - iat: Math.floor(now.getTime() / 1000), - }) - await platform.db.insert(downloaders).values({ - id, - name: input.name, - tokenHash: await hashDownloadToken(platform, token), - tokenJti: jti, - status: 'offline', - enabled: true, - version: input.heartbeat.version, - hostname: input.heartbeat.hostname, - platform: input.heartbeat.platform, - arch: input.heartbeat.arch, - engine: input.heartbeat.engine, - capabilities: JSON.stringify(input.heartbeat.capabilities), - maxConcurrentTasks: input.heartbeat.maxConcurrentTasks, - currentTasks: input.heartbeat.currentTasks, - downloadBps: input.heartbeat.downloadBps, - uploadBps: input.heartbeat.uploadBps, - freeDiskBytes: input.heartbeat.freeDiskBytes, - remoteDownloadCreditBillingEnabled: false, - remoteDownloadCreditUnitBytes: DEFAULT_REMOTE_DOWNLOAD_UNIT_BYTES, - remoteDownloadCreditPerUnit: 1, - lastHeartbeatAt: null, - createdBy: userId, - createdAt: now, - updatedAt: now, - }) - return { downloader: await getDownloader(platform, id), token } -} - -export async function listDownloaders(platform: Platform): Promise { - const rows = await platform.db.select().from(downloaders).orderBy(desc(downloaders.createdAt)) - return rows.map(toDownloader) -} - -export async function getDownloader(platform: Platform, id: string): Promise { - const rows = await platform.db.select().from(downloaders).where(eq(downloaders.id, id)).limit(1) - if (!rows[0]) throw new DownloadError('not_found') - return toDownloader(rows[0]) -} - -export async function updateDownloader( - platform: Platform, - id: string, - input: UpdateDownloaderInput, -): Promise { - const rows = await platform.db.select({ id: downloaders.id }).from(downloaders).where(eq(downloaders.id, id)).limit(1) - if (!rows[0]) throw new DownloadError('not_found') - const now = new Date() - await platform.db - .update(downloaders) - .set({ - ...(input.name !== undefined ? { name: input.name } : {}), - ...(input.enabled !== undefined - ? { enabled: input.enabled, status: input.enabled ? 'offline' : 'disabled' } - : {}), - ...(input.remoteDownloadCreditBillingEnabled !== undefined - ? { remoteDownloadCreditBillingEnabled: input.remoteDownloadCreditBillingEnabled } - : {}), - ...(input.remoteDownloadCreditUnitBytes !== undefined - ? { remoteDownloadCreditUnitBytes: input.remoteDownloadCreditUnitBytes } - : {}), - ...(input.remoteDownloadCreditPerUnit !== undefined - ? { remoteDownloadCreditPerUnit: input.remoteDownloadCreditPerUnit } - : {}), - updatedAt: now, - }) - .where(eq(downloaders.id, id)) - return getDownloader(platform, id) -} - -export async function deleteDownloader(platform: Platform, id: string): Promise<{ id: string; deleted: true }> { - const rows = await platform.db.select({ id: downloaders.id }).from(downloaders).where(eq(downloaders.id, id)).limit(1) - if (!rows[0]) throw new DownloadError('not_found') - const now = new Date() - - await platform.db - .update(downloadTasks) - .set({ - status: 'queued', - assignedDownloaderId: null, - uploadTokenHash: null, - uploadTokenJti: null, - uploadTokenExpiresAt: null, - assignedAt: null, - updatedAt: now, - }) - .where( - and( - eq(downloadTasks.assignedDownloaderId, id), - inArray(downloadTasks.status, [ - 'queued', - 'assigned', - 'running', - 'billing_paused', - 'pausing', - 'paused', - 'uploading', - 'canceling', - ]), - ), - ) - await platform.db.delete(downloaders).where(eq(downloaders.id, id)) - return { id, deleted: true } -} - -export async function recordDownloaderHeartbeat( - platform: Platform, - downloaderId: string, - heartbeat: DownloaderHeartbeatInput, -): Promise { - const rows = await platform.db - .select({ id: downloaders.id, enabled: downloaders.enabled }) - .from(downloaders) - .where(eq(downloaders.id, downloaderId)) - .limit(1) - if (!rows[0]) throw new DownloadError('not_found') - const now = new Date() - await platform.db - .update(downloaders) - .set({ - status: rows[0].enabled ? 'online' : 'disabled', - version: heartbeat.version, - hostname: heartbeat.hostname, - platform: heartbeat.platform, - arch: heartbeat.arch, - engine: heartbeat.engine, - capabilities: JSON.stringify(heartbeat.capabilities), - maxConcurrentTasks: heartbeat.maxConcurrentTasks, - currentTasks: heartbeat.currentTasks, - downloadBps: heartbeat.downloadBps, - uploadBps: heartbeat.uploadBps, - freeDiskBytes: heartbeat.freeDiskBytes, - lastHeartbeatAt: now, - updatedAt: now, - }) - .where(eq(downloaders.id, downloaderId)) - await assignQueuedTasks(platform) - return getDownloader(platform, downloaderId) -} - -export async function createDownloadTask( - platform: Platform, - orgId: string, - userId: string, - input: CreateDownloadTaskInput, -): Promise { - const now = new Date() - const id = nanoid() - const assigned = await selectDownloader(platform, input.source.type) - const uploadToken = assigned - ? await createTaskUploadToken(platform, { - taskId: id, - downloaderId: assigned.id, - orgId, - targetFolder: input.targetFolder, - createdByUserId: userId, - }) - : null - await platform.db.insert(downloadTasks).values({ - id, - orgId, - createdByUserId: userId, - sourceType: input.source.type, - sourceUri: input.source.uri, - name: input.name ?? null, - targetFolder: input.targetFolder, - category: input.category ?? null, - tags: JSON.stringify(input.tags ?? []), - assignedDownloaderId: assigned?.id ?? null, - status: assigned ? 'assigned' : 'queued', - uploadTokenHash: uploadToken?.hash ?? null, - uploadTokenJti: uploadToken?.jti ?? null, - uploadTokenExpiresAt: uploadToken?.expiresAt ?? null, - createdAt: now, - updatedAt: now, - assignedAt: assigned ? now : null, - }) - return getDownloadTask(platform, orgId, id) -} - -export async function listDownloadTasks( - platform: Platform, - opts: { - orgId?: string - downloaderId?: string - status?: string - category?: string - tag?: string - sortBy?: 'createdAt' | 'source' | 'category' | 'tags' | 'status' | 'progress' | 'eta' - sortDir?: 'asc' | 'desc' - page: number - pageSize: number - includeUploadToken?: boolean - }, -): Promise<{ items: DownloadTask[]; total: number }> { - const offset = (opts.page - 1) * opts.pageSize - const filters = [] - if (opts.orgId) filters.push(eq(downloadTasks.orgId, opts.orgId)) - if (opts.downloaderId) filters.push(eq(downloadTasks.assignedDownloaderId, opts.downloaderId)) - if (opts.status) filters.push(eq(downloadTasks.status, opts.status)) - if (opts.category) filters.push(eq(downloadTasks.category, opts.category)) - if (opts.tag) filters.push(like(downloadTasks.tags, `%${JSON.stringify(opts.tag)}%`)) - const where = filters.length ? and(...filters) : undefined - const [rows, totalRows] = await Promise.all([ - platform.db - .select() - .from(downloadTasks) - .where(where) - .orderBy(downloadTaskOrderBy(opts.sortBy ?? 'createdAt', opts.sortDir ?? 'desc')) - .limit(opts.pageSize) - .offset(offset), - platform.db.select({ count: count() }).from(downloadTasks).where(where), - ]) - return { - items: await Promise.all( - rows.map((row) => - toDownloadTaskWithToken( - platform, - row, - (opts.includeUploadToken ?? false) && (row.status === 'assigned' || row.status === 'billing_paused'), - ), - ), - ), - total: totalRows[0]?.count ?? 0, - } -} - -function downloadTaskOrderBy( - sortBy: 'createdAt' | 'source' | 'category' | 'tags' | 'status' | 'progress' | 'eta', - sortDir: 'asc' | 'desc', -) { - const direction = sortDir === 'asc' ? asc : desc - if (sortBy === 'source') return direction(downloadTasks.sourceUri) - if (sortBy === 'category') return direction(downloadTasks.category) - if (sortBy === 'tags') return direction(downloadTasks.tags) - if (sortBy === 'status') return direction(downloadTasks.status) - if (sortBy === 'progress') { - return direction(sql` - case - when ${downloadTasks.totalBytes} is null or ${downloadTasks.totalBytes} = 0 then 0 - else (${downloadTasks.downloadedBytes} * 1000000 / ${downloadTasks.totalBytes}) - end - `) - } - if (sortBy === 'eta') { - return direction(sql`coalesce(json_extract(${downloadTasks.detail}, '$.etaSeconds'), 9223372036854775807)`) - } - return direction(downloadTasks.createdAt) -} - -export async function getDownloadTask(platform: Platform, orgId: string, id: string): Promise { - const rows = await platform.db - .select() - .from(downloadTasks) - .where(and(eq(downloadTasks.id, id), eq(downloadTasks.orgId, orgId))) - .limit(1) - if (!rows[0]) throw new DownloadError('not_found') - return toDownloadTask(rows[0]) -} - -export async function updateDownloadTask( - platform: Platform, - id: string, - input: UpdateDownloadTaskInput, - actor: { orgId?: string; downloaderId?: string }, -): Promise { - const rows = await platform.db.select().from(downloadTasks).where(eq(downloadTasks.id, id)).limit(1) - const task = rows[0] - if (!task) throw new DownloadError('not_found') - if (actor.orgId && task.orgId !== actor.orgId) throw new DownloadError('not_found') - if (actor.downloaderId && task.assignedDownloaderId !== actor.downloaderId) throw new DownloadError('forbidden') - if (actor.downloaderId && task.status === 'pausing' && input.status === 'paused') { - const now = new Date() - await platform.db - .update(downloadTasks) - .set({ status: 'paused', downloadBps: 0, uploadBps: 0, updatedAt: now }) - .where(eq(downloadTasks.id, id)) - return getDownloadTask(platform, task.orgId, id) - } - if (actor.downloaderId && task.status === 'canceling' && input.status === 'canceled') { - const now = new Date() - await platform.db - .update(downloadTasks) - .set({ status: 'canceled', downloadBps: 0, uploadBps: 0, finishedAt: task.finishedAt ?? now, updatedAt: now }) - .where(eq(downloadTasks.id, id)) - return getDownloadTask(platform, task.orgId, id) - } - if (actor.downloaderId && ['pausing', 'paused', 'canceling', 'canceled'].includes(task.status)) { - throw new DownloadError('invalid_state', `Task is ${task.status}`) - } - if (actor.orgId && !actor.downloaderId) { - const onlyCancel = - input.status === 'canceled' && - input.downloadedBytes === undefined && - input.storageUploadedBytes === undefined && - input.totalBytes === undefined && - input.downloadBps === undefined && - input.storageUploadBps === undefined && - input.errorMessage === undefined && - input.resultObjectId === undefined && - input.detail === undefined - if (!onlyCancel) throw new DownloadError('forbidden') - } - - const now = new Date() - let status = input.status ?? task.status - let authorizedBytes = task.authorizedBytes - let billedBytes = task.billedBytes - let billedCredits = task.billedCredits - let billingStatus = task.billingStatus - const downloadedBytes = - actor.downloaderId && input.downloadedBytes !== undefined - ? Math.max(input.downloadedBytes, task.downloadedBytes) - : (input.downloadedBytes ?? task.downloadedBytes) - const storageUploadedBytes = - actor.downloaderId && input.storageUploadedBytes !== undefined - ? Math.max(input.storageUploadedBytes, task.uploadedBytes) - : (input.storageUploadedBytes ?? task.uploadedBytes) - - if (actor.downloaderId && downloadedBytes > task.downloadedBytes) { - const downloader = await loadDownloaderRow(platform, actor.downloaderId) - const targetUnits = Math.ceil(downloadedBytes / downloader.remoteDownloadCreditUnitBytes) - const currentUnits = Math.ceil(task.billedBytes / downloader.remoteDownloadCreditUnitBytes) - try { - for (let unit = currentUnits + 1; unit <= targetUnits; unit += 1) { - await reportRemoteDownloadUnit({ - platform, - orgId: task.orgId, - downloaderId: actor.downloaderId, - taskId: task.id, - unitIndex: unit, - unitBytes: downloader.remoteDownloadCreditUnitBytes, - creditsPerUnit: downloader.remoteDownloadCreditPerUnit, - enabled: downloader.remoteDownloadCreditBillingEnabled, - }) - billedCredits += downloader.remoteDownloadCreditBillingEnabled ? downloader.remoteDownloadCreditPerUnit : 0 - } - if (targetUnits > currentUnits) { - billedBytes = targetUnits * downloader.remoteDownloadCreditUnitBytes - authorizedBytes = billedBytes - billingStatus = 'ok' - } - } catch (error) { - if (error instanceof RemoteDownloadBillingBlockedError) { - status = 'billing_paused' - billingStatus = 'insufficient_credits' - } else { - throw error - } - } - } - - const nextFinishedAt = - task.finishedAt ?? (input.status !== undefined && ['completed', 'failed', 'canceled'].includes(status) ? now : null) - - await platform.db - .update(downloadTasks) - .set({ - status, - downloadedBytes, - uploadedBytes: storageUploadedBytes, - totalBytes: input.totalBytes === undefined ? task.totalBytes : input.totalBytes, - authorizedBytes, - billedBytes, - billedCredits, - billingStatus, - downloadBps: input.downloadBps ?? task.downloadBps, - uploadBps: input.storageUploadBps ?? task.uploadBps, - errorMessage: input.errorMessage === undefined ? task.errorMessage : input.errorMessage, - resultObjectId: input.resultObjectId === undefined ? task.resultObjectId : input.resultObjectId, - detail: input.detail === undefined ? task.detail : JSON.stringify(input.detail), - startedAt: task.startedAt ?? (status === 'running' ? now : null), - finishedAt: nextFinishedAt, - updatedAt: now, - }) - .where(eq(downloadTasks.id, id)) - - return getDownloadTask(platform, task.orgId, id) -} - -export async function performDownloadTaskAction( - platform: Platform, - orgId: string, - id: string, - action: DownloadTaskActionInput['action'], -): Promise { - const rows = await platform.db - .select() - .from(downloadTasks) - .where(and(eq(downloadTasks.id, id), eq(downloadTasks.orgId, orgId))) - .limit(1) - const task = rows[0] - if (!task) throw new DownloadError('not_found') - - if (action === 'delete') { - if (!TERMINAL_TASK_STATUSES.includes(task.status as (typeof TERMINAL_TASK_STATUSES)[number])) { - throw new DownloadError('invalid_state', 'Only completed, failed, or canceled tasks can be deleted') - } - await platform.db.delete(downloadTasks).where(eq(downloadTasks.id, id)) - return { id, deleted: true } - } - - const now = new Date() - if (action === 'pause') { - if (task.status === 'paused') return toDownloadTask(task) - if (!PAUSABLE_TASK_STATUSES.includes(task.status as (typeof PAUSABLE_TASK_STATUSES)[number])) { - throw new DownloadError('invalid_state', 'Only queued, assigned, or running tasks can be paused') - } - const status = task.status === 'running' ? 'pausing' : 'paused' - await platform.db - .update(downloadTasks) - .set({ status, downloadBps: 0, uploadBps: 0, updatedAt: now }) - .where(eq(downloadTasks.id, id)) - return getDownloadTask(platform, orgId, id) - } - - if (action === 'resume') { - if (task.status !== 'paused') throw new DownloadError('invalid_state', 'Only paused tasks can be resumed') - await platform.db - .update(downloadTasks) - .set({ - status: 'queued', - assignedDownloaderId: null, - uploadTokenHash: null, - uploadTokenJti: null, - uploadTokenExpiresAt: null, - assignedAt: null, - downloadBps: 0, - uploadBps: 0, - updatedAt: now, - }) - .where(eq(downloadTasks.id, id)) - await assignQueuedTasks(platform) - return getDownloadTask(platform, orgId, id) - } - - if (action === 'cancel') { - if (task.status === 'canceled') return toDownloadTask(task) - if (!CANCELABLE_TASK_STATUSES.includes(task.status as (typeof CANCELABLE_TASK_STATUSES)[number])) { - throw new DownloadError('invalid_state', 'Only active or paused tasks can be canceled') - } - const status = - task.assignedDownloaderId && ['assigned', 'running', 'uploading', 'pausing'].includes(task.status) - ? 'canceling' - : 'canceled' - await platform.db - .update(downloadTasks) - .set({ - status, - downloadBps: 0, - uploadBps: 0, - finishedAt: status === 'canceled' ? (task.finishedAt ?? now) : task.finishedAt, - updatedAt: now, - }) - .where(eq(downloadTasks.id, id)) - return getDownloadTask(platform, orgId, id) - } - - if (action === 'retry') { - if (!['failed', 'canceled'].includes(task.status)) { - throw new DownloadError('invalid_state', 'Only failed or canceled tasks can be retried') - } - await platform.db - .update(downloadTasks) - .set({ - status: 'queued', - assignedDownloaderId: null, - uploadTokenHash: null, - uploadTokenJti: null, - uploadTokenExpiresAt: null, - downloadedBytes: 0, - uploadedBytes: 0, - totalBytes: null, - downloadBps: 0, - uploadBps: 0, - errorMessage: null, - resultObjectId: null, - detail: null, - assignedAt: null, - startedAt: null, - finishedAt: null, - updatedAt: now, - }) - .where(eq(downloadTasks.id, id)) - await assignQueuedTasks(platform) - return getDownloadTask(platform, orgId, id) - } - - throw new DownloadError('invalid_state') -} - -export async function assertTaskUploadAllowed(platform: Platform, params: { taskId: string; downloaderId: string }) { - const rows = await platform.db.select().from(downloadTasks).where(eq(downloadTasks.id, params.taskId)).limit(1) - const task = rows[0] - if (!task || task.assignedDownloaderId !== params.downloaderId) throw new DownloadError('forbidden') - if (!['assigned', 'running', 'uploading'].includes(task.status)) throw new DownloadError('invalid_state') - return task -} - -async function assignQueuedTasks(platform: Platform): Promise { - const tasks = await platform.db - .select() - .from(downloadTasks) - .where(eq(downloadTasks.status, 'queued')) - .orderBy(asc(downloadTasks.createdAt)) - .limit(20) - for (const task of tasks) { - const downloader = await selectDownloader(platform, task.sourceType) - if (!downloader) continue - const token = await createTaskUploadToken(platform, { - taskId: task.id, - downloaderId: downloader.id, - orgId: task.orgId, - targetFolder: task.targetFolder, - createdByUserId: task.createdByUserId, - }) - const now = new Date() - await platform.db - .update(downloadTasks) - .set({ - status: 'assigned', - assignedDownloaderId: downloader.id, - uploadTokenHash: token.hash, - uploadTokenJti: token.jti, - uploadTokenExpiresAt: token.expiresAt, - assignedAt: now, - updatedAt: now, - }) - .where(eq(downloadTasks.id, task.id)) - } -} - -async function selectDownloader(platform: Platform, sourceType: string): Promise { - const needed = sourceType === 'http' ? ['http'] : ['magnet', 'torrent'] - const rows = await platform.db - .select() - .from(downloaders) - .where(and(eq(downloaders.enabled, true), eq(downloaders.status, 'online'))) - .orderBy(asc(downloaders.currentTasks), asc(downloaders.downloadBps)) - return ( - rows.find((row) => { - const capabilities = parseCapabilities(row.capabilities) - return needed.some((capability) => capabilities.includes(capability)) - }) ?? null - ) -} - -async function createTaskUploadToken( - platform: Platform, - params: { taskId: string; downloaderId: string; orgId: string; targetFolder: string; createdByUserId: string }, -) { - const now = Math.floor(Date.now() / 1000) - const jti = nanoid() - const expiresAt = new Date((now + UPLOAD_TOKEN_TTL_SECONDS) * 1000) - const token = await signDownloadToken(platform, { - v: 1, - typ: 'download-task-upload', - taskId: params.taskId, - downloaderId: params.downloaderId, - orgId: params.orgId, - targetFolder: params.targetFolder, - createdByUserId: params.createdByUserId, - scopes: ['objects:create', 'objects:upload', 'objects:confirm'], - jti, - iat: now, - exp: now + UPLOAD_TOKEN_TTL_SECONDS, - }) - return { token, hash: await hashDownloadToken(platform, token), jti, expiresAt } -} - -async function toDownloadTaskWithToken(platform: Platform, row: DownloadTaskRow, includeUploadToken: boolean) { - const task = toDownloadTask(row) - if (includeUploadToken && row.assignedDownloaderId && row.uploadTokenJti) { - const token = await createTaskUploadToken(platform, { - taskId: row.id, - downloaderId: row.assignedDownloaderId, - orgId: row.orgId, - targetFolder: row.targetFolder, - createdByUserId: row.createdByUserId, - }) - await platform.db - .update(downloadTasks) - .set({ uploadTokenHash: token.hash, uploadTokenJti: token.jti, uploadTokenExpiresAt: token.expiresAt }) - .where(eq(downloadTasks.id, row.id)) - task.uploadToken = token.token - } - return task -} - -async function loadDownloaderRow(platform: Platform, id: string): Promise { - const rows = await platform.db.select().from(downloaders).where(eq(downloaders.id, id)).limit(1) - if (!rows[0]) throw new DownloadError('not_found') - return rows[0] -} - -function toDownloader(row: DownloaderRow): Downloader { - return { - id: row.id, - name: row.name, - status: row.enabled ? (row.status as Downloader['status']) : 'disabled', - enabled: row.enabled, - version: row.version, - hostname: row.hostname, - platform: row.platform, - arch: row.arch, - engine: row.engine as Downloader['engine'], - capabilities: parseCapabilities(row.capabilities), - maxConcurrentTasks: row.maxConcurrentTasks, - currentTasks: row.currentTasks, - downloadBps: row.downloadBps, - uploadBps: row.uploadBps, - freeDiskBytes: row.freeDiskBytes, - remoteDownloadCreditBillingEnabled: row.remoteDownloadCreditBillingEnabled, - remoteDownloadCreditUnitBytes: row.remoteDownloadCreditUnitBytes, - remoteDownloadCreditPerUnit: row.remoteDownloadCreditPerUnit, - lastHeartbeatAt: row.lastHeartbeatAt?.toISOString() ?? null, - createdBy: row.createdBy, - createdAt: row.createdAt.toISOString(), - updatedAt: row.updatedAt.toISOString(), - } -} - -function toDownloadTask(row: DownloadTaskRow): DownloadTask { - return { - id: row.id, - orgId: row.orgId, - createdByUserId: row.createdByUserId, - sourceType: row.sourceType as DownloadTask['sourceType'], - sourceUri: row.sourceUri, - name: row.name, - targetFolder: row.targetFolder, - category: row.category, - tags: parseTaskTags(row.tags), - assignedDownloaderId: row.assignedDownloaderId, - status: row.status as DownloadTask['status'], - downloadedBytes: row.downloadedBytes, - storageUploadedBytes: row.uploadedBytes, - totalBytes: row.totalBytes, - authorizedBytes: row.authorizedBytes, - billedBytes: row.billedBytes, - billedCredits: row.billedCredits, - billingStatus: row.billingStatus, - downloadBps: row.downloadBps, - storageUploadBps: row.uploadBps, - errorMessage: row.errorMessage, - resultObjectId: row.resultObjectId, - detail: parseTaskDetail(row.detail), - createdAt: row.createdAt.toISOString(), - updatedAt: row.updatedAt.toISOString(), - assignedAt: row.assignedAt?.toISOString() ?? null, - startedAt: row.startedAt?.toISOString() ?? null, - finishedAt: row.finishedAt?.toISOString() ?? null, - } -} - -function parseTaskDetail(value: string | null): DownloadTask['detail'] { - if (!value) return null - try { - const detail = JSON.parse(value) as DownloadTask['detail'] & { uploadedBytes?: number } - if (detail.peerUploadedBytes === undefined && detail.uploadedBytes !== undefined) { - detail.peerUploadedBytes = detail.uploadedBytes - } - delete detail.uploadedBytes - return detail - } catch { - return null - } -} - -function parseTaskTags(value: string): string[] { - try { - const parsed = JSON.parse(value) - return Array.isArray(parsed) ? parsed.filter((item): item is string => typeof item === 'string') : [] - } catch { - return [] - } -} - -function parseCapabilities(value: string): string[] { - try { - const parsed = JSON.parse(value) - return Array.isArray(parsed) ? parsed.filter((item): item is string => typeof item === 'string') : [] - } catch { - return [] - } -} +export * from './downloads/core' +export * from './downloads/types' diff --git a/server/services/downloads/core.ts b/server/services/downloads/core.ts new file mode 100644 index 00000000..88f6ea44 --- /dev/null +++ b/server/services/downloads/core.ts @@ -0,0 +1,643 @@ +import type { + CreateDownloaderInput, + CreateDownloadTaskInput, + DownloaderHeartbeatInput, + DownloadTaskActionInput, + UpdateDownloaderInput, + UpdateDownloadTaskInput, +} from '@shared/schemas' +import type { Downloader, DownloadTask } from '@shared/types' +import { and, asc, count, desc, eq, inArray, like, sql } from 'drizzle-orm' +import { nanoid } from 'nanoid' +import { downloaders, downloadTasks } from '../../db/schema' +import type { Platform } from '../../platform/interface' +import { hashDownloadToken, signDownloadToken } from '../download-tokens' +import { RemoteDownloadBillingBlockedError, reportRemoteDownloadUnit } from '../remote-download-usage' +import { parseCapabilities, toDownloader, toDownloadTask } from './mappers' +import { DownloadError, type DownloaderRow, type DownloadTaskRow } from './types' + +const DEFAULT_REMOTE_DOWNLOAD_UNIT_BYTES = 100 * 1024 * 1024 +const UPLOAD_TOKEN_TTL_SECONDS = 24 * 60 * 60 +const PAUSABLE_TASK_STATUSES = ['queued', 'assigned', 'running'] as const +const CANCELABLE_TASK_STATUSES = [ + 'queued', + 'assigned', + 'running', + 'billing_paused', + 'paused', + 'uploading', + 'pausing', +] as const +const TERMINAL_TASK_STATUSES = ['completed', 'failed', 'canceled'] as const +const DOWNLOADER_TOKEN_TASK_STATUSES = ['assigned', 'running', 'uploading', 'billing_paused'] as const + +export async function createDownloader( + platform: Platform, + input: CreateDownloaderInput, + userId: string, +): Promise<{ downloader: Downloader; token: string }> { + const now = new Date() + const id = nanoid() + const jti = nanoid() + const token = await signDownloadToken(platform, { + v: 1, + typ: 'downloader', + downloaderId: id, + jti, + iat: Math.floor(now.getTime() / 1000), + }) + await platform.db.insert(downloaders).values({ + id, + name: input.name, + tokenHash: await hashDownloadToken(platform, token), + tokenJti: jti, + status: 'offline', + enabled: true, + version: input.heartbeat.version, + hostname: input.heartbeat.hostname, + platform: input.heartbeat.platform, + arch: input.heartbeat.arch, + engine: input.heartbeat.engine, + capabilities: JSON.stringify(input.heartbeat.capabilities), + maxConcurrentTasks: input.heartbeat.maxConcurrentTasks, + currentTasks: input.heartbeat.currentTasks, + downloadBps: input.heartbeat.downloadBps, + uploadBps: input.heartbeat.uploadBps, + freeDiskBytes: input.heartbeat.freeDiskBytes, + remoteDownloadCreditBillingEnabled: false, + remoteDownloadCreditUnitBytes: DEFAULT_REMOTE_DOWNLOAD_UNIT_BYTES, + remoteDownloadCreditPerUnit: 1, + lastHeartbeatAt: null, + createdBy: userId, + createdAt: now, + updatedAt: now, + }) + return { downloader: await getDownloader(platform, id), token } +} + +export async function listDownloaders(platform: Platform): Promise { + const rows = await platform.db.select().from(downloaders).orderBy(desc(downloaders.createdAt)) + return rows.map(toDownloader) +} + +export async function getDownloader(platform: Platform, id: string): Promise { + const rows = await platform.db.select().from(downloaders).where(eq(downloaders.id, id)).limit(1) + if (!rows[0]) throw new DownloadError('not_found') + return toDownloader(rows[0]) +} + +export async function updateDownloader( + platform: Platform, + id: string, + input: UpdateDownloaderInput, +): Promise { + const rows = await platform.db.select({ id: downloaders.id }).from(downloaders).where(eq(downloaders.id, id)).limit(1) + if (!rows[0]) throw new DownloadError('not_found') + const now = new Date() + await platform.db + .update(downloaders) + .set({ + ...(input.name !== undefined ? { name: input.name } : {}), + ...(input.enabled !== undefined + ? { enabled: input.enabled, status: input.enabled ? 'offline' : 'disabled' } + : {}), + ...(input.remoteDownloadCreditBillingEnabled !== undefined + ? { remoteDownloadCreditBillingEnabled: input.remoteDownloadCreditBillingEnabled } + : {}), + ...(input.remoteDownloadCreditUnitBytes !== undefined + ? { remoteDownloadCreditUnitBytes: input.remoteDownloadCreditUnitBytes } + : {}), + ...(input.remoteDownloadCreditPerUnit !== undefined + ? { remoteDownloadCreditPerUnit: input.remoteDownloadCreditPerUnit } + : {}), + updatedAt: now, + }) + .where(eq(downloaders.id, id)) + return getDownloader(platform, id) +} + +export async function deleteDownloader(platform: Platform, id: string): Promise<{ id: string; deleted: true }> { + const rows = await platform.db.select({ id: downloaders.id }).from(downloaders).where(eq(downloaders.id, id)).limit(1) + if (!rows[0]) throw new DownloadError('not_found') + const now = new Date() + + await platform.db + .update(downloadTasks) + .set({ + status: 'queued', + assignedDownloaderId: null, + uploadTokenHash: null, + uploadTokenJti: null, + uploadTokenExpiresAt: null, + assignedAt: null, + updatedAt: now, + }) + .where( + and( + eq(downloadTasks.assignedDownloaderId, id), + inArray(downloadTasks.status, [ + 'queued', + 'assigned', + 'running', + 'billing_paused', + 'pausing', + 'paused', + 'uploading', + 'canceling', + ]), + ), + ) + await platform.db.delete(downloaders).where(eq(downloaders.id, id)) + return { id, deleted: true } +} + +export async function recordDownloaderHeartbeat( + platform: Platform, + downloaderId: string, + heartbeat: DownloaderHeartbeatInput, +): Promise { + const rows = await platform.db + .select({ id: downloaders.id, enabled: downloaders.enabled }) + .from(downloaders) + .where(eq(downloaders.id, downloaderId)) + .limit(1) + if (!rows[0]) throw new DownloadError('not_found') + const now = new Date() + await platform.db + .update(downloaders) + .set({ + status: rows[0].enabled ? 'online' : 'disabled', + version: heartbeat.version, + hostname: heartbeat.hostname, + platform: heartbeat.platform, + arch: heartbeat.arch, + engine: heartbeat.engine, + capabilities: JSON.stringify(heartbeat.capabilities), + maxConcurrentTasks: heartbeat.maxConcurrentTasks, + currentTasks: heartbeat.currentTasks, + downloadBps: heartbeat.downloadBps, + uploadBps: heartbeat.uploadBps, + freeDiskBytes: heartbeat.freeDiskBytes, + lastHeartbeatAt: now, + updatedAt: now, + }) + .where(eq(downloaders.id, downloaderId)) + await assignQueuedTasks(platform) + return getDownloader(platform, downloaderId) +} + +export async function createDownloadTask( + platform: Platform, + orgId: string, + userId: string, + input: CreateDownloadTaskInput, +): Promise { + const now = new Date() + const id = nanoid() + const assigned = await selectDownloader(platform, input.source.type) + const uploadToken = assigned + ? await createTaskUploadToken(platform, { + taskId: id, + downloaderId: assigned.id, + orgId, + targetFolder: input.targetFolder, + createdByUserId: userId, + }) + : null + await platform.db.insert(downloadTasks).values({ + id, + orgId, + createdByUserId: userId, + sourceType: input.source.type, + sourceUri: input.source.uri, + name: input.name ?? null, + targetFolder: input.targetFolder, + category: input.category ?? null, + tags: JSON.stringify(input.tags ?? []), + assignedDownloaderId: assigned?.id ?? null, + status: assigned ? 'assigned' : 'queued', + uploadTokenHash: uploadToken?.hash ?? null, + uploadTokenJti: uploadToken?.jti ?? null, + uploadTokenExpiresAt: uploadToken?.expiresAt ?? null, + createdAt: now, + updatedAt: now, + assignedAt: assigned ? now : null, + }) + return getDownloadTask(platform, orgId, id) +} + +export async function listDownloadTasks( + platform: Platform, + opts: { + orgId?: string + downloaderId?: string + status?: string + category?: string + tag?: string + sortBy?: 'createdAt' | 'source' | 'category' | 'tags' | 'status' | 'progress' | 'eta' + sortDir?: 'asc' | 'desc' + page: number + pageSize: number + includeUploadToken?: boolean + }, +): Promise<{ items: DownloadTask[]; total: number }> { + const offset = (opts.page - 1) * opts.pageSize + const filters = [] + if (opts.orgId) filters.push(eq(downloadTasks.orgId, opts.orgId)) + if (opts.downloaderId) filters.push(eq(downloadTasks.assignedDownloaderId, opts.downloaderId)) + if (opts.status) filters.push(eq(downloadTasks.status, opts.status)) + if (opts.category) filters.push(eq(downloadTasks.category, opts.category)) + if (opts.tag) filters.push(like(downloadTasks.tags, `%${JSON.stringify(opts.tag)}%`)) + const where = filters.length ? and(...filters) : undefined + const [rows, totalRows] = await Promise.all([ + platform.db + .select() + .from(downloadTasks) + .where(where) + .orderBy(downloadTaskOrderBy(opts.sortBy ?? 'createdAt', opts.sortDir ?? 'desc')) + .limit(opts.pageSize) + .offset(offset), + platform.db.select({ count: count() }).from(downloadTasks).where(where), + ]) + return { + items: await Promise.all( + rows.map((row) => + toDownloadTaskWithToken( + platform, + row, + (opts.includeUploadToken ?? false) && + DOWNLOADER_TOKEN_TASK_STATUSES.includes(row.status as (typeof DOWNLOADER_TOKEN_TASK_STATUSES)[number]), + ), + ), + ), + total: totalRows[0]?.count ?? 0, + } +} + +function downloadTaskOrderBy( + sortBy: 'createdAt' | 'source' | 'category' | 'tags' | 'status' | 'progress' | 'eta', + sortDir: 'asc' | 'desc', +) { + const direction = sortDir === 'asc' ? asc : desc + if (sortBy === 'source') return direction(downloadTasks.sourceUri) + if (sortBy === 'category') return direction(downloadTasks.category) + if (sortBy === 'tags') return direction(downloadTasks.tags) + if (sortBy === 'status') return direction(downloadTasks.status) + if (sortBy === 'progress') { + return direction(sql` + case + when ${downloadTasks.totalBytes} is null or ${downloadTasks.totalBytes} = 0 then 0 + else (${downloadTasks.downloadedBytes} * 1000000 / ${downloadTasks.totalBytes}) + end + `) + } + if (sortBy === 'eta') { + return direction(sql`coalesce(json_extract(${downloadTasks.detail}, '$.etaSeconds'), 9223372036854775807)`) + } + return direction(downloadTasks.createdAt) +} + +export async function getDownloadTask(platform: Platform, orgId: string, id: string): Promise { + const rows = await platform.db + .select() + .from(downloadTasks) + .where(and(eq(downloadTasks.id, id), eq(downloadTasks.orgId, orgId))) + .limit(1) + if (!rows[0]) throw new DownloadError('not_found') + return toDownloadTask(rows[0]) +} + +export async function updateDownloadTask( + platform: Platform, + id: string, + input: UpdateDownloadTaskInput, + actor: { orgId?: string; downloaderId?: string }, +): Promise { + const rows = await platform.db.select().from(downloadTasks).where(eq(downloadTasks.id, id)).limit(1) + const task = rows[0] + if (!task) throw new DownloadError('not_found') + if (actor.orgId && task.orgId !== actor.orgId) throw new DownloadError('not_found') + if (actor.downloaderId && task.assignedDownloaderId !== actor.downloaderId) throw new DownloadError('forbidden') + if (actor.downloaderId && task.status === 'pausing' && input.status === 'paused') { + const now = new Date() + await platform.db + .update(downloadTasks) + .set({ status: 'paused', downloadBps: 0, uploadBps: 0, updatedAt: now }) + .where(eq(downloadTasks.id, id)) + return getDownloadTask(platform, task.orgId, id) + } + if (actor.downloaderId && task.status === 'canceling' && input.status === 'canceled') { + const now = new Date() + await platform.db + .update(downloadTasks) + .set({ status: 'canceled', downloadBps: 0, uploadBps: 0, finishedAt: task.finishedAt ?? now, updatedAt: now }) + .where(eq(downloadTasks.id, id)) + return getDownloadTask(platform, task.orgId, id) + } + if (actor.downloaderId && ['pausing', 'paused', 'canceling', 'canceled'].includes(task.status)) { + throw new DownloadError('invalid_state', `Task is ${task.status}`) + } + if (actor.orgId && !actor.downloaderId) { + const onlyCancel = + input.status === 'canceled' && + input.downloadedBytes === undefined && + input.storageUploadedBytes === undefined && + input.totalBytes === undefined && + input.downloadBps === undefined && + input.storageUploadBps === undefined && + input.errorMessage === undefined && + input.resultObjectId === undefined && + input.detail === undefined + if (!onlyCancel) throw new DownloadError('forbidden') + } + + const now = new Date() + let status = input.status ?? task.status + let authorizedBytes = task.authorizedBytes + let billedBytes = task.billedBytes + let billedCredits = task.billedCredits + let billingStatus = task.billingStatus + const downloadedBytes = + actor.downloaderId && input.downloadedBytes !== undefined + ? Math.max(input.downloadedBytes, task.downloadedBytes) + : (input.downloadedBytes ?? task.downloadedBytes) + const storageUploadedBytes = + actor.downloaderId && input.storageUploadedBytes !== undefined + ? Math.max(input.storageUploadedBytes, task.uploadedBytes) + : (input.storageUploadedBytes ?? task.uploadedBytes) + + if (actor.downloaderId && downloadedBytes > task.downloadedBytes) { + const downloader = await loadDownloaderRow(platform, actor.downloaderId) + const targetUnits = Math.ceil(downloadedBytes / downloader.remoteDownloadCreditUnitBytes) + const currentUnits = Math.ceil(task.billedBytes / downloader.remoteDownloadCreditUnitBytes) + try { + for (let unit = currentUnits + 1; unit <= targetUnits; unit += 1) { + await reportRemoteDownloadUnit({ + platform, + orgId: task.orgId, + downloaderId: actor.downloaderId, + taskId: task.id, + unitIndex: unit, + unitBytes: downloader.remoteDownloadCreditUnitBytes, + creditsPerUnit: downloader.remoteDownloadCreditPerUnit, + enabled: downloader.remoteDownloadCreditBillingEnabled, + }) + billedCredits += downloader.remoteDownloadCreditBillingEnabled ? downloader.remoteDownloadCreditPerUnit : 0 + } + if (targetUnits > currentUnits) { + billedBytes = targetUnits * downloader.remoteDownloadCreditUnitBytes + authorizedBytes = billedBytes + billingStatus = 'ok' + } + } catch (error) { + if (error instanceof RemoteDownloadBillingBlockedError) { + status = 'billing_paused' + billingStatus = 'insufficient_credits' + } else { + throw error + } + } + } + + const nextFinishedAt = + task.finishedAt ?? (input.status !== undefined && ['completed', 'failed', 'canceled'].includes(status) ? now : null) + + await platform.db + .update(downloadTasks) + .set({ + status, + downloadedBytes, + uploadedBytes: storageUploadedBytes, + totalBytes: input.totalBytes === undefined ? task.totalBytes : input.totalBytes, + authorizedBytes, + billedBytes, + billedCredits, + billingStatus, + downloadBps: input.downloadBps ?? task.downloadBps, + uploadBps: input.storageUploadBps ?? task.uploadBps, + errorMessage: input.errorMessage === undefined ? task.errorMessage : input.errorMessage, + resultObjectId: input.resultObjectId === undefined ? task.resultObjectId : input.resultObjectId, + detail: input.detail === undefined ? task.detail : JSON.stringify(input.detail), + startedAt: task.startedAt ?? (status === 'running' ? now : null), + finishedAt: nextFinishedAt, + updatedAt: now, + }) + .where(eq(downloadTasks.id, id)) + + return getDownloadTask(platform, task.orgId, id) +} + +export async function performDownloadTaskAction( + platform: Platform, + orgId: string, + id: string, + action: DownloadTaskActionInput['action'], +): Promise { + const rows = await platform.db + .select() + .from(downloadTasks) + .where(and(eq(downloadTasks.id, id), eq(downloadTasks.orgId, orgId))) + .limit(1) + const task = rows[0] + if (!task) throw new DownloadError('not_found') + + if (action === 'delete') { + if (!TERMINAL_TASK_STATUSES.includes(task.status as (typeof TERMINAL_TASK_STATUSES)[number])) { + throw new DownloadError('invalid_state', 'Only completed, failed, or canceled tasks can be deleted') + } + await platform.db.delete(downloadTasks).where(eq(downloadTasks.id, id)) + return { id, deleted: true } + } + + const now = new Date() + if (action === 'pause') { + if (task.status === 'paused') return toDownloadTask(task) + if (!PAUSABLE_TASK_STATUSES.includes(task.status as (typeof PAUSABLE_TASK_STATUSES)[number])) { + throw new DownloadError('invalid_state', 'Only queued, assigned, or running tasks can be paused') + } + const status = task.status === 'running' ? 'pausing' : 'paused' + await platform.db + .update(downloadTasks) + .set({ status, downloadBps: 0, uploadBps: 0, updatedAt: now }) + .where(eq(downloadTasks.id, id)) + return getDownloadTask(platform, orgId, id) + } + + if (action === 'resume') { + if (task.status !== 'paused') throw new DownloadError('invalid_state', 'Only paused tasks can be resumed') + await platform.db + .update(downloadTasks) + .set({ + status: 'queued', + assignedDownloaderId: null, + uploadTokenHash: null, + uploadTokenJti: null, + uploadTokenExpiresAt: null, + assignedAt: null, + downloadBps: 0, + uploadBps: 0, + updatedAt: now, + }) + .where(eq(downloadTasks.id, id)) + await assignQueuedTasks(platform) + return getDownloadTask(platform, orgId, id) + } + + if (action === 'cancel') { + if (task.status === 'canceled') return toDownloadTask(task) + if (!CANCELABLE_TASK_STATUSES.includes(task.status as (typeof CANCELABLE_TASK_STATUSES)[number])) { + throw new DownloadError('invalid_state', 'Only active or paused tasks can be canceled') + } + const status = + task.assignedDownloaderId && ['assigned', 'running', 'uploading', 'pausing'].includes(task.status) + ? 'canceling' + : 'canceled' + await platform.db + .update(downloadTasks) + .set({ + status, + downloadBps: 0, + uploadBps: 0, + finishedAt: status === 'canceled' ? (task.finishedAt ?? now) : task.finishedAt, + updatedAt: now, + }) + .where(eq(downloadTasks.id, id)) + return getDownloadTask(platform, orgId, id) + } + + if (action === 'retry') { + if (!['failed', 'canceled'].includes(task.status)) { + throw new DownloadError('invalid_state', 'Only failed or canceled tasks can be retried') + } + await platform.db + .update(downloadTasks) + .set({ + status: 'queued', + assignedDownloaderId: null, + uploadTokenHash: null, + uploadTokenJti: null, + uploadTokenExpiresAt: null, + downloadedBytes: 0, + uploadedBytes: 0, + totalBytes: null, + downloadBps: 0, + uploadBps: 0, + errorMessage: null, + resultObjectId: null, + detail: null, + assignedAt: null, + startedAt: null, + finishedAt: null, + updatedAt: now, + }) + .where(eq(downloadTasks.id, id)) + await assignQueuedTasks(platform) + return getDownloadTask(platform, orgId, id) + } + + throw new DownloadError('invalid_state') +} + +export async function assertTaskUploadAllowed(platform: Platform, params: { taskId: string; downloaderId: string }) { + const rows = await platform.db.select().from(downloadTasks).where(eq(downloadTasks.id, params.taskId)).limit(1) + const task = rows[0] + if (!task || task.assignedDownloaderId !== params.downloaderId) throw new DownloadError('forbidden') + if (!['assigned', 'running', 'uploading'].includes(task.status)) throw new DownloadError('invalid_state') + return task +} + +async function assignQueuedTasks(platform: Platform): Promise { + const tasks = await platform.db + .select() + .from(downloadTasks) + .where(eq(downloadTasks.status, 'queued')) + .orderBy(asc(downloadTasks.createdAt)) + .limit(20) + for (const task of tasks) { + const downloader = await selectDownloader(platform, task.sourceType) + if (!downloader) continue + const token = await createTaskUploadToken(platform, { + taskId: task.id, + downloaderId: downloader.id, + orgId: task.orgId, + targetFolder: task.targetFolder, + createdByUserId: task.createdByUserId, + }) + const now = new Date() + await platform.db + .update(downloadTasks) + .set({ + status: 'assigned', + assignedDownloaderId: downloader.id, + uploadTokenHash: token.hash, + uploadTokenJti: token.jti, + uploadTokenExpiresAt: token.expiresAt, + assignedAt: now, + updatedAt: now, + }) + .where(eq(downloadTasks.id, task.id)) + } +} + +async function selectDownloader(platform: Platform, sourceType: string): Promise { + const needed = sourceType === 'http' ? ['http'] : ['magnet', 'torrent'] + const rows = await platform.db + .select() + .from(downloaders) + .where(and(eq(downloaders.enabled, true), eq(downloaders.status, 'online'))) + .orderBy(asc(downloaders.currentTasks), asc(downloaders.downloadBps)) + return ( + rows.find((row) => { + const capabilities = parseCapabilities(row.capabilities) + return needed.some((capability) => capabilities.includes(capability)) + }) ?? null + ) +} + +async function createTaskUploadToken( + platform: Platform, + params: { taskId: string; downloaderId: string; orgId: string; targetFolder: string; createdByUserId: string }, +) { + const now = Math.floor(Date.now() / 1000) + const jti = nanoid() + const expiresAt = new Date((now + UPLOAD_TOKEN_TTL_SECONDS) * 1000) + const token = await signDownloadToken(platform, { + v: 1, + typ: 'download-task-upload', + taskId: params.taskId, + downloaderId: params.downloaderId, + orgId: params.orgId, + targetFolder: params.targetFolder, + createdByUserId: params.createdByUserId, + scopes: ['objects:create', 'objects:upload', 'objects:confirm'], + jti, + iat: now, + exp: now + UPLOAD_TOKEN_TTL_SECONDS, + }) + return { token, hash: await hashDownloadToken(platform, token), jti, expiresAt } +} + +async function toDownloadTaskWithToken(platform: Platform, row: DownloadTaskRow, includeUploadToken: boolean) { + const task = toDownloadTask(row) + if (includeUploadToken && row.assignedDownloaderId) { + const token = await createTaskUploadToken(platform, { + taskId: row.id, + downloaderId: row.assignedDownloaderId, + orgId: row.orgId, + targetFolder: row.targetFolder, + createdByUserId: row.createdByUserId, + }) + await platform.db + .update(downloadTasks) + .set({ uploadTokenHash: token.hash, uploadTokenJti: token.jti, uploadTokenExpiresAt: token.expiresAt }) + .where(eq(downloadTasks.id, row.id)) + task.uploadToken = token.token + } + return task +} + +async function loadDownloaderRow(platform: Platform, id: string): Promise { + const rows = await platform.db.select().from(downloaders).where(eq(downloaders.id, id)).limit(1) + if (!rows[0]) throw new DownloadError('not_found') + return rows[0] +} diff --git a/server/services/downloads/mappers.ts b/server/services/downloads/mappers.ts new file mode 100644 index 00000000..5339249d --- /dev/null +++ b/server/services/downloads/mappers.ts @@ -0,0 +1,93 @@ +import type { Downloader, DownloadTask } from '@shared/types' +import type { DownloaderRow, DownloadTaskRow } from './types' + +export function toDownloader(row: DownloaderRow): Downloader { + return { + id: row.id, + name: row.name, + status: row.enabled ? (row.status as Downloader['status']) : 'disabled', + enabled: row.enabled, + version: row.version, + hostname: row.hostname, + platform: row.platform, + arch: row.arch, + engine: row.engine as Downloader['engine'], + capabilities: parseCapabilities(row.capabilities), + maxConcurrentTasks: row.maxConcurrentTasks, + currentTasks: row.currentTasks, + downloadBps: row.downloadBps, + uploadBps: row.uploadBps, + freeDiskBytes: row.freeDiskBytes, + remoteDownloadCreditBillingEnabled: row.remoteDownloadCreditBillingEnabled, + remoteDownloadCreditUnitBytes: row.remoteDownloadCreditUnitBytes, + remoteDownloadCreditPerUnit: row.remoteDownloadCreditPerUnit, + lastHeartbeatAt: row.lastHeartbeatAt?.toISOString() ?? null, + createdBy: row.createdBy, + createdAt: row.createdAt.toISOString(), + updatedAt: row.updatedAt.toISOString(), + } +} + +export function toDownloadTask(row: DownloadTaskRow): DownloadTask { + return { + id: row.id, + orgId: row.orgId, + createdByUserId: row.createdByUserId, + sourceType: row.sourceType as DownloadTask['sourceType'], + sourceUri: row.sourceUri, + name: row.name, + targetFolder: row.targetFolder, + category: row.category, + tags: parseTaskTags(row.tags), + assignedDownloaderId: row.assignedDownloaderId, + status: row.status as DownloadTask['status'], + downloadedBytes: row.downloadedBytes, + storageUploadedBytes: row.uploadedBytes, + totalBytes: row.totalBytes, + authorizedBytes: row.authorizedBytes, + billedBytes: row.billedBytes, + billedCredits: row.billedCredits, + billingStatus: row.billingStatus, + downloadBps: row.downloadBps, + storageUploadBps: row.uploadBps, + errorMessage: row.errorMessage, + resultObjectId: row.resultObjectId, + detail: parseTaskDetail(row.detail), + createdAt: row.createdAt.toISOString(), + updatedAt: row.updatedAt.toISOString(), + assignedAt: row.assignedAt?.toISOString() ?? null, + startedAt: row.startedAt?.toISOString() ?? null, + finishedAt: row.finishedAt?.toISOString() ?? null, + } +} + +export function parseCapabilities(value: string): string[] { + return parseStringArray(value) +} + +function parseTaskDetail(value: string | null): DownloadTask['detail'] { + if (!value) return null + try { + const detail = JSON.parse(value) as DownloadTask['detail'] & { uploadedBytes?: number } + if (detail.peerUploadedBytes === undefined && detail.uploadedBytes !== undefined) { + detail.peerUploadedBytes = detail.uploadedBytes + } + delete detail.uploadedBytes + return detail + } catch { + return null + } +} + +function parseTaskTags(value: string): string[] { + return parseStringArray(value) +} + +function parseStringArray(value: string): string[] { + try { + const parsed = JSON.parse(value) + return Array.isArray(parsed) ? parsed.filter((item): item is string => typeof item === 'string') : [] + } catch { + return [] + } +} diff --git a/server/services/downloads/types.ts b/server/services/downloads/types.ts new file mode 100644 index 00000000..472731a4 --- /dev/null +++ b/server/services/downloads/types.ts @@ -0,0 +1,20 @@ +import type { downloaders, downloadTasks } from '../../db/schema' + +export class DownloadError extends Error { + constructor( + readonly code: + | 'not_found' + | 'forbidden' + | 'no_downloader' + | 'invalid_state' + | 'billing_paused' + | 'unsupported_source', + message: string = code, + ) { + super(message) + this.name = 'DownloadError' + } +} + +export type DownloaderRow = typeof downloaders.$inferSelect +export type DownloadTaskRow = typeof downloadTasks.$inferSelect diff --git a/shared/schemas/downloads.ts b/shared/schemas/downloads.ts index 48741fcb..326ddba6 100644 --- a/shared/schemas/downloads.ts +++ b/shared/schemas/downloads.ts @@ -1,4 +1,4 @@ -import { z } from 'zod' +import { z } from '@hono/zod-openapi' export const downloaderStatusSchema = z.enum(['online', 'offline', 'disabled']) export const downloaderEngineSchema = z.enum(['builtin', 'aria2', 'qbittorrent']) @@ -19,6 +19,15 @@ export const downloadTaskActionSchema = z.enum(['pause', 'resume', 'cancel', 're export const downloadSourceTypeSchema = z.enum(['http', 'magnet', 'torrent_url']) export const downloadTaskPhaseSchema = z.enum(['metadata', 'downloading', 'uploading', 'seeding', 'completed', 'error']) +const int64Schema = () => z.number().int().min(0).openapi({ type: 'integer', format: 'int64' }) +const nullableInt64Schema = () => + z + .number() + .int() + .min(0) + .nullable() + .openapi({ type: 'integer', format: 'int64', nullable: true } as never) + const downloadTaskTrackerSchema = z.object({ url: z.string().max(1024), status: z.string().max(80).optional(), @@ -32,14 +41,14 @@ const downloadTaskPeerSchema = z.object({ address: z.string().max(160), client: z.string().max(160).optional(), progress: z.number().min(0).max(1).optional(), - downloadBps: z.number().int().min(0).optional(), - uploadBps: z.number().int().min(0).optional(), + downloadBps: int64Schema().optional(), + uploadBps: int64Schema().optional(), }) const downloadTaskFileSchema = z.object({ path: z.string().max(1024), - size: z.number().int().min(0), - completedBytes: z.number().int().min(0).optional(), + size: int64Schema(), + completedBytes: int64Schema().optional(), selected: z.boolean().optional(), }) @@ -55,13 +64,41 @@ export const downloadTaskDetailSchema = z.object({ seeders: z.number().int().min(0).optional(), leechers: z.number().int().min(0).optional(), peers: z.number().int().min(0).optional(), - peerUploadedBytes: z.number().int().min(0).optional(), - peerUploadBps: z.number().int().min(0).optional(), + peerUploadedBytes: int64Schema().optional(), + peerUploadBps: int64Schema().optional(), trackers: z.array(downloadTaskTrackerSchema).max(20).optional(), peerSamples: z.array(downloadTaskPeerSchema).max(20).optional(), files: z.array(downloadTaskFileSchema).max(50).optional(), }) +export const downloadTaskSchema = z.object({ + id: z.string(), + sourceType: downloadSourceTypeSchema, + sourceUri: z.string(), + name: z.string(), + targetFolder: z.string(), + category: z.string().nullable(), + tags: z.array(z.string()), + status: downloadTaskStatusSchema, + downloadedBytes: int64Schema(), + storageUploadedBytes: int64Schema(), + totalBytes: nullableInt64Schema(), + downloadBps: int64Schema(), + storageUploadBps: int64Schema(), + errorMessage: z.string().nullable().optional(), + resultObjectId: z.string().nullable().optional(), + detail: downloadTaskDetailSchema.nullable().optional(), + uploadToken: z.string().optional(), + assignedDownloaderId: z.string().nullable().optional(), +}) + +export const downloadTaskPageSchema = z.object({ + items: z.array(downloadTaskSchema), + total: z.number().int(), + page: z.number().int(), + pageSize: z.number().int(), +}) + export const downloaderHeartbeatSchema = z.object({ version: z.string().min(1).max(80), hostname: z.string().min(1).max(160), @@ -71,9 +108,46 @@ export const downloaderHeartbeatSchema = z.object({ capabilities: z.array(z.string().min(1).max(80)).max(32), maxConcurrentTasks: z.number().int().min(1).max(100), currentTasks: z.number().int().min(0).max(100), - downloadBps: z.number().int().min(0).default(0), - uploadBps: z.number().int().min(0).default(0), - freeDiskBytes: z.number().int().min(0).default(0), + downloadBps: int64Schema().default(0), + uploadBps: int64Schema().default(0), + freeDiskBytes: int64Schema().default(0), +}) + +export const downloaderHeartbeatResponseSchema = z.object({ + version: z.string(), + hostname: z.string(), + platform: z.string(), + arch: z.string(), + engine: downloaderEngineSchema, + capabilities: z.array(z.string()), + maxConcurrentTasks: z.number().int(), + currentTasks: z.number().int(), + downloadBps: int64Schema(), + uploadBps: int64Schema(), + freeDiskBytes: int64Schema(), +}) + +export const downloaderSchema = z.object({ + id: z.string(), + name: z.string().optional(), + status: downloaderStatusSchema.optional(), + enabled: z.boolean().optional(), + heartbeat: downloaderHeartbeatResponseSchema.optional(), +}) + +export const downloaderListSchema = z.object({ + items: z.array(downloaderSchema), + total: z.number().int(), +}) + +export const createDownloaderResponseSchema = z.object({ + downloader: downloaderSchema, + token: z.string(), +}) + +export const deleteDownloaderResponseSchema = z.object({ + id: z.string(), + deleted: z.boolean(), }) export const updateDownloaderSchema = z.object({ @@ -118,11 +192,11 @@ export const createDownloadTaskSchema = z.object({ export const updateDownloadTaskSchema = z.object({ status: downloadTaskStatusSchema.optional(), - downloadedBytes: z.number().int().min(0).optional(), - storageUploadedBytes: z.number().int().min(0).optional(), - totalBytes: z.number().int().min(0).nullable().optional(), - downloadBps: z.number().int().min(0).optional(), - storageUploadBps: z.number().int().min(0).optional(), + downloadedBytes: int64Schema().optional(), + storageUploadedBytes: int64Schema().optional(), + totalBytes: nullableInt64Schema().optional(), + downloadBps: int64Schema().optional(), + storageUploadBps: int64Schema().optional(), errorMessage: z.string().max(1000).nullable().optional(), resultObjectId: z.string().min(1).nullable().optional(), detail: downloadTaskDetailSchema.nullable().optional(), @@ -185,6 +259,7 @@ export type UpdateDownloadTaskInput = z.infer export type DownloadTaskActionInput = z.infer export type ListDownloadTasksQuery = z.infer export type DownloadTaskDetail = z.infer +export type DownloadTaskSchema = z.infer export type CreateObjectUploadSessionInput = z.infer export type PresignObjectUploadPartsInput = z.infer export type PatchObjectUploadSessionInput = z.infer diff --git a/shared/schemas/index.ts b/shared/schemas/index.ts index 33a14979..dbc17f91 100644 --- a/shared/schemas/index.ts +++ b/shared/schemas/index.ts @@ -78,6 +78,7 @@ export type { DownloaderHeartbeatInput, DownloadTaskActionInput, DownloadTaskDetail, + DownloadTaskSchema, ListDownloadTasksQuery, PatchObjectUploadSessionInput, PresignObjectUploadPartsInput, @@ -85,16 +86,23 @@ export type { UpdateDownloadTaskInput, } from './downloads' export { + createDownloaderResponseSchema, createDownloaderSchema, createDownloadTaskSchema, createObjectUploadSessionSchema, + deleteDownloaderResponseSchema, downloaderEngineSchema, + downloaderHeartbeatResponseSchema, downloaderHeartbeatSchema, + downloaderListSchema, + downloaderSchema, downloaderStatusSchema, downloadSourceTypeSchema, downloadTaskActionInputSchema, downloadTaskActionSchema, downloadTaskDetailSchema, + downloadTaskPageSchema, + downloadTaskSchema, downloadTaskStatusSchema, listDownloadTasksQuerySchema, patchObjectUploadSessionSchema, @@ -132,9 +140,9 @@ export type ConflictStrategy = z.infer export const createMatterSchema = z.object({ name: z.string().min(1), type: z.string().min(1), - size: z.number().optional(), + size: z.number().int().min(0).optional(), parent: z.string().default(''), - dirtype: z.number().default(0), + dirtype: z.number().int().default(0), onConflict: conflictStrategySchema.optional(), }) @@ -154,6 +162,35 @@ export const confirmMatterSchema = z.object({ onConflict: conflictStrategySchema.optional(), }) +export const objectDraftSchema = z.object({ + id: z.string(), + name: z.string(), + uploadUrl: z.string().optional(), + contentDisposition: z.string().optional(), +}) + +export const objectUploadSessionSchema = z.object({ + id: z.string(), + objectId: z.string(), + uploadId: z.string(), + partSize: z.number().int(), + status: z.enum(['active', 'completed', 'aborted']), + expiresAt: z.string(), + createdAt: z.string(), + updatedAt: z.string(), +}) + +export const presignedObjectUploadPartSchema = z.object({ + partNumber: z.number().int(), + url: z.string(), +}) + +export const presignObjectUploadPartsResponseSchema = z.object({ + uploadId: z.string(), + partSize: z.number().int(), + parts: z.array(presignedObjectUploadPartSchema), +}) + export const trashMatterSchema = z.object({ action: z.literal('trash'), })