opensource/zpan
3
0
Fork 0
mirror of https://github.com/saltbo/zpan.git synced 2026-08-30 17:50:07 +08:00
Code Issues Packages Projects Releases Wiki Activity

refactor(downloader): consolidate api contract generation

Browse Source
This commit is contained in:
saltbo
2026-06-05 15:24:56 -04:00
parent 0b3bc2da09
commit 3f7a859e73
33 changed files with 4310 additions and 3222 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/ci.yml
+5
View File
@@ -49,9 +49,14 @@ jobs:
with:
node-version: 24
cache: pnpm
- uses: actions/setup-go@v6
with:
go-version-file: downloader/go.mod
cache-dependency-path: downloader/go.sum
- run: pnpm install --frozen-lockfile
- run: pnpm lint
- run: pnpm typecheck
- run: pnpm openapi:downloader:check
- run: pnpm exec vitest run --project unit --coverage
- uses: codecov/codecov-action@v5
if: always()
.github/workflows/release.yml
+5
View File
@@ -18,9 +18,14 @@ jobs:
with:
node-version: 24
cache: pnpm
- uses: actions/setup-go@v6
with:
go-version-file: downloader/go.mod
cache-dependency-path: downloader/go.sum
- run: pnpm install --frozen-lockfile
- run: pnpm lint
- run: pnpm typecheck
- run: pnpm openapi:downloader:check
- run: pnpm test
- run: mkdir -p dist # required by vitest cloudflare config
- run: pnpm test:cf
docs/openapi/downloader.json
+323 -276
View File
@@ -104,215 +104,6 @@
"device_code",
"client_id"
]
},
"ObjectDraft": {
"type": "object",
"properties": {
"id": {
"type": "string"
},
"name": {
"type": "string"
},
"uploadUrl": {
"type": "string"
},
"contentDisposition": {
"type": "string"
}
},
"required": [
"id",
"name"
]
},
"ConfirmObjectRequest": {
"type": "object",
"properties": {
"action": {
"type": "string",
"enum": [
"confirm"
]
},
"onConflict": {
"type": "string",
"enum": [
"fail",
"rename"
]
}
},
"required": [
"action"
]
},
"ObjectUploadSession": {
"type": "object",
"properties": {
"id": {
"type": "string"
},
"objectId": {
"type": "string"
},
"uploadId": {
"type": "string"
},
"partSize": {
"type": "integer"
},
"status": {
"type": "string",
"enum": [
"active",
"completed",
"aborted"
]
},
"expiresAt": {
"type": "string"
},
"createdAt": {
"type": "string"
},
"updatedAt": {
"type": "string"
}
},
"required": [
"id",
"objectId",
"uploadId",
"partSize",
"status",
"expiresAt",
"createdAt",
"updatedAt"
]
},
"CreateObjectUploadSessionRequest": {
"type": "object",
"properties": {
"partSize": {
"type": "integer",
"minimum": 5242880,
"maximum": 536870912
}
}
},
"PresignObjectUploadPartsResponse": {
"type": "object",
"properties": {
"uploadId": {
"type": "string"
},
"partSize": {
"type": "integer"
},
"parts": {
"type": "array",
"items": {
"$ref": "#/components/schemas/PresignedObjectUploadPart"
}
}
},
"required": [
"uploadId",
"partSize",
"parts"
]
},
"PresignedObjectUploadPart": {
"type": "object",
"properties": {
"partNumber": {
"type": "integer"
},
"url": {
"type": "string"
}
},
"required": [
"partNumber",
"url"
]
},
"PresignObjectUploadPartsRequest": {
"type": "object",
"properties": {
"partNumbers": {
"type": "array",
"items": {
"type": "integer",
"minimum": 1,
"maximum": 10000
},
"minItems": 1,
"maxItems": 100
}
},
"required": [
"partNumbers"
]
},
"PatchObjectUploadSessionRequest": {
"anyOf": [
{
"$ref": "#/components/schemas/CompleteObjectUploadSessionRequest"
},
{
"$ref": "#/components/schemas/AbortObjectUploadSessionRequest"
}
]
},
"CompleteObjectUploadSessionRequest": {
"type": "object",
"properties": {
"action": {
"type": "string",
"enum": [
"complete"
]
},
"parts": {
"type": "array",
"items": {
"type": "object",
"properties": {
"partNumber": {
"type": "integer"
},
"etag": {
"type": "string"
}
},
"required": [
"partNumber",
"etag"
]
},
"minItems": 1
}
},
"required": [
"action",
"parts"
]
},
"AbortObjectUploadSessionRequest": {
"type": "object",
"properties": {
"action": {
"type": "string",
"enum": [
"abort"
]
}
},
"required": [
"action"
]
}
},
"parameters": {}
@@ -638,11 +429,11 @@
},
"peerUploadedBytes": {
"type": "integer",
"minimum": 0
"format": "int64"
},
"peerUploadBps": {
"type": "integer",
"minimum": 0
"format": "int64"
},
"trackers": {
"type": "array",
@@ -700,11 +491,11 @@
},
"downloadBps": {
"type": "integer",
"minimum": 0
"format": "int64"
},
"uploadBps": {
"type": "integer",
"minimum": 0
"format": "int64"
}
},
"required": [
@@ -724,11 +515,11 @@
},
"size": {
"type": "integer",
"minimum": 0
"format": "int64"
},
"completedBytes": {
"type": "integer",
"minimum": 0
"format": "int64"
},
"selected": {
"type": "boolean"
@@ -839,7 +630,8 @@
]
},
"targetFolder": {
"type": "string"
"type": "string",
"maxLength": 1024
},
"name": {
"type": "string",
@@ -1014,11 +806,11 @@
},
"peerUploadedBytes": {
"type": "integer",
"minimum": 0
"format": "int64"
},
"peerUploadBps": {
"type": "integer",
"minimum": 0
"format": "int64"
},
"trackers": {
"type": "array",
@@ -1076,11 +868,11 @@
},
"downloadBps": {
"type": "integer",
"minimum": 0
"format": "int64"
},
"uploadBps": {
"type": "integer",
"minimum": 0
"format": "int64"
}
},
"required": [
@@ -1100,11 +892,11 @@
},
"size": {
"type": "integer",
"minimum": 0
"format": "int64"
},
"completedBytes": {
"type": "integer",
"minimum": 0
"format": "int64"
},
"selected": {
"type": "boolean"
@@ -1500,11 +1292,11 @@
},
"peerUploadedBytes": {
"type": "integer",
"minimum": 0
"format": "int64"
},
"peerUploadBps": {
"type": "integer",
"minimum": 0
"format": "int64"
},
"trackers": {
"type": "array",
@@ -1562,11 +1354,11 @@
},
"downloadBps": {
"type": "integer",
"minimum": 0
"format": "int64"
},
"uploadBps": {
"type": "integer",
"minimum": 0
"format": "int64"
}
},
"required": [
@@ -1586,11 +1378,11 @@
},
"size": {
"type": "integer",
"minimum": 0
"format": "int64"
},
"completedBytes": {
"type": "integer",
"minimum": 0
"format": "int64"
},
"selected": {
"type": "boolean"
@@ -1688,24 +1480,24 @@
},
"downloadedBytes": {
"type": "integer",
"minimum": 0
"format": "int64"
},
"storageUploadedBytes": {
"type": "integer",
"minimum": 0
"format": "int64"
},
"totalBytes": {
"type": "integer",
"nullable": true,
"minimum": 0
"format": "int64",
"nullable": true
},
"downloadBps": {
"type": "integer",
"minimum": 0
"format": "int64"
},
"storageUploadBps": {
"type": "integer",
"minimum": 0
"format": "int64"
},
"errorMessage": {
"type": "string",
@@ -1779,11 +1571,11 @@
},
"peerUploadedBytes": {
"type": "integer",
"minimum": 0
"format": "int64"
},
"peerUploadBps": {
"type": "integer",
"minimum": 0
"format": "int64"
},
"trackers": {
"type": "array",
@@ -1841,11 +1633,11 @@
},
"downloadBps": {
"type": "integer",
"minimum": 0
"format": "int64"
},
"uploadBps": {
"type": "integer",
"minimum": 0
"format": "int64"
}
},
"required": [
@@ -1865,11 +1657,11 @@
},
"size": {
"type": "integer",
"minimum": 0
"format": "int64"
},
"completedBytes": {
"type": "integer",
"minimum": 0
"format": "int64"
},
"selected": {
"type": "boolean"
@@ -2034,11 +1826,11 @@
},
"peerUploadedBytes": {
"type": "integer",
"minimum": 0
"format": "int64"
},
"peerUploadBps": {
"type": "integer",
"minimum": 0
"format": "int64"
},
"trackers": {
"type": "array",
@@ -2096,11 +1888,11 @@
},
"downloadBps": {
"type": "integer",
"minimum": 0
"format": "int64"
},
"uploadBps": {
"type": "integer",
"minimum": 0
"format": "int64"
}
},
"required": [
@@ -2120,11 +1912,11 @@
},
"size": {
"type": "integer",
"minimum": 0
"format": "int64"
},
"completedBytes": {
"type": "integer",
"minimum": 0
"format": "int64"
},
"selected": {
"type": "boolean"
@@ -2425,11 +2217,11 @@
},
"peerUploadedBytes": {
"type": "integer",
"minimum": 0
"format": "int64"
},
"peerUploadBps": {
"type": "integer",
"minimum": 0
"format": "int64"
},
"trackers": {
"type": "array",
@@ -2487,11 +2279,11 @@
},
"downloadBps": {
"type": "integer",
"minimum": 0
"format": "int64"
},
"uploadBps": {
"type": "integer",
"minimum": 0
"format": "int64"
}
},
"required": [
@@ -2511,11 +2303,11 @@
},
"size": {
"type": "integer",
"minimum": 0
"format": "int64"
},
"completedBytes": {
"type": "integer",
"minimum": 0
"format": "int64"
},
"selected": {
"type": "boolean"
@@ -2710,18 +2502,15 @@
},
"downloadBps": {
"type": "integer",
"minimum": 0,
"default": 0
"format": "int64"
},
"uploadBps": {
"type": "integer",
"minimum": 0,
"default": 0
"format": "int64"
},
"freeDiskBytes": {
"type": "integer",
"minimum": 0,
"default": 0
"format": "int64"
}
},
"required": [
@@ -3051,18 +2840,15 @@
},
"downloadBps": {
"type": "integer",
"minimum": 0,
"default": 0
"format": "int64"
},
"uploadBps": {
"type": "integer",
"minimum": 0,
"default": 0
"format": "int64"
},
"freeDiskBytes": {
"type": "integer",
"minimum": 0,
"default": 0
"format": "int64"
}
},
"required": [
@@ -3446,14 +3232,15 @@
"minLength": 1
},
"size": {
"type": "number"
"type": "integer",
"minimum": 0
},
"parent": {
"type": "string",
"default": ""
},
"dirtype": {
"type": "number",
"type": "integer",
"default": 0
},
"onConflict": {
@@ -3479,7 +3266,25 @@
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/ObjectDraft"
"type": "object",
"properties": {
"id": {
"type": "string"
},
"name": {
"type": "string"
},
"uploadUrl": {
"type": "string"
},
"contentDisposition": {
"type": "string"
}
},
"required": [
"id",
"name"
]
}
}
}
@@ -3489,7 +3294,25 @@
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/ObjectDraft"
"type": "object",
"properties": {
"id": {
"type": "string"
},
"name": {
"type": "string"
},
"uploadUrl": {
"type": "string"
},
"contentDisposition": {
"type": "string"
}
},
"required": [
"id",
"name"
]
}
}
}
@@ -3534,7 +3357,26 @@
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/ConfirmObjectRequest"
"type": "object",
"properties": {
"action": {
"type": "string",
"enum": [
"confirm"
]
},
"onConflict": {
"type": "string",
"enum": [
"fail",
"rename",
"replace"
]
}
},
"required": [
"action"
]
}
}
}
@@ -3545,7 +3387,25 @@
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/ObjectDraft"
"type": "object",
"properties": {
"id": {
"type": "string"
},
"name": {
"type": "string"
},
"uploadUrl": {
"type": "string"
},
"contentDisposition": {
"type": "string"
}
},
"required": [
"id",
"name"
]
}
}
}
@@ -3590,7 +3450,14 @@
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/CreateObjectUploadSessionRequest"
"type": "object",
"properties": {
"partSize": {
"type": "integer",
"minimum": 5242880,
"maximum": 536870912
}
}
}
}
}
@@ -3601,7 +3468,48 @@
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/ObjectUploadSession"
"type": "object",
"properties": {
"id": {
"type": "string"
},
"objectId": {
"type": "string"
},
"uploadId": {
"type": "string"
},
"partSize": {
"type": "integer"
},
"status": {
"type": "string",
"enum": [
"active",
"completed",
"aborted"
]
},
"expiresAt": {
"type": "string"
},
"createdAt": {
"type": "string"
},
"updatedAt": {
"type": "string"
}
},
"required": [
"id",
"objectId",
"uploadId",
"partSize",
"status",
"expiresAt",
"createdAt",
"updatedAt"
]
}
}
}
@@ -3664,7 +3572,22 @@
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/PresignObjectUploadPartsRequest"
"type": "object",
"properties": {
"partNumbers": {
"type": "array",
"items": {
"type": "integer",
"minimum": 1,
"maximum": 10000
},
"minItems": 1,
"maxItems": 100
}
},
"required": [
"partNumbers"
]
}
}
}
@@ -3675,7 +3598,38 @@
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/PresignObjectUploadPartsResponse"
"type": "object",
"properties": {
"uploadId": {
"type": "string"
},
"partSize": {
"type": "integer"
},
"parts": {
"type": "array",
"items": {
"type": "object",
"properties": {
"partNumber": {
"type": "integer"
},
"url": {
"type": "string"
}
},
"required": [
"partNumber",
"url"
]
}
}
},
"required": [
"uploadId",
"partSize",
"parts"
]
}
}
}
@@ -3738,7 +3692,59 @@
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/PatchObjectUploadSessionRequest"
"oneOf": [
{
"type": "object",
"properties": {
"action": {
"type": "string",
"enum": [
"complete"
]
},
"parts": {
"type": "array",
"items": {
"type": "object",
"properties": {
"partNumber": {
"type": "integer",
"minimum": 1,
"maximum": 10000
},
"etag": {
"type": "string",
"minLength": 1
}
},
"required": [
"partNumber",
"etag"
]
},
"minItems": 1
}
},
"required": [
"action",
"parts"
]
},
{
"type": "object",
"properties": {
"action": {
"type": "string",
"enum": [
"abort"
]
}
},
"required": [
"action"
]
}
]
}
}
}
@@ -3749,7 +3755,48 @@
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/ObjectUploadSession"
"type": "object",
"properties": {
"id": {
"type": "string"
},
"objectId": {
"type": "string"
},
"uploadId": {
"type": "string"
},
"partSize": {
"type": "integer"
},
"status": {
"type": "string",
"enum": [
"active",
"completed",
"aborted"
]
},
"expiresAt": {
"type": "string"
},
"createdAt": {
"type": "string"
},
"updatedAt": {
"type": "string"
}
},
"required": [
"id",
"objectId",
"uploadId",
"partSize",
"status",
"expiresAt",
"createdAt",
"updatedAt"
]
}
}
}
downloader/cmd/zpan-downloader/main.go
+9 -2
View File
@@ -90,7 +90,11 @@ func runCommand(v *viper.Viper) *cobra.Command {
)
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
defer stop()
return worker.New(cfg).Run(ctx)
downloader, err := worker.New(cfg)
if err != nil {
return err
}
return downloader.Run(ctx)
},
}
}
@@ -123,7 +127,10 @@ func loginCommand(v *viper.Viper, cfgFile *string) *cobra.Command {
}
}
api := client.New(serverURL, "")
api, err := client.New(serverURL, "")
if err != nil {
return err
}
ctx := cmd.Context()
code, err := api.RequestDeviceCode(ctx)
if err != nil {
downloader/internal/client/client.go
+218 -75
View File
@@ -171,25 +171,21 @@ type CreateDownloaderResponse struct {
Token string `json:"token"`
}
func New(baseURL, token string) *Client {
func New(baseURL, token string) (*Client, error) {
httpClient := &http.Client{Timeout: 60 * time.Second}
api, err := openapi.NewClientWithResponses(strings.TrimRight(baseURL, "/"), openapi.WithHTTPClient(httpClient))
if err != nil {
panic(err)
return nil, err
}
return &Client{
baseURL: baseURL,
token: token,
api: api,
}
}, nil
}
func (c *Client) Heartbeat(ctx context.Context, heartbeat Heartbeat) error {
body, err := jsonBody(heartbeat)
if err != nil {
return err
}
res, err := c.api.PostApiDownloaderHeartbeatWithBodyWithResponse(ctx, "application/json", body, bearer(c.token))
res, err := c.api.PostApiDownloaderHeartbeatWithResponse(ctx, heartbeatRequestBody(heartbeat), bearer(c.token))
if err != nil {
return err
}
@@ -199,6 +195,8 @@ func (c *Client) Heartbeat(ctx context.Context, heartbeat Heartbeat) error {
func (c *Client) AssignedTasks(ctx context.Context) ([]DownloadTask, error) {
return c.assignedTasks(ctx, []openapi.GetApiDownloadTasksParamsStatus{
openapi.GetApiDownloadTasksParamsStatusAssigned,
openapi.GetApiDownloadTasksParamsStatusRunning,
openapi.GetApiDownloadTasksParamsStatusUploading,
})
}
@@ -227,11 +225,16 @@ func (c *Client) assignedTasks(ctx context.Context, statuses []openapi.GetApiDow
if err := expectStatus("GET", "/api/download-tasks", res.StatusCode(), res.Body, http.StatusOK); err != nil {
return nil, err
}
var result Page[DownloadTask]
if err := decodeJSON(res.Body, &result); err != nil {
return nil, fmt.Errorf("GET /api/download-tasks failed: %w", err)
if res.JSON200 == nil {
return nil, fmt.Errorf("GET /api/download-tasks failed: empty response")
}
for _, item := range res.JSON200.Items {
task, err := downloadTaskFromOpenAPI(item)
if err != nil {
return nil, fmt.Errorf("GET /api/download-tasks failed: %w", err)
}
tasks = append(tasks, task)
}
tasks = append(tasks, result.Items...)
}
return tasks, nil
}
@@ -284,38 +287,160 @@ func (c *Client) PollDeviceToken(ctx context.Context, deviceCode string) (Device
}
func (c *Client) CreateDownloader(ctx context.Context, accessToken string, req CreateDownloaderRequest) (CreateDownloaderResponse, error) {
body, err := jsonBody(req)
if err != nil {
return CreateDownloaderResponse{}, err
}
res, err := c.api.PostApiAdminDownloadersWithBodyWithResponse(ctx, "application/json", body, bearer(accessToken))
res, err := c.api.PostApiAdminDownloadersWithResponse(ctx, createDownloaderRequestBody(req), bearer(accessToken))
if err != nil {
return CreateDownloaderResponse{}, err
}
if err := expectStatus("POST", "/api/admin/downloaders", res.StatusCode(), res.Body, http.StatusCreated); err != nil {
return CreateDownloaderResponse{}, err
}
var out CreateDownloaderResponse
if err := decodeJSON(res.Body, &out); err != nil {
return CreateDownloaderResponse{}, fmt.Errorf("POST /api/admin/downloaders failed: %w", err)
if res.JSON201 == nil {
return CreateDownloaderResponse{}, fmt.Errorf("POST /api/admin/downloaders failed: empty response")
}
out := CreateDownloaderResponse{Token: res.JSON201.Token}
out.Downloader.ID = res.JSON201.Downloader.Id
return out, nil
}
func (c *Client) UpdateTask(ctx context.Context, id string, patch TaskPatch) (DownloadTask, error) {
body, err := jsonBody(patch)
func heartbeatRequestBody(heartbeat Heartbeat) openapi.PostApiDownloaderHeartbeatJSONRequestBody {
return openapi.PostApiDownloaderHeartbeatJSONRequestBody{
Arch: heartbeat.Arch,
Capabilities: heartbeat.Capabilities,
CurrentTasks: heartbeat.CurrentTasks,
DownloadBps: &heartbeat.DownloadBps,
Engine: openapi.PostApiDownloaderHeartbeatJSONBodyEngine(heartbeat.Engine),
FreeDiskBytes: &heartbeat.FreeDiskBytes,
Hostname: heartbeat.Hostname,
MaxConcurrentTasks: heartbeat.MaxConcurrentTasks,
Platform: heartbeat.Platform,
UploadBps: &heartbeat.UploadBps,
Version: heartbeat.Version,
}
}
func createDownloaderRequestBody(req CreateDownloaderRequest) openapi.PostApiAdminDownloadersJSONRequestBody {
return openapi.PostApiAdminDownloadersJSONRequestBody{
Name: req.Name,
Heartbeat: struct {
Arch string `json:"arch"`
Capabilities []string `json:"capabilities"`
CurrentTasks int `json:"currentTasks"`
DownloadBps *int64 `json:"downloadBps,omitempty"`
Engine openapi.PostApiAdminDownloadersJSONBodyHeartbeatEngine `json:"engine"`
FreeDiskBytes *int64 `json:"freeDiskBytes,omitempty"`
Hostname string `json:"hostname"`
MaxConcurrentTasks int `json:"maxConcurrentTasks"`
Platform string `json:"platform"`
UploadBps *int64 `json:"uploadBps,omitempty"`
Version string `json:"version"`
}{
Arch: req.Heartbeat.Arch,
Capabilities: req.Heartbeat.Capabilities,
CurrentTasks: req.Heartbeat.CurrentTasks,
DownloadBps: &req.Heartbeat.DownloadBps,
Engine: openapi.PostApiAdminDownloadersJSONBodyHeartbeatEngine(req.Heartbeat.Engine),
FreeDiskBytes: &req.Heartbeat.FreeDiskBytes,
Hostname: req.Heartbeat.Hostname,
MaxConcurrentTasks: req.Heartbeat.MaxConcurrentTasks,
Platform: req.Heartbeat.Platform,
UploadBps: &req.Heartbeat.UploadBps,
Version: req.Heartbeat.Version,
},
}
}
func taskPatchRequestBody(patch TaskPatch) (openapi.PatchApiDownloadTasksIdJSONRequestBody, error) {
body := openapi.PatchApiDownloadTasksIdJSONRequestBody{
DownloadBps: patch.DownloadBps,
DownloadedBytes: patch.DownloadedBytes,
ErrorMessage: patch.ErrorMessage,
ResultObjectId: patch.ResultObjectID,
StorageUploadBps: patch.StorageUploadBps,
StorageUploadedBytes: patch.StorageUploadedBytes,
TotalBytes: patch.TotalBytes,
}
if patch.Status != "" {
status := openapi.PatchApiDownloadTasksIdJSONBodyStatus(patch.Status)
body.Status = &status
}
if patch.Detail != nil {
data, err := json.Marshal(patch.Detail)
if err != nil {
return body, err
}
var detail struct {
Connections *int `json:"connections,omitempty"`
Engine *openapi.PatchApiDownloadTasksIdJSONBodyDetailEngine `json:"engine,omitempty"`
EngineState *string `json:"engineState,omitempty"`
EtaSeconds *int `json:"etaSeconds,omitempty"`
Files *[]struct {
CompletedBytes *int64 `json:"completedBytes,omitempty"`
Path string `json:"path"`
Selected *bool `json:"selected,omitempty"`
Size int64 `json:"size"`
} `json:"files,omitempty"`
InfoHash *string `json:"infoHash,omitempty"`
Leechers *int `json:"leechers,omitempty"`
Message *string `json:"message,omitempty"`
PeerSamples *[]struct {
Address string `json:"address"`
Client *string `json:"client,omitempty"`
DownloadBps *int64 `json:"downloadBps,omitempty"`
Progress *float32 `json:"progress,omitempty"`
UploadBps *int64 `json:"uploadBps,omitempty"`
} `json:"peerSamples,omitempty"`
PeerUploadBps *int64 `json:"peerUploadBps,omitempty"`
PeerUploadedBytes *int64 `json:"peerUploadedBytes,omitempty"`
Peers *int `json:"peers,omitempty"`
Phase *openapi.PatchApiDownloadTasksIdJSONBodyDetailPhase `json:"phase,omitempty"`
Seeders *int `json:"seeders,omitempty"`
TorrentName *string `json:"torrentName,omitempty"`
Trackers *[]struct {
Leechers *int `json:"leechers,omitempty"`
Message *string `json:"message,omitempty"`
Peers *int `json:"peers,omitempty"`
Seeds *int `json:"seeds,omitempty"`
Status *string `json:"status,omitempty"`
Url string `json:"url"`
} `json:"trackers,omitempty"`
}
if err := json.Unmarshal(data, &detail); err != nil {
return body, err
}
body.Detail = &detail
}
return body, nil
}
func downloadTaskFromOpenAPI(value any) (DownloadTask, error) {
data, err := json.Marshal(value)
if err != nil {
return DownloadTask{}, err
}
res, err := c.api.PatchApiDownloadTasksIdWithBodyWithResponse(ctx, id, "application/json", body, bearer(c.token))
var task DownloadTask
if err := json.Unmarshal(data, &task); err != nil {
return DownloadTask{}, err
}
return task, nil
}
func (c *Client) UpdateTask(ctx context.Context, id string, patch TaskPatch) (DownloadTask, error) {
body, err := taskPatchRequestBody(patch)
if err != nil {
return DownloadTask{}, err
}
res, err := c.api.PatchApiDownloadTasksIdWithResponse(ctx, id, body, bearer(c.token))
if err != nil {
return DownloadTask{}, err
}
if err := expectStatus("PATCH", "/api/download-tasks/"+id, res.StatusCode(), res.Body, http.StatusOK); err != nil {
return DownloadTask{}, err
}
var task DownloadTask
if err := decodeJSON(res.Body, &task); err != nil {
if res.JSON200 == nil {
return DownloadTask{}, fmt.Errorf("PATCH /api/download-tasks/%s failed: empty response", id)
}
task, err := downloadTaskFromOpenAPI(*res.JSON200)
if err != nil {
return DownloadTask{}, fmt.Errorf("PATCH /api/download-tasks/%s failed: %w", id, err)
}
return task, nil
@@ -338,47 +463,47 @@ func (c *Client) createMatter(
parent string,
dirtype int,
) (ObjectDraft, error) {
body, err := jsonBody(struct {
Name string `json:"name"`
Type string `json:"type"`
Size int64 `json:"size"`
Parent string `json:"parent"`
Dirtype int `json:"dirtype"`
OnConflict string `json:"onConflict"`
}{
sizeInt := int(size)
onConflict := openapi.PostApiObjectsJSONBodyOnConflictRename
res, err := c.api.PostApiObjectsWithResponse(ctx, openapi.PostApiObjectsJSONRequestBody{
Name: name,
Type: contentType,
Size: size,
Parent: parent,
Dirtype: dirtype,
OnConflict: "rename",
})
if err != nil {
return ObjectDraft{}, err
}
res, err := c.api.PostApiObjectsWithBodyWithResponse(ctx, "application/json", body, bearer(token))
Size: &sizeInt,
Parent: &parent,
Dirtype: &dirtype,
OnConflict: &onConflict,
}, bearer(token))
if err != nil {
return ObjectDraft{}, err
}
if err := expectStatus("POST", "/api/objects", res.StatusCode(), res.Body, http.StatusOK, http.StatusCreated); err != nil {
return ObjectDraft{}, err
}
var draft ObjectDraft
if err := decodeJSON(res.Body, &draft); err != nil {
return ObjectDraft{}, fmt.Errorf("POST /api/objects failed: %w", err)
if res.JSON200 != nil {
return ObjectDraft{
ID: res.JSON200.Id,
Name: res.JSON200.Name,
UploadURL: derefString(res.JSON200.UploadUrl),
ContentDisposition: derefString(res.JSON200.ContentDisposition),
}, nil
}
return draft, nil
if res.JSON201 != nil {
return ObjectDraft{
ID: res.JSON201.Id,
Name: res.JSON201.Name,
UploadURL: derefString(res.JSON201.UploadUrl),
ContentDisposition: derefString(res.JSON201.ContentDisposition),
}, nil
}
return ObjectDraft{}, fmt.Errorf("POST /api/objects failed: empty response")
}
func (c *Client) ConfirmObject(ctx context.Context, token string, id string) error {
body, err := jsonBody(struct {
Action string `json:"action"`
OnConflict string `json:"onConflict"`
}{Action: "confirm", OnConflict: "rename"})
if err != nil {
return err
}
res, err := c.api.PatchApiObjectsIdWithBodyWithResponse(ctx, id, "application/json", body, bearer(token))
onConflict := openapi.PatchApiObjectsIdJSONBodyOnConflictRename
res, err := c.api.PatchApiObjectsIdWithResponse(ctx, id, openapi.PatchApiObjectsIdJSONRequestBody{
Action: openapi.Confirm,
OnConflict: &onConflict,
}, bearer(token))
if err != nil {
return err
}
@@ -386,13 +511,10 @@ func (c *Client) ConfirmObject(ctx context.Context, token string, id string) err
}
func (c *Client) CreateObjectUploadSession(ctx context.Context, token string, id string, partSize int64) (ObjectUploadSession, error) {
body, err := jsonBody(struct {
PartSize int64 `json:"partSize"`
}{PartSize: partSize})
if err != nil {
return ObjectUploadSession{}, err
}
res, err := c.api.PostApiObjectsIdUploadsWithBodyWithResponse(ctx, id, "application/json", body, bearer(token))
partSizeInt := int(partSize)
res, err := c.api.PostApiObjectsIdUploadsWithResponse(ctx, id, openapi.PostApiObjectsIdUploadsJSONRequestBody{
PartSize: &partSizeInt,
}, bearer(token))
if err != nil {
return ObjectUploadSession{}, err
}
@@ -411,13 +533,9 @@ func (c *Client) CreateObjectUploadSession(ctx context.Context, token string, id
}
func (c *Client) PresignObjectUploadParts(ctx context.Context, token string, id string, sessionID string, partNumbers []int) ([]PresignedObjectUploadPart, error) {
body, err := jsonBody(struct {
PartNumbers []int `json:"partNumbers"`
}{PartNumbers: partNumbers})
if err != nil {
return nil, err
}
res, err := c.api.PostApiObjectsIdUploadsUploadSessionIdPartsWithBodyWithResponse(ctx, id, sessionID, "application/json", body, bearer(token))
res, err := c.api.PostApiObjectsIdUploadsUploadSessionIdPartsWithResponse(ctx, id, sessionID, openapi.PostApiObjectsIdUploadsUploadSessionIdPartsJSONRequestBody{
PartNumbers: partNumbers,
}, bearer(token))
if err != nil {
return nil, err
}
@@ -436,10 +554,24 @@ func (c *Client) PresignObjectUploadParts(ctx context.Context, token string, id
}
func (c *Client) CompleteObjectUploadSession(ctx context.Context, token string, id string, sessionID string, parts []CompletedObjectUploadPart) error {
body, err := jsonBody(struct {
Action string `json:"action"`
Parts []CompletedObjectUploadPart `json:"parts"`
}{Action: "complete", Parts: parts})
complete := openapi.PatchApiObjectsIdUploadsUploadSessionIdJSONBody0{
Action: openapi.Complete,
Parts: make([]struct {
Etag string `json:"etag"`
PartNumber int `json:"partNumber"`
}, 0, len(parts)),
}
for _, part := range parts {
complete.Parts = append(complete.Parts, struct {
Etag string `json:"etag"`
PartNumber int `json:"partNumber"`
}{Etag: part.ETag, PartNumber: part.PartNumber})
}
var union openapi.PatchApiObjectsIdUploadsUploadSessionIdJSONBody
if err := union.FromPatchApiObjectsIdUploadsUploadSessionIdJSONBody0(complete); err != nil {
return err
}
body, err := jsonBody(union)
if err != nil {
return err
}
@@ -451,9 +583,13 @@ func (c *Client) CompleteObjectUploadSession(ctx context.Context, token string,
}
func (c *Client) AbortObjectUploadSession(ctx context.Context, token string, id string, sessionID string) error {
body, err := jsonBody(struct {
Action string `json:"action"`
}{Action: "abort"})
var union openapi.PatchApiObjectsIdUploadsUploadSessionIdJSONBody
if err := union.FromPatchApiObjectsIdUploadsUploadSessionIdJSONBody1(openapi.PatchApiObjectsIdUploadsUploadSessionIdJSONBody1{
Action: openapi.Abort,
}); err != nil {
return err
}
body, err := jsonBody(union)
if err != nil {
return err
}
@@ -472,6 +608,13 @@ func jsonBody(value any) (*bytes.Reader, error) {
return bytes.NewReader(data), nil
}
func derefString(value *string) string {
if value == nil {
return ""
}
return *value
}
func decodeJSON(data []byte, out any) error {
if len(data) == 0 {
return fmt.Errorf("empty response body")
downloader/internal/client/client_test.go
+98 -3
View File
@@ -6,6 +6,7 @@ import (
"net/http"
"net/http/httptest"
"reflect"
"sort"
"testing"
)
@@ -18,11 +19,12 @@ func TestCreateObjectUsesRenameConflictStrategy(t *testing.T) {
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
t.Fatal(err)
}
w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode(ObjectDraft{ID: "object-1", Name: "movie (1).mkv"})
}))
defer server.Close()
_, err := New(server.URL, "token").CreateObject(context.Background(), "upload-token", "movie.mkv", 1024, "Downloads")
_, err := mustClient(t, server.URL, "token").CreateObject(context.Background(), "upload-token", "movie.mkv", 1024, "Downloads")
if err != nil {
t.Fatal(err)
}
@@ -31,6 +33,90 @@ func TestCreateObjectUsesRenameConflictStrategy(t *testing.T) {
}
}
func TestAssignedTasksFetchesRecoverableStatuses(t *testing.T) {
var statuses []string
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path != "/api/download-tasks" {
t.Fatalf("unexpected path: %s", r.URL.Path)
}
w.Header().Set("Content-Type", "application/json")
status := r.URL.Query().Get("status")
statuses = append(statuses, status)
_ = json.NewEncoder(w).Encode(Page[DownloadTask]{
Items: []DownloadTask{{ID: "task-" + status, Status: status, UploadToken: "upload-token"}},
})
}))
defer server.Close()
tasks, err := mustClient(t, server.URL, "token").AssignedTasks(context.Background())
if err != nil {
t.Fatal(err)
}
sort.Strings(statuses)
expected := []string{"assigned", "running", "uploading"}
if !reflect.DeepEqual(statuses, expected) {
t.Fatalf("expected recoverable statuses %v, got %v", expected, statuses)
}
if len(tasks) != 3 {
t.Fatalf("expected three tasks, got %d", len(tasks))
}
}
func TestUpdateTaskUsesGeneratedRequestShape(t *testing.T) {
var body map[string]any
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodPatch || r.URL.Path != "/api/download-tasks/task-1" {
t.Fatalf("unexpected request: %s %s", r.Method, r.URL.Path)
}
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
t.Fatal(err)
}
w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode(DownloadTask{
ID: "task-1",
SourceType: "http",
SourceURI: "https://example.com/file.bin",
Name: "file.bin",
TargetFolder: "",
Tags: []string{},
Status: "running",
DownloadedBytes: 1024,
StorageUploadedBytes: 0,
DownloadBps: 10,
StorageUploadBps: 0,
})
}))
defer server.Close()
downloadedBytes := int64(1024)
totalBytes := int64(2048)
etaSeconds := int64(30)
task, err := mustClient(t, server.URL, "token").UpdateTask(context.Background(), "task-1", TaskPatch{
Status: "running",
DownloadedBytes: &downloadedBytes,
TotalBytes: &totalBytes,
Detail: &DownloadTaskDetail{
Engine: "builtin",
Phase: "downloading",
ETASeconds: &etaSeconds,
Files: []DownloadTaskFile{{Path: "file.bin", Size: 2048, CompletedBytes: &downloadedBytes}},
},
})
if err != nil {
t.Fatal(err)
}
if task.ID != "task-1" || task.Status != "running" {
t.Fatalf("unexpected task: %#v", task)
}
if body["status"] != "running" || body["downloadedBytes"] != float64(1024) || body["totalBytes"] != float64(2048) {
t.Fatalf("unexpected patch body: %#v", body)
}
detail, ok := body["detail"].(map[string]any)
if !ok || detail["engine"] != "builtin" || detail["phase"] != "downloading" || detail["etaSeconds"] != float64(30) {
t.Fatalf("unexpected detail body: %#v", body["detail"])
}
}
func TestConfirmObjectUsesRenameConflictStrategy(t *testing.T) {
var body map[string]any
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
@@ -44,7 +130,7 @@ func TestConfirmObjectUsesRenameConflictStrategy(t *testing.T) {
}))
defer server.Close()
if err := New(server.URL, "token").ConfirmObject(context.Background(), "upload-token", "object-1"); err != nil {
if err := mustClient(t, server.URL, "token").ConfirmObject(context.Background(), "upload-token", "object-1"); err != nil {
t.Fatal(err)
}
if body["action"] != "confirm" {
@@ -116,7 +202,7 @@ func TestMultipartUploadSessionClientMethods(t *testing.T) {
}))
defer server.Close()
api := New(server.URL, "token")
api := mustClient(t, server.URL, "token")
session, err := api.CreateObjectUploadSession(context.Background(), "upload-token", "object-1", 64*1024*1024)
if err != nil {
t.Fatal(err)
@@ -150,3 +236,12 @@ func TestMultipartUploadSessionClientMethods(t *testing.T) {
t.Fatalf("unexpected patch bodies: complete=%#v abort=%#v", completeBody, abortBody)
}
}
func mustClient(t *testing.T, baseURL string, token string) *Client {
t.Helper()
api, err := New(baseURL, token)
if err != nil {
t.Fatal(err)
}
return api
}
downloader/internal/engine/aria2.go
+584
View File
@@ -0,0 +1,584 @@
package engine
import (
"context"
"crypto/sha256"
"encoding/hex"
"errors"
"fmt"
"io"
"os"
"os/exec"
"path/filepath"
"strings"
"time"
"github.com/Braurbeki/arigo"
"github.com/cenkalti/rpc2"
"github.com/saltbo/zpan/downloader/internal/client"
)
type Aria2 struct {
URL string
Secret string
Dir string
RetainSeed bool
}
func (a Aria2) Name() string {
return "aria2"
}
func (a Aria2) Capabilities() []string {
return []string{"http", "magnet", "torrent"}
}
func (a Aria2) Start(ctx context.Context) (*exec.Cmd, error) {
path, err := exec.LookPath("aria2c")
if err != nil {
return nil, err
}
rpcURL, err := parseLocalEngineURL(a.URL, "6800")
if err != nil {
return nil, err
}
args := []string{
"--enable-rpc=true",
"--rpc-listen-all=false",
"--rpc-listen-port=" + rpcURL.port,
"--dir=" + a.Dir,
"--continue=true",
"--allow-overwrite=true",
"--auto-file-renaming=false",
}
if a.Secret != "" {
args = append(args, "--rpc-secret="+a.Secret)
}
cmd := exec.CommandContext(ctx, path, args...)
if err := cmd.Start(); err != nil {
return nil, err
}
go func() { _ = cmd.Wait() }()
return cmd, nil
}
func (a Aria2) Check(ctx context.Context) error {
client, err := a.client(ctx)
if err != nil {
return err
}
defer client.Close()
version, err := client.GetVersion()
if err != nil {
return err
}
if version.Version == "" {
return errors.New("aria2 rpc did not return a version")
}
return nil
}
func (a Aria2) Recover(ctx context.Context, task client.DownloadTask) (Result, bool, error) {
aria, err := a.client(ctx)
if err == nil {
defer aria.Close()
status, ok, findErr := a.findTask(ctx, &aria, task)
if findErr != nil {
return Result{}, false, findErr
}
if ok && string(status.Status) == string(arigo.StatusCompleted) {
files, err := a.getAria2Files(ctx, &aria, status.GID)
if err != nil {
return Result{}, false, err
}
result, err := resultFromAria2Files(task, filepath.Join(a.Dir, task.ID), status.BitTorrent.Info.Name, files)
return result, err == nil, err
}
}
return recoverFromTaskDir(task, a.Dir)
}
func (a Aria2) Download(ctx context.Context, task client.DownloadTask, progress Progress) (Result, error) {
taskDir := filepath.Join(a.Dir, task.ID)
if err := os.MkdirAll(taskDir, 0o755); err != nil {
return Result{}, err
}
aria, err := a.client(ctx)
if err != nil {
return Result{}, err
}
defer aria.Close()
status, ok, err := a.findTask(ctx, &aria, task)
if err != nil {
return Result{}, err
}
if ok {
if string(status.Status) == string(arigo.StatusPaused) {
_ = aria.Unpause(status.GID)
}
return a.waitResult(ctx, &aria, task, taskDir, status.GID, progress)
}
options := &arigo.Options{
Dir: taskDir,
GID: aria2TaskGID(task.ID),
FollowTorrent: true,
BTSaveMetadata: true,
Continue: true,
SeedRatio: 0,
SeedTime: 0,
AllowOverwrite: true,
AutoFileRenaming: false,
}
if a.RetainSeed && task.SourceType != "http" {
options.SeedTime = 1000000
}
if task.Name != "" && task.SourceType == "http" {
options.Out = task.Name
}
gid, err := aria.AddURI(arigo.URIs(task.SourceURI), options)
if err != nil {
status, ok, findErr := a.findTask(ctx, &aria, task)
if findErr != nil {
return Result{}, findErr
}
if !ok {
return Result{}, err
}
gid.GID = status.GID
}
return a.waitResult(ctx, &aria, task, taskDir, gid.GID, progress)
}
func (a Aria2) waitResult(ctx context.Context, aria **arigo.Client, task client.DownloadTask, taskDir string, gid string, progress Progress) (Result, error) {
initialProgress := progress
if task.SourceType != "http" {
initialProgress = func(downloaded int64, total *int64, bps int64, detail *client.DownloadTaskDetail) error { return nil }
}
status, err := a.waitAria2(ctx, aria, gid, initialProgress)
if err != nil {
_ = (*aria).Remove(gid)
return Result{}, err
}
if len(status.FollowedBy) > 0 {
childGID := status.FollowedBy[0]
status, err = a.waitAria2(ctx, aria, childGID, progress)
if err != nil {
_ = (*aria).Remove(childGID)
return Result{}, err
}
}
files, err := a.getAria2Files(ctx, aria, status.GID)
if err != nil {
return Result{}, err
}
result, err := resultFromAria2Files(task, taskDir, status.BitTorrent.Info.Name, files)
if err != nil {
return Result{}, err
}
if a.RetainSeed && task.SourceType != "http" {
result.Seed = &Seed{
Engine: "aria2",
ID: status.GID,
Path: taskDir,
Snapshot: a.seedSnapshot(status.GID),
Cleanup: a.cleanupSeed(status.GID, taskDir),
}
return result, nil
}
_ = (*aria).ForceRemove(status.GID)
_ = (*aria).RemoveDownloadResult(status.GID)
return result, nil
}
func (a Aria2) client(ctx context.Context) (*arigo.Client, error) {
return arigo.DialContext(ctx, a.URL, a.Secret)
}
func (a Aria2) findTask(ctx context.Context, aria **arigo.Client, task client.DownloadTask) (arigo.Status, bool, error) {
gid := aria2TaskGID(task.ID)
status, err := (*aria).TellStatus(gid)
if err == nil {
return status, true, nil
}
if isAria2RPCDisconnected(err) {
if err := a.reconnect(ctx, aria); err != nil {
return arigo.Status{}, false, err
}
status, err = (*aria).TellStatus(gid)
if err == nil {
return status, true, nil
}
}
statuses, err := a.taskStatuses(ctx, aria)
if err != nil {
return arigo.Status{}, false, err
}
taskDir := filepath.Clean(filepath.Join(a.Dir, task.ID))
for _, status := range statuses {
if aria2StatusMatchesTask(status, taskDir, gid) {
return status, true, nil
}
}
return arigo.Status{}, false, nil
}
func (a Aria2) taskStatuses(ctx context.Context, aria **arigo.Client) ([]arigo.Status, error) {
active, err := (*aria).TellActive()
if err != nil {
if !isAria2RPCDisconnected(err) {
return nil, err
}
if err := a.reconnect(ctx, aria); err != nil {
return nil, err
}
active, err = (*aria).TellActive()
if err != nil {
return nil, err
}
}
waiting, err := (*aria).TellWaiting(0, 1000)
if err != nil {
return nil, err
}
stopped, err := (*aria).TellStopped(0, 1000)
if err != nil {
return nil, err
}
statuses := make([]arigo.Status, 0, len(active)+len(waiting)+len(stopped))
statuses = append(statuses, active...)
statuses = append(statuses, waiting...)
statuses = append(statuses, stopped...)
return statuses, nil
}
func aria2TaskGID(taskID string) string {
sum := sha256.Sum256([]byte(taskID))
return hex.EncodeToString(sum[:])[:16]
}
func aria2StatusMatchesTask(status arigo.Status, taskDir string, gid string) bool {
if status.GID == gid || status.Following == gid || status.BelongsTo == gid {
return true
}
if filepath.Clean(status.Dir) == taskDir {
return true
}
for _, file := range status.Files {
if file.Path == "" {
continue
}
abs, _ := downloadedPath(taskDir, file.Path)
if strings.HasPrefix(filepath.Clean(abs), taskDir+string(filepath.Separator)) {
return true
}
}
return false
}
func (a Aria2) seedSnapshot(gid string) func(context.Context) (SeedSnapshot, error) {
return func(ctx context.Context) (SeedSnapshot, error) {
aria, err := a.client(ctx)
if err != nil {
return SeedSnapshot{}, err
}
defer aria.Close()
status, err := aria.TellStatus(gid)
if err != nil {
return SeedSnapshot{}, err
}
peers := a.getAria2Peers(ctx, &aria, gid)
total := int64(status.TotalLength)
var totalPtr *int64
if total > 0 {
totalPtr = &total
}
detail := aria2Detail(status, peers)
detail.Phase = "seeding"
return SeedSnapshot{
Downloaded: int64(status.CompletedLength),
Total: totalPtr,
Bps: int64(status.DownloadSpeed),
Detail: detail,
}, nil
}
}
func (a Aria2) cleanupSeed(gid string, localPath string) func(context.Context) error {
return func(ctx context.Context) error {
var errs []error
aria, err := a.client(ctx)
if err != nil {
errs = append(errs, err)
} else {
_ = aria.ForceRemove(gid)
_ = aria.RemoveDownloadResult(gid)
_ = aria.Close()
}
if err := os.RemoveAll(localPath); err != nil {
errs = append(errs, err)
}
return errors.Join(errs...)
}
}
func (a Aria2) waitAria2(ctx context.Context, aria **arigo.Client, gid string, progress Progress) (arigo.Status, error) {
ticker := time.NewTicker(time.Second)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
return arigo.Status{}, ctx.Err()
case <-ticker.C:
status, err := (*aria).TellStatus(gid)
if err != nil {
if isAria2RPCDisconnected(err) {
if err := a.reconnect(ctx, aria); err != nil {
return arigo.Status{}, err
}
continue
}
return arigo.Status{}, err
}
total := int64(status.TotalLength)
completed := int64(status.CompletedLength)
bps := int64(status.DownloadSpeed)
var totalPtr *int64
if total > 0 {
totalPtr = &total
}
peers := a.getAria2Peers(ctx, aria, gid)
if err := progress(completed, totalPtr, bps, aria2Detail(status, peers)); err != nil {
_ = (*aria).ForcePause(gid)
return arigo.Status{}, err
}
switch string(status.Status) {
case "complete", string(arigo.StatusCompleted):
if len(status.FollowedBy) == 0 && !hasAria2LocalFile(status.Files) {
continue
}
return status, nil
case string(arigo.StatusActive):
if total > 0 && completed >= total && hasAria2LocalFile(status.Files) {
return status, nil
}
case string(arigo.StatusError), string(arigo.StatusRemoved):
if status.ErrorMessage != "" {
return arigo.Status{}, errors.New(status.ErrorMessage)
}
return arigo.Status{}, fmt.Errorf("aria2 download ended with status %s", status.Status)
}
}
}
}
func (a Aria2) getAria2Peers(ctx context.Context, aria **arigo.Client, gid string) []arigo.Peer {
peers, err := (*aria).GetPeers(gid)
if err == nil {
return peers
}
if !isAria2RPCDisconnected(err) {
return nil
}
if err := a.reconnect(ctx, aria); err != nil {
return nil
}
peers, err = (*aria).GetPeers(gid)
if err != nil {
return nil
}
return peers
}
func (a Aria2) getAria2Files(ctx context.Context, aria **arigo.Client, gid string) ([]arigo.File, error) {
files, err := (*aria).GetFiles(gid)
if err == nil {
return files, nil
}
if !isAria2RPCDisconnected(err) {
return nil, err
}
if err := a.reconnect(ctx, aria); err != nil {
return nil, err
}
return (*aria).GetFiles(gid)
}
func (a Aria2) reconnect(ctx context.Context, aria **arigo.Client) error {
_ = (*aria).Close()
next, err := a.client(ctx)
if err != nil {
return err
}
*aria = next
return nil
}
func isAria2RPCDisconnected(err error) bool {
return errors.Is(err, rpc2.ErrShutdown) || errors.Is(err, io.ErrClosedPipe) || strings.Contains(err.Error(), "connection is shut down")
}
func aria2Detail(status arigo.Status, peers []arigo.Peer) *client.DownloadTaskDetail {
connections := int64(status.Connections)
seeders := int64(status.NumSeeders)
peerCount := int64(len(peers))
leechers := aria2Leechers(peers)
uploaded := int64(status.UploadLength)
uploadBps := int64(status.UploadSpeed)
detail := &client.DownloadTaskDetail{
Engine: "aria2",
Phase: aria2Phase(string(status.Status), status.FollowedBy),
EngineState: string(status.Status),
ETASeconds: aria2ETA(status),
Connections: &connections,
InfoHash: status.InfoHash,
TorrentName: status.BitTorrent.Info.Name,
Seeders: &seeders,
Leechers: leechers,
Peers: &peerCount,
PeerUploadedBytes: &uploaded,
PeerUploadBps: &uploadBps,
Trackers: aria2Trackers(status.BitTorrent.AnnounceList),
PeerSamples: aria2Peers(peers),
Files: aria2Files(status.Files),
}
if status.ErrorMessage != "" {
detail.Message = status.ErrorMessage
}
return detail
}
func aria2ETA(status arigo.Status) *int64 {
total := int64(status.TotalLength)
completed := int64(status.CompletedLength)
bps := int64(status.DownloadSpeed)
if total <= 0 || completed >= total || bps <= 0 {
return nil
}
remaining := total - completed
eta := (remaining + bps - 1) / bps
return &eta
}
func aria2Phase(state string, followedBy []string) string {
switch state {
case string(arigo.StatusWaiting):
if len(followedBy) > 0 {
return "metadata"
}
return "downloading"
case string(arigo.StatusActive):
return "downloading"
case "complete", string(arigo.StatusCompleted):
return "completed"
case string(arigo.StatusError), string(arigo.StatusRemoved):
return "error"
default:
return "downloading"
}
}
func aria2Trackers(announceList [][]string) []client.DownloadTaskTracker {
trackers := make([]client.DownloadTaskTracker, 0, 20)
seen := map[string]struct{}{}
for _, tier := range announceList {
for _, url := range tier {
if url == "" {
continue
}
if _, exists := seen[url]; exists {
continue
}
seen[url] = struct{}{}
trackers = append(trackers, client.DownloadTaskTracker{
URL: url,
Status: "announce",
Message: "aria2 exposes announce URLs only",
})
if len(trackers) >= 20 {
return trackers
}
}
}
return trackers
}
func aria2Leechers(peers []arigo.Peer) *int64 {
if len(peers) == 0 {
return nil
}
var count int64
for _, peer := range peers {
if !peer.Seeder {
count++
}
}
return &count
}
func aria2Peers(peers []arigo.Peer) []client.DownloadTaskPeer {
out := make([]client.DownloadTaskPeer, 0, min(len(peers), 20))
for _, peer := range peers {
if peer.IP == "" {
continue
}
down := int64(peer.DownloadSpeed)
up := int64(peer.UploadSpeed)
out = append(out, client.DownloadTaskPeer{
Address: fmt.Sprintf("%s:%d", peer.IP, peer.Port),
DownloadBps: &down,
UploadBps: &up,
})
if len(out) >= 20 {
break
}
}
return out
}
func aria2Files(files []arigo.File) []client.DownloadTaskFile {
out := make([]client.DownloadTaskFile, 0, min(len(files), 50))
for _, file := range files {
if file.Path == "" || isAria2MetadataPath(file.Path) {
continue
}
size := int64(file.Length)
completed := int64(file.CompletedLength)
selected := file.Selected
out = append(out, client.DownloadTaskFile{
Path: file.Path,
Size: size,
CompletedBytes: &completed,
Selected: &selected,
})
if len(out) >= 50 {
break
}
}
return out
}
func hasAria2LocalFile(files []arigo.File) bool {
for _, file := range files {
if file.Path != "" && !isAria2MetadataPath(file.Path) {
return true
}
}
return false
}
func resultFromAria2Files(task client.DownloadTask, taskDir string, fallbackName string, files []arigo.File) (Result, error) {
downloaded := make([]downloadedFile, 0, len(files))
for _, file := range files {
if file.Selected && file.Length > 0 && !isDownloadSidecarPath(file.Path) {
abs, rel := downloadedPath(taskDir, file.Path)
downloaded = append(downloaded, downloadedFile{path: abs, relativePath: rel})
}
}
if len(downloaded) == 0 {
return resultFromPath(task, taskDir, fallbackName)
}
return resultFromDownloadedFiles(task, taskDir, fallbackName, downloaded)
}
downloader/internal/engine/engine.go
+18 -811
View File
@@ -2,19 +2,12 @@ package engine
import (
"context"
"errors"
"fmt"
"io"
"net/http"
"net/url"
"os"
"path/filepath"
"strings"
"time"
"github.com/Braurbeki/arigo"
qbittorrent "github.com/autobrr/go-qbittorrent"
"github.com/cenkalti/rpc2"
"github.com/saltbo/zpan/downloader/internal/client"
)
@@ -44,376 +37,13 @@ type SeedSnapshot struct {
type Progress func(downloaded int64, total *int64, bps int64, detail *client.DownloadTaskDetail) error
type Engine interface {
Name() string
Capabilities() []string
Check(ctx context.Context) error
Recover(ctx context.Context, task client.DownloadTask) (Result, bool, error)
Download(ctx context.Context, task client.DownloadTask, progress Progress) (Result, error)
}
type HTTP struct {
Dir string
}
func (h HTTP) Check(ctx context.Context) error {
if err := os.MkdirAll(h.Dir, 0o755); err != nil {
return err
}
file, err := os.CreateTemp(h.Dir, ".zpan-check-*")
if err != nil {
return err
}
path := file.Name()
if err := file.Close(); err != nil {
return err
}
return os.Remove(path)
}
func (h HTTP) Download(ctx context.Context, task client.DownloadTask, progress Progress) (Result, error) {
if task.SourceType != "http" {
return Result{}, errors.New("http engine only supports http sources")
}
taskDir := filepath.Join(h.Dir, task.ID)
if err := os.MkdirAll(taskDir, 0o755); err != nil {
return Result{}, err
}
req, err := http.NewRequestWithContext(ctx, http.MethodGet, task.SourceURI, nil)
if err != nil {
return Result{}, err
}
res, err := http.DefaultClient.Do(req)
if err != nil {
return Result{}, err
}
defer res.Body.Close()
if res.StatusCode < 200 || res.StatusCode >= 300 {
return Result{}, errors.New(res.Status)
}
name := outputName(task, filenameFromURL(req.URL))
path := filepath.Join(taskDir, name)
file, err := os.Create(path)
if err != nil {
return Result{}, err
}
defer file.Close()
var total *int64
if res.ContentLength > 0 {
total = &res.ContentLength
}
counter := &progressWriter{progress: progress, total: total, lastAt: time.Now()}
if _, err := io.Copy(file, io.TeeReader(res.Body, counter)); err != nil {
return Result{}, err
}
if err := progress(counter.downloaded, total, 0, &client.DownloadTaskDetail{Engine: "builtin", Phase: "completed"}); err != nil {
return Result{}, err
}
return Result{Path: path, Name: name, Size: counter.downloaded}, nil
}
type Aria2 struct {
URL string
Secret string
Dir string
RetainSeed bool
}
func (a Aria2) Check(ctx context.Context) error {
client, err := a.client(ctx)
if err != nil {
return err
}
defer client.Close()
version, err := client.GetVersion()
if err != nil {
return err
}
if version.Version == "" {
return errors.New("aria2 rpc did not return a version")
}
return nil
}
func (a Aria2) Download(ctx context.Context, task client.DownloadTask, progress Progress) (Result, error) {
taskDir := filepath.Join(a.Dir, task.ID)
if err := os.MkdirAll(taskDir, 0o755); err != nil {
return Result{}, err
}
aria, err := a.client(ctx)
if err != nil {
return Result{}, err
}
defer aria.Close()
options := &arigo.Options{
Dir: taskDir,
FollowTorrent: true,
BTSaveMetadata: true,
SeedRatio: 0,
SeedTime: 0,
AllowOverwrite: true,
AutoFileRenaming: false,
}
if a.RetainSeed && task.SourceType != "http" {
options.SeedTime = 1000000
}
if task.Name != "" && task.SourceType == "http" {
options.Out = task.Name
}
gid, err := aria.AddURI(arigo.URIs(task.SourceURI), options)
if err != nil {
return Result{}, err
}
initialProgress := progress
if task.SourceType != "http" {
initialProgress = func(downloaded int64, total *int64, bps int64, detail *client.DownloadTaskDetail) error { return nil }
}
status, err := a.waitAria2(ctx, &aria, gid.GID, initialProgress)
if err != nil {
_ = aria.Remove(gid.GID)
return Result{}, err
}
if len(status.FollowedBy) > 0 {
childGID := status.FollowedBy[0]
status, err = a.waitAria2(ctx, &aria, childGID, progress)
if err != nil {
_ = aria.Remove(childGID)
return Result{}, err
}
}
files, err := a.getAria2Files(ctx, &aria, status.GID)
if err != nil {
return Result{}, err
}
result, err := resultFromAria2Files(task, taskDir, status.BitTorrent.Info.Name, files)
if err != nil {
return Result{}, err
}
if a.RetainSeed && task.SourceType != "http" {
result.Seed = &Seed{
Engine: "aria2",
ID: status.GID,
Path: taskDir,
Snapshot: a.seedSnapshot(status.GID),
Cleanup: a.cleanupSeed(status.GID, taskDir),
}
return result, nil
}
_ = aria.ForceRemove(status.GID)
_ = aria.RemoveDownloadResult(status.GID)
return result, nil
}
func (a Aria2) client(ctx context.Context) (*arigo.Client, error) {
return arigo.DialContext(ctx, a.URL, a.Secret)
}
func (a Aria2) seedSnapshot(gid string) func(context.Context) (SeedSnapshot, error) {
return func(ctx context.Context) (SeedSnapshot, error) {
aria, err := a.client(ctx)
if err != nil {
return SeedSnapshot{}, err
}
defer aria.Close()
status, err := aria.TellStatus(gid)
if err != nil {
return SeedSnapshot{}, err
}
peers := a.getAria2Peers(ctx, &aria, gid)
total := int64(status.TotalLength)
var totalPtr *int64
if total > 0 {
totalPtr = &total
}
detail := aria2Detail(status, peers)
detail.Phase = "seeding"
return SeedSnapshot{
Downloaded: int64(status.CompletedLength),
Total: totalPtr,
Bps: int64(status.DownloadSpeed),
Detail: detail,
}, nil
}
}
func (a Aria2) cleanupSeed(gid string, localPath string) func(context.Context) error {
return func(ctx context.Context) error {
var errs []error
aria, err := a.client(ctx)
if err != nil {
errs = append(errs, err)
} else {
_ = aria.ForceRemove(gid)
_ = aria.RemoveDownloadResult(gid)
_ = aria.Close()
}
if err := os.RemoveAll(localPath); err != nil {
errs = append(errs, err)
}
return errors.Join(errs...)
}
}
type QBittorrent struct {
URL string
Username string
Password string
Dir string
RetainSeed bool
}
func (q QBittorrent) Check(ctx context.Context) error {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, strings.TrimRight(q.URL, "/")+"/api/v2/app/version", nil)
if err != nil {
return err
}
res, err := (&http.Client{Timeout: 2 * time.Second}).Do(req)
if err != nil {
return err
}
defer res.Body.Close()
if res.StatusCode < 200 || res.StatusCode >= 300 {
return fmt.Errorf("qbittorrent web api returned %s", res.Status)
}
version, err := io.ReadAll(io.LimitReader(res.Body, 256))
if err != nil {
return err
}
if strings.TrimSpace(string(version)) == "" {
return errors.New("qbittorrent web api did not return a version")
}
return nil
}
func (q QBittorrent) login(ctx context.Context) (*qbittorrent.Client, error) {
qbt := qbittorrent.NewClient(qbittorrent.Config{
Host: q.URL,
Username: q.Username,
Password: q.Password,
Timeout: 10,
})
if err := qbt.LoginCtx(ctx); err != nil {
return nil, err
}
return qbt, nil
}
func (q QBittorrent) Download(ctx context.Context, task client.DownloadTask, progress Progress) (Result, error) {
if task.SourceType == "http" {
return HTTP{Dir: q.Dir}.Download(ctx, task, progress)
}
taskDir := filepath.Join(q.Dir, task.ID)
if err := os.MkdirAll(taskDir, 0o755); err != nil {
return Result{}, err
}
qbt, err := q.login(ctx)
if err != nil {
return Result{}, err
}
tag := qbittorrentTrackingTag(task.ID)
options := qbittorrentAddOptions(task, taskDir, tag)
if _, err := qbt.AddTorrentFromUrlCtx(ctx, task.SourceURI, options); err != nil {
return Result{}, err
}
torrent, err := waitQBittorrent(ctx, qbt, tag, progress)
if err != nil {
return Result{}, err
}
result, err := resultFromQBittorrentFiles(ctx, qbt, task, taskDir, torrent)
if err != nil {
return Result{}, err
}
if q.RetainSeed {
result.Seed = &Seed{
Engine: "qbittorrent",
ID: torrent.Hash,
Path: taskDir,
Snapshot: q.seedSnapshot(torrent.Hash),
Cleanup: q.cleanupSeed(torrent.Hash, taskDir),
}
return result, nil
}
_ = qbt.DeleteTorrentsCtx(ctx, []string{torrent.Hash}, false)
return result, nil
}
func qbittorrentAddOptions(task client.DownloadTask, taskDir string, trackingTag string) map[string]string {
category := "zpan"
if task.Category != "" {
category = task.Category
}
tags := append([]string{trackingTag}, task.Tags...)
options := (&qbittorrent.TorrentAddOptions{
SavePath: taskDir,
Category: category,
Tags: strings.Join(tags, ","),
LimitRatio: 0,
LimitSeedTime: 0,
SequentialDownload: false,
}).Prepare()
if name := requestedOutputName(task); name != "" {
options["rename"] = name
}
return options
}
func qbittorrentTrackingTag(taskID string) string {
return "ztid=" + taskID
}
func (q QBittorrent) cleanupSeed(hash string, localPath string) func(context.Context) error {
return func(ctx context.Context) error {
var errs []error
qbt, err := q.login(ctx)
if err != nil {
errs = append(errs, err)
} else if err := qbt.DeleteTorrentsCtx(ctx, []string{hash}, false); err != nil {
errs = append(errs, err)
}
if err := os.RemoveAll(localPath); err != nil {
errs = append(errs, err)
}
return errors.Join(errs...)
}
}
func (q QBittorrent) seedSnapshot(hash string) func(context.Context) (SeedSnapshot, error) {
return func(ctx context.Context) (SeedSnapshot, error) {
qbt, err := q.login(ctx)
if err != nil {
return SeedSnapshot{}, err
}
torrents, err := qbt.GetTorrentsCtx(ctx, qbittorrent.TorrentFilterOptions{Hashes: []string{hash}})
if err != nil {
return SeedSnapshot{}, err
}
if len(torrents) == 0 {
return SeedSnapshot{}, fmt.Errorf("qbittorrent torrent %s not found", hash)
}
torrent := torrents[0]
total := torrent.TotalSize
if total <= 0 {
total = torrent.Size
}
var totalPtr *int64
if total > 0 {
totalPtr = &total
}
detail := qbittorrentDetail(ctx, qbt, torrent)
detail.Phase = "seeding"
return SeedSnapshot{
Downloaded: torrent.Completed,
Total: totalPtr,
Bps: torrent.DlSpeed,
Detail: detail,
}, nil
}
}
type progressWriter struct {
progress Progress
total *int64
@@ -437,439 +67,6 @@ func (p *progressWriter) Write(data []byte) (int, error) {
return n, nil
}
func (a Aria2) waitAria2(ctx context.Context, aria **arigo.Client, gid string, progress Progress) (arigo.Status, error) {
ticker := time.NewTicker(time.Second)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
return arigo.Status{}, ctx.Err()
case <-ticker.C:
status, err := (*aria).TellStatus(gid)
if err != nil {
if isAria2RPCDisconnected(err) {
if err := a.reconnect(ctx, aria); err != nil {
return arigo.Status{}, err
}
continue
}
return arigo.Status{}, err
}
total := int64(status.TotalLength)
completed := int64(status.CompletedLength)
bps := int64(status.DownloadSpeed)
var totalPtr *int64
if total > 0 {
totalPtr = &total
}
peers := a.getAria2Peers(ctx, aria, gid)
if err := progress(completed, totalPtr, bps, aria2Detail(status, peers)); err != nil {
_ = (*aria).ForcePause(gid)
return arigo.Status{}, err
}
switch string(status.Status) {
case "complete", string(arigo.StatusCompleted):
if len(status.FollowedBy) == 0 && !hasAria2LocalFile(status.Files) {
continue
}
return status, nil
case string(arigo.StatusActive):
if total > 0 && completed >= total && hasAria2LocalFile(status.Files) {
return status, nil
}
case string(arigo.StatusError), string(arigo.StatusRemoved):
if status.ErrorMessage != "" {
return arigo.Status{}, errors.New(status.ErrorMessage)
}
return arigo.Status{}, fmt.Errorf("aria2 download ended with status %s", status.Status)
}
}
}
}
func (a Aria2) getAria2Peers(ctx context.Context, aria **arigo.Client, gid string) []arigo.Peer {
peers, err := (*aria).GetPeers(gid)
if err == nil {
return peers
}
if !isAria2RPCDisconnected(err) {
return nil
}
if err := a.reconnect(ctx, aria); err != nil {
return nil
}
peers, err = (*aria).GetPeers(gid)
if err != nil {
return nil
}
return peers
}
func (a Aria2) getAria2Files(ctx context.Context, aria **arigo.Client, gid string) ([]arigo.File, error) {
files, err := (*aria).GetFiles(gid)
if err == nil {
return files, nil
}
if !isAria2RPCDisconnected(err) {
return nil, err
}
if err := a.reconnect(ctx, aria); err != nil {
return nil, err
}
return (*aria).GetFiles(gid)
}
func (a Aria2) reconnect(ctx context.Context, aria **arigo.Client) error {
_ = (*aria).Close()
next, err := a.client(ctx)
if err != nil {
return err
}
*aria = next
return nil
}
func isAria2RPCDisconnected(err error) bool {
return errors.Is(err, rpc2.ErrShutdown) || errors.Is(err, io.ErrClosedPipe) || strings.Contains(err.Error(), "connection is shut down")
}
func aria2Detail(status arigo.Status, peers []arigo.Peer) *client.DownloadTaskDetail {
connections := int64(status.Connections)
seeders := int64(status.NumSeeders)
peerCount := int64(len(peers))
leechers := aria2Leechers(peers)
uploaded := int64(status.UploadLength)
uploadBps := int64(status.UploadSpeed)
detail := &client.DownloadTaskDetail{
Engine: "aria2",
Phase: aria2Phase(string(status.Status), status.FollowedBy),
EngineState: string(status.Status),
ETASeconds: aria2ETA(status),
Connections: &connections,
InfoHash: status.InfoHash,
TorrentName: status.BitTorrent.Info.Name,
Seeders: &seeders,
Leechers: leechers,
Peers: &peerCount,
PeerUploadedBytes: &uploaded,
PeerUploadBps: &uploadBps,
Trackers: aria2Trackers(status.BitTorrent.AnnounceList),
PeerSamples: aria2Peers(peers),
Files: aria2Files(status.Files),
}
if status.ErrorMessage != "" {
detail.Message = status.ErrorMessage
}
return detail
}
func aria2ETA(status arigo.Status) *int64 {
total := int64(status.TotalLength)
completed := int64(status.CompletedLength)
bps := int64(status.DownloadSpeed)
if total <= 0 || completed >= total || bps <= 0 {
return nil
}
remaining := total - completed
eta := (remaining + bps - 1) / bps
return &eta
}
func aria2Phase(state string, followedBy []string) string {
switch state {
case string(arigo.StatusWaiting):
if len(followedBy) > 0 {
return "metadata"
}
return "downloading"
case string(arigo.StatusActive):
return "downloading"
case "complete", string(arigo.StatusCompleted):
return "completed"
case string(arigo.StatusError), string(arigo.StatusRemoved):
return "error"
default:
return "downloading"
}
}
func aria2Trackers(announceList [][]string) []client.DownloadTaskTracker {
trackers := make([]client.DownloadTaskTracker, 0, 20)
seen := map[string]struct{}{}
for _, tier := range announceList {
for _, url := range tier {
if url == "" {
continue
}
if _, exists := seen[url]; exists {
continue
}
seen[url] = struct{}{}
trackers = append(trackers, client.DownloadTaskTracker{
URL: url,
Status: "announce",
Message: "aria2 exposes announce URLs only",
})
if len(trackers) >= 20 {
return trackers
}
}
}
return trackers
}
func aria2Leechers(peers []arigo.Peer) *int64 {
if len(peers) == 0 {
return nil
}
var count int64
for _, peer := range peers {
if !peer.Seeder {
count++
}
}
return &count
}
func aria2Peers(peers []arigo.Peer) []client.DownloadTaskPeer {
out := make([]client.DownloadTaskPeer, 0, min(len(peers), 20))
for _, peer := range peers {
if peer.IP == "" {
continue
}
down := int64(peer.DownloadSpeed)
up := int64(peer.UploadSpeed)
out = append(out, client.DownloadTaskPeer{
Address: fmt.Sprintf("%s:%d", peer.IP, peer.Port),
DownloadBps: &down,
UploadBps: &up,
})
if len(out) >= 20 {
break
}
}
return out
}
func aria2Files(files []arigo.File) []client.DownloadTaskFile {
out := make([]client.DownloadTaskFile, 0, min(len(files), 50))
for _, file := range files {
if file.Path == "" || isAria2MetadataPath(file.Path) {
continue
}
size := int64(file.Length)
completed := int64(file.CompletedLength)
selected := file.Selected
out = append(out, client.DownloadTaskFile{
Path: file.Path,
Size: size,
CompletedBytes: &completed,
Selected: &selected,
})
if len(out) >= 50 {
break
}
}
return out
}
func hasAria2LocalFile(files []arigo.File) bool {
for _, file := range files {
if file.Path != "" && !isAria2MetadataPath(file.Path) {
return true
}
}
return false
}
func waitQBittorrent(
ctx context.Context,
qbt *qbittorrent.Client,
tag string,
progress Progress,
) (qbittorrent.Torrent, error) {
ticker := time.NewTicker(time.Second)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
return qbittorrent.Torrent{}, ctx.Err()
case <-ticker.C:
torrents, err := qbt.GetTorrentsCtx(ctx, qbittorrent.TorrentFilterOptions{Tag: tag})
if err != nil {
return qbittorrent.Torrent{}, err
}
if len(torrents) == 0 {
continue
}
torrent := torrents[0]
total := torrent.TotalSize
if total <= 0 {
total = torrent.Size
}
var totalPtr *int64
if total > 0 {
totalPtr = &total
}
if err := progress(torrent.Completed, totalPtr, torrent.DlSpeed, qbittorrentDetail(ctx, qbt, torrent)); err != nil {
_ = qbt.StopCtx(ctx, []string{torrent.Hash})
return qbittorrent.Torrent{}, err
}
if torrent.Progress >= 1 || (torrent.AmountLeft == 0 && total > 0) {
return torrent, nil
}
if isQBittorrentErrorState(torrent.State) {
return qbittorrent.Torrent{}, fmt.Errorf("qbittorrent download ended with state %s", torrent.State)
}
}
}
}
func qbittorrentDetail(ctx context.Context, qbt *qbittorrent.Client, torrent qbittorrent.Torrent) *client.DownloadTaskDetail {
connections := int64(torrent.NumSeeds + torrent.NumLeechs)
seeders := torrent.NumSeeds
leechers := torrent.NumLeechs
peers := torrent.NumComplete + torrent.NumIncomplete
uploaded := torrent.Uploaded
uploadBps := torrent.UpSpeed
var eta *int64
if torrent.ETA >= 0 {
eta = &torrent.ETA
}
return &client.DownloadTaskDetail{
Engine: "qbittorrent",
Phase: qbittorrentPhase(string(torrent.State)),
EngineState: string(torrent.State),
ETASeconds: eta,
Connections: &connections,
InfoHash: torrent.Hash,
TorrentName: torrent.Name,
Seeders: &seeders,
Leechers: &leechers,
Peers: &peers,
PeerUploadedBytes: &uploaded,
PeerUploadBps: &uploadBps,
Trackers: qbittorrentTrackers(ctx, qbt, torrent),
PeerSamples: qbittorrentPeers(ctx, qbt, torrent.Hash),
}
}
func qbittorrentPhase(state string) string {
normalized := strings.ToLower(state)
switch {
case strings.Contains(normalized, "meta"):
return "metadata"
case strings.Contains(normalized, "up"), strings.Contains(normalized, "seed"):
return "seeding"
case strings.Contains(normalized, "error"), strings.Contains(normalized, "missing"):
return "error"
case strings.Contains(normalized, "paused"):
return "downloading"
case normalized == "uploading":
return "seeding"
default:
return "downloading"
}
}
func qbittorrentTrackers(ctx context.Context, qbt *qbittorrent.Client, torrent qbittorrent.Torrent) []client.DownloadTaskTracker {
trackers := torrent.Trackers
if len(trackers) == 0 && torrent.Hash != "" {
loaded, err := qbt.GetTorrentTrackersCtx(ctx, torrent.Hash)
if err == nil {
trackers = loaded
}
}
out := make([]client.DownloadTaskTracker, 0, min(len(trackers), 20))
for _, tracker := range trackers {
peers := int64(tracker.NumPeers)
seeds := int64(tracker.NumSeeds)
leechers := int64(tracker.NumLeechers)
out = append(out, client.DownloadTaskTracker{
URL: tracker.Url,
Status: fmt.Sprint(tracker.Status),
Peers: &peers,
Seeds: &seeds,
Leechers: &leechers,
Message: tracker.Message,
})
if len(out) >= 20 {
break
}
}
return out
}
func qbittorrentPeers(ctx context.Context, qbt *qbittorrent.Client, hash string) []client.DownloadTaskPeer {
if hash == "" {
return nil
}
peers, err := qbt.GetTorrentPeersCtx(ctx, hash, 0)
if err != nil || peers == nil {
return nil
}
out := make([]client.DownloadTaskPeer, 0, min(len(peers.Peers), 20))
for address, peer := range peers.Peers {
progress := peer.Progress
down := peer.DownSpeed
up := peer.UpSpeed
label := address
if peer.IP != "" && peer.Port > 0 {
label = fmt.Sprintf("%s:%d", peer.IP, peer.Port)
}
out = append(out, client.DownloadTaskPeer{
Address: label,
Client: peer.Client,
Progress: &progress,
DownloadBps: &down,
UploadBps: &up,
})
if len(out) >= 20 {
break
}
}
return out
}
func resultFromAria2Files(task client.DownloadTask, taskDir string, fallbackName string, files []arigo.File) (Result, error) {
downloaded := make([]downloadedFile, 0, len(files))
for _, file := range files {
if file.Selected && file.Length > 0 && !isDownloadSidecarPath(file.Path) {
abs, rel := downloadedPath(taskDir, file.Path)
downloaded = append(downloaded, downloadedFile{path: abs, relativePath: rel})
}
}
if len(downloaded) == 0 {
return resultFromPath(task, taskDir, fallbackName)
}
return resultFromDownloadedFiles(task, taskDir, fallbackName, downloaded)
}
func resultFromQBittorrentFiles(
ctx context.Context,
qbt *qbittorrent.Client,
task client.DownloadTask,
taskDir string,
torrent qbittorrent.Torrent,
) (Result, error) {
files, err := qbt.GetFilesInformationCtx(ctx, torrent.Hash)
if err != nil || files == nil {
return resultFromPath(task, taskDir, torrent.Name)
}
downloaded := make([]downloadedFile, 0, len(*files))
for _, file := range *files {
if file.Priority == 0 || file.Size <= 0 {
continue
}
abs, rel := downloadedPath(taskDir, file.Name)
downloaded = append(downloaded, downloadedFile{path: abs, relativePath: rel})
}
if len(downloaded) == 0 {
return resultFromPath(task, taskDir, torrent.Name)
}
return resultFromDownloadedFiles(task, taskDir, torrent.Name, downloaded)
}
func resultFromPath(task client.DownloadTask, path string, fallbackName string) (Result, error) {
info, err := os.Stat(path)
if err != nil {
@@ -911,6 +108,21 @@ func resultFromPath(task client.DownloadTask, path string, fallbackName string)
return Result{Path: path, Name: outputName(task, fallbackName), Size: size, IsDir: true}, nil
}
func recoverFromTaskDir(task client.DownloadTask, dir string) (Result, bool, error) {
taskDir := filepath.Join(dir, task.ID)
if _, err := os.Stat(taskDir); err != nil {
if os.IsNotExist(err) {
return Result{}, false, nil
}
return Result{}, false, err
}
result, err := resultFromPath(task, taskDir, requestedOutputName(task))
if err != nil {
return Result{}, false, err
}
return result, true, nil
}
type downloadedFile struct {
path string
relativePath string
@@ -1113,8 +325,3 @@ func filenameFromURL(parsed *url.URL) string {
}
return name
}
func isQBittorrentErrorState(state qbittorrent.TorrentState) bool {
value := strings.ToLower(string(state))
return strings.Contains(value, "error") || strings.Contains(value, "missing")
}
downloader/internal/engine/engine_test.go
+43
View File
@@ -72,6 +72,49 @@ func TestHTTPRejectsMagnet(t *testing.T) {
}
}
func TestHTTPDownloadResumesExistingFile(t *testing.T) {
var rangeHeader string
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
rangeHeader = r.Header.Get("Range")
if rangeHeader != "bytes=5-" {
t.Fatalf("expected resume range bytes=5-, got %q", rangeHeader)
}
w.Header().Set("Content-Length", "6")
w.WriteHeader(http.StatusPartialContent)
_, _ = w.Write([]byte(" world"))
}))
defer server.Close()
dir := t.TempDir()
taskDir := filepath.Join(dir, "task-1")
if err := os.MkdirAll(taskDir, 0o755); err != nil {
t.Fatal(err)
}
path := filepath.Join(taskDir, "file.txt")
if err := os.WriteFile(path, []byte("hello"), 0o644); err != nil {
t.Fatal(err)
}
result, err := (HTTP{Dir: dir}).Download(
context.Background(),
client.DownloadTask{ID: "task-1", SourceType: "http", SourceURI: server.URL + "/file.txt"},
func(downloaded int64, total *int64, bps int64, detail *client.DownloadTaskDetail) error { return nil },
)
if err != nil {
t.Fatal(err)
}
data, err := os.ReadFile(result.Path)
if err != nil {
t.Fatal(err)
}
if string(data) != "hello world" {
t.Fatalf("expected resumed content, got %q", string(data))
}
if result.Size != 11 {
t.Fatalf("expected size 11, got %d", result.Size)
}
}
func TestQBittorrentCheckUsesWebAPIVersion(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path != "/api/v2/app/version" {
downloader/internal/engine/http.go
+126
View File
@@ -0,0 +1,126 @@
package engine
import (
"context"
"errors"
"io"
"net/http"
"os"
"path/filepath"
"strconv"
"time"
"github.com/saltbo/zpan/downloader/internal/client"
)
type HTTP struct {
Dir string
}
func (h HTTP) Name() string {
return "builtin"
}
func (h HTTP) Capabilities() []string {
return []string{"http"}
}
func (h HTTP) Check(ctx context.Context) error {
if err := os.MkdirAll(h.Dir, 0o755); err != nil {
return err
}
file, err := os.CreateTemp(h.Dir, ".zpan-check-*")
if err != nil {
return err
}
path := file.Name()
if err := file.Close(); err != nil {
return err
}
return os.Remove(path)
}
func (h HTTP) Recover(ctx context.Context, task client.DownloadTask) (Result, bool, error) {
return recoverFromTaskDir(task, h.Dir)
}
func (h HTTP) Download(ctx context.Context, task client.DownloadTask, progress Progress) (Result, error) {
if task.SourceType != "http" {
return Result{}, errors.New("http engine only supports http sources")
}
taskDir := filepath.Join(h.Dir, task.ID)
if err := os.MkdirAll(taskDir, 0o755); err != nil {
return Result{}, err
}
req, err := http.NewRequestWithContext(ctx, http.MethodGet, task.SourceURI, nil)
if err != nil {
return Result{}, err
}
name := outputName(task, filenameFromURL(req.URL))
path := filepath.Join(taskDir, name)
existingSize, err := existingFileSize(path)
if err != nil {
return Result{}, err
}
if existingSize > 0 {
req.Header.Set("Range", "bytes="+strconv.FormatInt(existingSize, 10)+"-")
}
res, err := http.DefaultClient.Do(req)
if err != nil {
return Result{}, err
}
defer res.Body.Close()
if res.StatusCode == http.StatusRequestedRangeNotSatisfiable && existingSize > 0 {
return resultFromFile(task, path)
}
if res.StatusCode < 200 || res.StatusCode >= 300 {
return Result{}, errors.New(res.Status)
}
appendExisting := existingSize > 0 && res.StatusCode == http.StatusPartialContent
if existingSize > 0 && !appendExisting {
existingSize = 0
}
file, err := openOutputFile(path, appendExisting)
if err != nil {
return Result{}, err
}
defer file.Close()
var total *int64
if res.ContentLength > 0 {
value := res.ContentLength + existingSize
total = &value
}
counter := &progressWriter{progress: progress, total: total, downloaded: existingSize, lastBytes: existingSize, lastAt: time.Now()}
if _, err := io.Copy(file, io.TeeReader(res.Body, counter)); err != nil {
return Result{}, err
}
if err := progress(counter.downloaded, total, 0, &client.DownloadTaskDetail{Engine: "builtin", Phase: "completed"}); err != nil {
return Result{}, err
}
return Result{Path: path, Name: name, Size: counter.downloaded}, nil
}
func existingFileSize(path string) (int64, error) {
info, err := os.Stat(path)
if err == nil {
if info.IsDir() {
return 0, nil
}
return info.Size(), nil
}
if os.IsNotExist(err) {
return 0, nil
}
return 0, err
}
func openOutputFile(path string, appendExisting bool) (*os.File, error) {
if appendExisting {
return os.OpenFile(path, os.O_WRONLY|os.O_APPEND, 0o644)
}
return os.Create(path)
}
downloader/internal/engine/process.go
+64
View File
@@ -0,0 +1,64 @@
package engine
import (
"context"
"fmt"
"net/url"
"os/exec"
"strconv"
"strings"
)
type Starter interface {
Start(ctx context.Context) (*exec.Cmd, error)
}
type localEngineURL struct {
port string
}
func lookPathAny(names ...string) (string, error) {
var lastErr error
for _, name := range names {
path, err := exec.LookPath(name)
if err == nil {
return path, nil
}
lastErr = err
}
return "", lastErr
}
func parseLocalEngineURL(raw string, defaultPort string) (localEngineURL, error) {
normalized := raw
if strings.HasPrefix(normalized, "ws://") {
normalized = "http://" + strings.TrimPrefix(normalized, "ws://")
}
if strings.HasPrefix(normalized, "wss://") {
normalized = "https://" + strings.TrimPrefix(normalized, "wss://")
}
parsed, err := url.Parse(normalized)
if err != nil {
return localEngineURL{}, err
}
host := parsed.Hostname()
if host != "" && host != "127.0.0.1" && host != "localhost" && host != "::1" {
return localEngineURL{}, fmt.Errorf("auto start only supports local engine URLs, got %s", host)
}
port := parsed.Port()
if port == "" {
port = defaultPort
}
if _, err := strconv.Atoi(port); err != nil {
return localEngineURL{}, fmt.Errorf("invalid engine port %q", port)
}
return localEngineURL{port: port}, nil
}
func filepathBase(path string) string {
parts := strings.FieldsFunc(path, func(r rune) bool { return r == '/' || r == '\\' })
if len(parts) == 0 {
return path
}
return parts[len(parts)-1]
}
downloader/internal/engine/qbittorrent.go
+455
View File
@@ -0,0 +1,455 @@
package engine
import (
"context"
"errors"
"fmt"
"io"
"net/http"
"os"
"os/exec"
"path/filepath"
"strings"
"time"
qbittorrent "github.com/autobrr/go-qbittorrent"
"github.com/saltbo/zpan/downloader/internal/client"
)
type QBittorrent struct {
URL string
Username string
Password string
Dir string
RetainSeed bool
}
func (q QBittorrent) Name() string {
return "qbittorrent"
}
func (q QBittorrent) Capabilities() []string {
return []string{"http", "magnet", "torrent"}
}
func (q QBittorrent) Start(ctx context.Context) (*exec.Cmd, error) {
path, err := lookPathAny("qbittorrent-nox", "qbittorrent")
if err != nil {
return nil, err
}
webURL, err := parseLocalEngineURL(q.URL, "8080")
if err != nil {
return nil, err
}
args := []string{}
if strings.Contains(filepathBase(path), "qbittorrent-nox") {
args = append(args, "--webui-port="+webURL.port)
}
cmd := exec.CommandContext(ctx, path, args...)
if err := cmd.Start(); err != nil {
return nil, err
}
go func() { _ = cmd.Wait() }()
return cmd, nil
}
func (q QBittorrent) Check(ctx context.Context) error {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, strings.TrimRight(q.URL, "/")+"/api/v2/app/version", nil)
if err != nil {
return err
}
res, err := (&http.Client{Timeout: 2 * time.Second}).Do(req)
if err != nil {
return err
}
defer res.Body.Close()
if res.StatusCode < 200 || res.StatusCode >= 300 {
return fmt.Errorf("qbittorrent web api returned %s", res.Status)
}
version, err := io.ReadAll(io.LimitReader(res.Body, 256))
if err != nil {
return err
}
if strings.TrimSpace(string(version)) == "" {
return errors.New("qbittorrent web api did not return a version")
}
return nil
}
func (q QBittorrent) Recover(ctx context.Context, task client.DownloadTask) (Result, bool, error) {
if task.SourceType == "http" {
return HTTP{Dir: q.Dir}.Recover(ctx, task)
}
qbt, err := q.login(ctx)
if err != nil {
return Result{}, false, err
}
torrent, ok, err := q.findTask(ctx, qbt, task)
if err != nil {
return Result{}, false, err
}
if ok && (torrent.Progress >= 1 || (torrent.AmountLeft == 0 && torrent.TotalSize > 0)) {
result, err := resultFromQBittorrentFiles(ctx, qbt, task, filepath.Join(q.Dir, task.ID), torrent)
return result, err == nil, err
}
return recoverFromTaskDir(task, q.Dir)
}
func (q QBittorrent) login(ctx context.Context) (*qbittorrent.Client, error) {
qbt := qbittorrent.NewClient(qbittorrent.Config{
Host: q.URL,
Username: q.Username,
Password: q.Password,
Timeout: 10,
})
if err := qbt.LoginCtx(ctx); err != nil {
return nil, err
}
return qbt, nil
}
func (q QBittorrent) Download(ctx context.Context, task client.DownloadTask, progress Progress) (Result, error) {
if task.SourceType == "http" {
return HTTP{Dir: q.Dir}.Download(ctx, task, progress)
}
taskDir := filepath.Join(q.Dir, task.ID)
if err := os.MkdirAll(taskDir, 0o755); err != nil {
return Result{}, err
}
qbt, err := q.login(ctx)
if err != nil {
return Result{}, err
}
tag := qbittorrentTrackingTag(task.ID)
torrent, ok, err := q.findTask(ctx, qbt, task)
if err != nil {
return Result{}, err
}
if ok {
_ = qbt.StartCtx(ctx, []string{torrent.Hash})
torrent, err = waitQBittorrent(ctx, qbt, tag, progress)
if err != nil {
return Result{}, err
}
return q.resultFromTorrent(ctx, qbt, task, taskDir, torrent)
}
options := qbittorrentAddOptions(task, taskDir, tag)
if _, err := qbt.AddTorrentFromUrlCtx(ctx, task.SourceURI, options); err != nil {
torrent, ok, findErr := q.findTask(ctx, qbt, task)
if findErr != nil {
return Result{}, findErr
}
if !ok {
return Result{}, err
}
_ = qbt.StartCtx(ctx, []string{torrent.Hash})
}
torrent, err = waitQBittorrent(ctx, qbt, tag, progress)
if err != nil {
return Result{}, err
}
return q.resultFromTorrent(ctx, qbt, task, taskDir, torrent)
}
func (q QBittorrent) resultFromTorrent(
ctx context.Context,
qbt *qbittorrent.Client,
task client.DownloadTask,
taskDir string,
torrent qbittorrent.Torrent,
) (Result, error) {
result, err := resultFromQBittorrentFiles(ctx, qbt, task, taskDir, torrent)
if err != nil {
return Result{}, err
}
if q.RetainSeed {
result.Seed = &Seed{
Engine: "qbittorrent",
ID: torrent.Hash,
Path: taskDir,
Snapshot: q.seedSnapshot(torrent.Hash),
Cleanup: q.cleanupSeed(torrent.Hash, taskDir),
}
return result, nil
}
_ = qbt.DeleteTorrentsCtx(ctx, []string{torrent.Hash}, false)
return result, nil
}
func (q QBittorrent) findTask(ctx context.Context, qbt *qbittorrent.Client, task client.DownloadTask) (qbittorrent.Torrent, bool, error) {
tag := qbittorrentTrackingTag(task.ID)
torrents, err := qbt.GetTorrentsCtx(ctx, qbittorrent.TorrentFilterOptions{Tag: tag})
if err != nil {
return qbittorrent.Torrent{}, false, err
}
if len(torrents) > 0 {
return torrents[0], true, nil
}
taskDir := filepath.Clean(filepath.Join(q.Dir, task.ID))
torrents, err = qbt.GetTorrentsCtx(ctx, qbittorrent.TorrentFilterOptions{})
if err != nil {
return qbittorrent.Torrent{}, false, err
}
for _, torrent := range torrents {
if filepath.Clean(torrent.SavePath) == taskDir {
return torrent, true, nil
}
}
return qbittorrent.Torrent{}, false, nil
}
func qbittorrentAddOptions(task client.DownloadTask, taskDir string, trackingTag string) map[string]string {
category := "zpan"
if task.Category != "" {
category = task.Category
}
tags := append([]string{trackingTag}, task.Tags...)
options := (&qbittorrent.TorrentAddOptions{
SavePath: taskDir,
Category: category,
Tags: strings.Join(tags, ","),
LimitRatio: 0,
LimitSeedTime: 0,
SequentialDownload: false,
}).Prepare()
if name := requestedOutputName(task); name != "" {
options["rename"] = name
}
return options
}
func qbittorrentTrackingTag(taskID string) string {
return "ztid=" + taskID
}
func (q QBittorrent) cleanupSeed(hash string, localPath string) func(context.Context) error {
return func(ctx context.Context) error {
var errs []error
qbt, err := q.login(ctx)
if err != nil {
errs = append(errs, err)
} else if err := qbt.DeleteTorrentsCtx(ctx, []string{hash}, false); err != nil {
errs = append(errs, err)
}
if err := os.RemoveAll(localPath); err != nil {
errs = append(errs, err)
}
return errors.Join(errs...)
}
}
func (q QBittorrent) seedSnapshot(hash string) func(context.Context) (SeedSnapshot, error) {
return func(ctx context.Context) (SeedSnapshot, error) {
qbt, err := q.login(ctx)
if err != nil {
return SeedSnapshot{}, err
}
torrents, err := qbt.GetTorrentsCtx(ctx, qbittorrent.TorrentFilterOptions{Hashes: []string{hash}})
if err != nil {
return SeedSnapshot{}, err
}
if len(torrents) == 0 {
return SeedSnapshot{}, fmt.Errorf("qbittorrent torrent %s not found", hash)
}
torrent := torrents[0]
total := torrent.TotalSize
if total <= 0 {
total = torrent.Size
}
var totalPtr *int64
if total > 0 {
totalPtr = &total
}
detail := qbittorrentDetail(ctx, qbt, torrent)
detail.Phase = "seeding"
return SeedSnapshot{
Downloaded: torrent.Completed,
Total: totalPtr,
Bps: torrent.DlSpeed,
Detail: detail,
}, nil
}
}
func waitQBittorrent(
ctx context.Context,
qbt *qbittorrent.Client,
tag string,
progress Progress,
) (qbittorrent.Torrent, error) {
ticker := time.NewTicker(time.Second)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
return qbittorrent.Torrent{}, ctx.Err()
case <-ticker.C:
torrents, err := qbt.GetTorrentsCtx(ctx, qbittorrent.TorrentFilterOptions{Tag: tag})
if err != nil {
return qbittorrent.Torrent{}, err
}
if len(torrents) == 0 {
continue
}
torrent := torrents[0]
total := torrent.TotalSize
if total <= 0 {
total = torrent.Size
}
var totalPtr *int64
if total > 0 {
totalPtr = &total
}
if err := progress(torrent.Completed, totalPtr, torrent.DlSpeed, qbittorrentDetail(ctx, qbt, torrent)); err != nil {
_ = qbt.StopCtx(ctx, []string{torrent.Hash})
return qbittorrent.Torrent{}, err
}
if torrent.Progress >= 1 || (torrent.AmountLeft == 0 && total > 0) {
return torrent, nil
}
if isQBittorrentErrorState(torrent.State) {
return qbittorrent.Torrent{}, fmt.Errorf("qbittorrent download ended with state %s", torrent.State)
}
}
}
}
func qbittorrentDetail(ctx context.Context, qbt *qbittorrent.Client, torrent qbittorrent.Torrent) *client.DownloadTaskDetail {
connections := int64(torrent.NumSeeds + torrent.NumLeechs)
seeders := torrent.NumSeeds
leechers := torrent.NumLeechs
peers := torrent.NumComplete + torrent.NumIncomplete
uploaded := torrent.Uploaded
uploadBps := torrent.UpSpeed
var eta *int64
if torrent.ETA >= 0 {
eta = &torrent.ETA
}
return &client.DownloadTaskDetail{
Engine: "qbittorrent",
Phase: qbittorrentPhase(string(torrent.State)),
EngineState: string(torrent.State),
ETASeconds: eta,
Connections: &connections,
InfoHash: torrent.Hash,
TorrentName: torrent.Name,
Seeders: &seeders,
Leechers: &leechers,
Peers: &peers,
PeerUploadedBytes: &uploaded,
PeerUploadBps: &uploadBps,
Trackers: qbittorrentTrackers(ctx, qbt, torrent),
PeerSamples: qbittorrentPeers(ctx, qbt, torrent.Hash),
}
}
func qbittorrentPhase(state string) string {
normalized := strings.ToLower(state)
switch {
case strings.Contains(normalized, "meta"):
return "metadata"
case strings.Contains(normalized, "up"), strings.Contains(normalized, "seed"):
return "seeding"
case strings.Contains(normalized, "error"), strings.Contains(normalized, "missing"):
return "error"
case strings.Contains(normalized, "paused"):
return "downloading"
case normalized == "uploading":
return "seeding"
default:
return "downloading"
}
}
func qbittorrentTrackers(ctx context.Context, qbt *qbittorrent.Client, torrent qbittorrent.Torrent) []client.DownloadTaskTracker {
trackers := torrent.Trackers
if len(trackers) == 0 && torrent.Hash != "" {
loaded, err := qbt.GetTorrentTrackersCtx(ctx, torrent.Hash)
if err == nil {
trackers = loaded
}
}
out := make([]client.DownloadTaskTracker, 0, min(len(trackers), 20))
for _, tracker := range trackers {
peers := int64(tracker.NumPeers)
seeds := int64(tracker.NumSeeds)
leechers := int64(tracker.NumLeechers)
out = append(out, client.DownloadTaskTracker{
URL: tracker.Url,
Status: fmt.Sprint(tracker.Status),
Peers: &peers,
Seeds: &seeds,
Leechers: &leechers,
Message: tracker.Message,
})
if len(out) >= 20 {
break
}
}
return out
}
func qbittorrentPeers(ctx context.Context, qbt *qbittorrent.Client, hash string) []client.DownloadTaskPeer {
if hash == "" {
return nil
}
peers, err := qbt.GetTorrentPeersCtx(ctx, hash, 0)
if err != nil || peers == nil {
return nil
}
out := make([]client.DownloadTaskPeer, 0, min(len(peers.Peers), 20))
for address, peer := range peers.Peers {
progress := peer.Progress
down := peer.DownSpeed
up := peer.UpSpeed
label := address
if peer.IP != "" && peer.Port > 0 {
label = fmt.Sprintf("%s:%d", peer.IP, peer.Port)
}
out = append(out, client.DownloadTaskPeer{
Address: label,
Client: peer.Client,
Progress: &progress,
DownloadBps: &down,
UploadBps: &up,
})
if len(out) >= 20 {
break
}
}
return out
}
func resultFromQBittorrentFiles(
ctx context.Context,
qbt *qbittorrent.Client,
task client.DownloadTask,
taskDir string,
torrent qbittorrent.Torrent,
) (Result, error) {
files, err := qbt.GetFilesInformationCtx(ctx, torrent.Hash)
if err != nil || files == nil {
return resultFromPath(task, taskDir, torrent.Name)
}
downloaded := make([]downloadedFile, 0, len(*files))
for _, file := range *files {
if file.Priority == 0 || file.Size <= 0 {
continue
}
abs, rel := downloadedPath(taskDir, file.Name)
downloaded = append(downloaded, downloadedFile{path: abs, relativePath: rel})
}
if len(downloaded) == 0 {
return resultFromPath(task, taskDir, torrent.Name)
}
return resultFromDownloadedFiles(task, taskDir, torrent.Name, downloaded)
}
func isQBittorrentErrorState(state qbittorrent.TorrentState) bool {
value := strings.ToLower(string(state))
return strings.Contains(value, "error") || strings.Contains(value, "missing")
}
downloader/internal/openapi/client.gen.go
+415 -341
View File
File diff suppressed because it is too large Load Diff
downloader/internal/worker/api.go
+143
View File
@@ -0,0 +1,143 @@
package worker
import (
"context"
"fmt"
"time"
"github.com/saltbo/zpan/downloader/internal/client"
)
const apiRetryAttempts = 3
func (w *Worker) updateTask(ctx context.Context, id string, patch client.TaskPatch) (client.DownloadTask, error) {
var task client.DownloadTask
err := w.callAPI(ctx, "update task", func(ctx context.Context) error {
var err error
task, err = w.api.UpdateTask(ctx, id, patch)
return err
})
return task, err
}
func (w *Worker) createFolder(ctx context.Context, token string, name string, parent string) (client.ObjectDraft, error) {
var draft client.ObjectDraft
err := w.callAPI(ctx, "create folder", func(ctx context.Context) error {
var err error
draft, err = w.api.CreateFolder(ctx, token, name, parent)
return err
})
return draft, err
}
func (w *Worker) createObject(ctx context.Context, token string, name string, size int64, parent string) (client.ObjectDraft, error) {
var draft client.ObjectDraft
err := w.callAPI(ctx, "create object", func(ctx context.Context) error {
var err error
draft, err = w.api.CreateObject(ctx, token, name, size, parent)
return err
})
return draft, err
}
func (w *Worker) confirmObject(ctx context.Context, token string, id string) error {
return w.callAPI(ctx, "confirm object", func(ctx context.Context) error {
return w.api.ConfirmObject(ctx, token, id)
})
}
func (w *Worker) createObjectUploadSession(ctx context.Context, token string, id string, partSize int64) (client.ObjectUploadSession, error) {
var session client.ObjectUploadSession
err := w.callAPI(ctx, "create multipart upload session", func(ctx context.Context) error {
var err error
session, err = w.api.CreateObjectUploadSession(ctx, token, id, partSize)
return err
})
return session, err
}
func (w *Worker) presignObjectUploadParts(ctx context.Context, token string, id string, sessionID string, partNumbers []int) ([]client.PresignedObjectUploadPart, error) {
var parts []client.PresignedObjectUploadPart
err := w.callAPI(ctx, "presign multipart upload parts", func(ctx context.Context) error {
var err error
parts, err = w.api.PresignObjectUploadParts(ctx, token, id, sessionID, partNumbers)
return err
})
return parts, err
}
func (w *Worker) completeObjectUploadSession(ctx context.Context, token string, id string, sessionID string, parts []client.CompletedObjectUploadPart) error {
return w.callAPI(ctx, "complete multipart upload session", func(ctx context.Context) error {
return w.api.CompleteObjectUploadSession(ctx, token, id, sessionID, parts)
})
}
func (w *Worker) abortObjectUploadSession(ctx context.Context, token string, id string, sessionID string) error {
return w.callAPI(ctx, "abort multipart upload session", func(ctx context.Context) error {
return w.api.AbortObjectUploadSession(ctx, token, id, sessionID)
})
}
func (w *Worker) callAPI(ctx context.Context, operation string, call func(context.Context) error) error {
var last error
for attempt := 1; attempt <= apiRetryAttempts; attempt++ {
if err := ctx.Err(); err != nil {
return err
}
if err := call(ctx); err != nil {
last = err
if attempt == apiRetryAttempts || !isRetryableAPIError(err) {
return err
}
delay := time.Duration(attempt) * 500 * time.Millisecond
w.logger.Warn("retrying downloader api call", "operation", operation, "attempt", attempt, "delay", delay.String(), "error", err)
select {
case <-ctx.Done():
return ctx.Err()
case <-time.After(delay):
}
continue
}
return nil
}
return fmt.Errorf("%s failed: %w", operation, last)
}
func isRetryableAPIError(err error) bool {
if err == nil {
return false
}
message := err.Error()
return containsAny(message,
"connection refused",
"connection reset",
"connection is shut down",
"timeout",
"temporary failure",
"Too Many Requests",
"Bad Gateway",
"Service Unavailable",
"Gateway Timeout",
)
}
func containsAny(value string, needles ...string) bool {
for _, needle := range needles {
if needle != "" && contains(value, needle) {
return true
}
}
return false
}
func contains(value string, needle string) bool {
if len(needle) > len(value) {
return false
}
for i := 0; i <= len(value)-len(needle); i++ {
if value[i:i+len(needle)] == needle {
return true
}
}
return false
}
downloader/internal/worker/api_test.go
+46
View File
@@ -0,0 +1,46 @@
package worker
import (
"context"
"errors"
"testing"
"github.com/saltbo/zpan/downloader/internal/config"
)
func TestCallAPIRetriesTransientErrors(t *testing.T) {
w := NewWithAPI(config.Config{}, nil)
attempts := 0
err := w.callAPI(context.Background(), "test", func(context.Context) error {
attempts++
if attempts < 3 {
return errors.New("503 Service Unavailable")
}
return nil
})
if err != nil {
t.Fatalf("expected retry to succeed, got %v", err)
}
if attempts != 3 {
t.Fatalf("expected 3 attempts, got %d", attempts)
}
}
func TestCallAPIDoesNotRetryApplicationErrors(t *testing.T) {
w := NewWithAPI(config.Config{}, nil)
attempts := 0
err := w.callAPI(context.Background(), "test", func(context.Context) error {
attempts++
return errors.New("Task is paused")
})
if err == nil {
t.Fatal("expected application error")
}
if attempts != 1 {
t.Fatalf("expected 1 attempt, got %d", attempts)
}
}
downloader/internal/worker/engines.go
+153
View File
@@ -0,0 +1,153 @@
package worker
import (
"context"
"fmt"
"strings"
"time"
"github.com/saltbo/zpan/downloader/internal/config"
"github.com/saltbo/zpan/downloader/internal/engine"
)
func (w *Worker) resolveEngine(ctx context.Context) error {
if w.cfg.Engine == "" || w.cfg.Engine == "auto" {
return w.resolveAutoEngine(ctx)
}
downloader, err := configuredEngine(w.cfg)
if err != nil {
return err
}
w.logger.Info("checking configured downloader engine", "engine", downloader.Name())
if w.checkEngine(ctx, downloader) == nil {
w.useEngine(downloader, "configured engine is already running")
return nil
}
if _, ok := downloader.(engine.Starter); !ok {
w.useEngine(downloader, "configured built-in engine")
return nil
}
if err := w.startEngine(ctx, downloader); err != nil {
w.logger.Warn("configured downloader engine could not be started", "engine", downloader.Name(), "error", err)
w.useEngine(downloader, "configured engine selected despite failed auto start")
return nil
}
w.useEngine(downloader, "configured engine started")
return nil
}
func (w *Worker) resolveAutoEngine(ctx context.Context) error {
candidates := externalEngines(w.cfg)
w.logger.Info("auto selecting downloader engine", "priority", engineNames(candidates))
for _, downloader := range candidates {
w.logger.Info("checking downloader engine availability", "engine", downloader.Name())
if err := w.checkEngine(ctx, downloader); err == nil {
w.useEngine(downloader, "engine is already running")
return nil
} else {
w.logger.Info("downloader engine is not running", "engine", downloader.Name(), "error", err)
}
}
for _, downloader := range candidates {
w.logger.Info("checking downloader engine binary", "engine", downloader.Name())
if err := w.startEngine(ctx, downloader); err != nil {
w.logger.Info("downloader engine is not available for auto start", "engine", downloader.Name(), "error", err)
continue
}
w.useEngine(downloader, "engine binary found and started")
return nil
}
downloader := engine.HTTP{Dir: w.cfg.DownloadDir}
w.useEngine(downloader, "no external downloader engine is running or installed")
return nil
}
func (w *Worker) useEngine(downloader engine.Engine, reason string) {
w.cfg.Engine = downloader.Name()
w.engine = downloader
w.logger.Info("selected downloader engine", "engine", downloader.Name(), "reason", reason)
}
func (w *Worker) checkEngine(ctx context.Context, downloader engine.Engine) error {
checkCtx, cancel := context.WithTimeout(ctx, 2*time.Second)
defer cancel()
return downloader.Check(checkCtx)
}
func (w *Worker) startEngine(ctx context.Context, downloader engine.Engine) error {
starter, ok := downloader.(engine.Starter)
if !ok {
return fmt.Errorf("%s cannot be auto started", downloader.Name())
}
w.logger.Info("starting downloader engine", "engine", downloader.Name())
cmd, err := starter.Start(ctx)
if err != nil {
return err
}
if cmd.Process != nil {
w.logger.Info("downloader engine process started", "engine", downloader.Name(), "pid", cmd.Process.Pid)
}
w.started = append(w.started, cmd)
if err := waitForEngine(ctx, downloader); err != nil {
if cmd.Process != nil {
_ = cmd.Process.Kill()
}
return err
}
return nil
}
func (w *Worker) stopStartedEngines() {
for _, cmd := range w.started {
if cmd.Process == nil {
continue
}
w.logger.Info("stopping auto-started downloader engine", "pid", cmd.Process.Pid)
_ = cmd.Process.Kill()
}
}
func configuredEngine(cfg config.Config) (engine.Engine, error) {
for _, downloader := range append(externalEngines(cfg), engine.HTTP{Dir: cfg.DownloadDir}) {
if downloader.Name() == cfg.Engine {
return downloader, nil
}
}
return nil, fmt.Errorf("unsupported downloader engine %q; expected auto, builtin, aria2, or qbittorrent", cfg.Engine)
}
func externalEngines(cfg config.Config) []engine.Engine {
return []engine.Engine{
engine.Aria2{URL: cfg.Aria2URL, Secret: cfg.Aria2Secret, Dir: cfg.DownloadDir, RetainSeed: cfg.SeedEnabled},
engine.QBittorrent{URL: cfg.QBittorrentURL, Username: cfg.QBittorrentUser, Password: cfg.QBittorrentPass, Dir: cfg.DownloadDir, RetainSeed: cfg.SeedEnabled},
}
}
func waitForEngine(ctx context.Context, downloader engine.Engine) error {
deadline := time.Now().Add(8 * time.Second)
var lastErr error
for time.Now().Before(deadline) {
checkCtx, cancel := context.WithTimeout(ctx, time.Second)
err := downloader.Check(checkCtx)
cancel()
if err == nil {
return nil
}
lastErr = err
select {
case <-ctx.Done():
return ctx.Err()
case <-time.After(500 * time.Millisecond):
}
}
return lastErr
}
func engineNames(engines []engine.Engine) string {
names := make([]string, 0, len(engines)+1)
for _, downloader := range engines {
names = append(names, downloader.Name())
}
names = append(names, "builtin")
return strings.Join(names, ",")
}
downloader/internal/worker/seeds.go
+181
View File
@@ -0,0 +1,181 @@
package worker
import (
"context"
"log/slog"
"os"
"sort"
"time"
"github.com/saltbo/zpan/downloader/internal/client"
"github.com/saltbo/zpan/downloader/internal/engine"
)
const retainedSeedReportInterval = 5 * time.Second
type retainedSeed struct {
taskID string
engine string
seedID string
path string
retainedAt time.Time
expiresAt time.Time
snapshot func(context.Context) (engine.SeedSnapshot, error)
cleanup func(context.Context) error
}
func cleanupDownloadedResult(ctx context.Context, result engine.Result) error {
if result.Seed != nil && result.Seed.Cleanup != nil {
return result.Seed.Cleanup(ctx)
}
return os.RemoveAll(result.Path)
}
func (w *Worker) retainSeed(task client.DownloadTask, result engine.Result, log *slog.Logger) bool {
if !w.cfg.SeedEnabled || result.Seed == nil || result.Seed.Cleanup == nil || result.Seed.Snapshot == nil {
return false
}
now := time.Now()
seed := retainedSeed{
taskID: task.ID,
engine: result.Seed.Engine,
seedID: result.Seed.ID,
path: result.Seed.Path,
retainedAt: now,
snapshot: result.Seed.Snapshot,
cleanup: result.Seed.Cleanup,
}
if w.cfg.SeedDuration > 0 {
seed.expiresAt = now.Add(w.cfg.SeedDuration)
}
w.mu.Lock()
w.retainedSeeds = append(w.retainedSeeds, seed)
count := len(w.retainedSeeds)
w.mu.Unlock()
log.Info("retaining completed bt task for seeding",
"engine", seed.engine,
"seed_id", seed.seedID,
"path", seed.path,
"expires_at", optionalTime(seed.expiresAt),
"retained_seeds", count,
)
return true
}
func (w *Worker) reportRetainedSeeds(ctx context.Context) {
for _, seed := range w.retainedSeedSnapshot() {
log := w.logger.With("task_id", seed.taskID, "engine", seed.engine, "seed_id", seed.seedID)
snapshot, err := seed.snapshot(ctx)
if err != nil {
log.Warn("failed to inspect retained bt seed", "error", err)
continue
}
if snapshot.Detail == nil {
continue
}
snapshot.Detail.Phase = "seeding"
zero := int64(0)
_, err = w.updateTask(ctx, seed.taskID, client.TaskPatch{
DownloadedBytes: &snapshot.Downloaded,
TotalBytes: snapshot.Total,
DownloadBps: &snapshot.Bps,
StorageUploadBps: &zero,
Detail: snapshot.Detail,
})
if err != nil {
log.Warn("failed to report retained bt seed", "error", err)
continue
}
log.Debug("reported retained bt seed", "downloaded_bytes", snapshot.Downloaded, "bps", snapshot.Bps)
}
}
func (w *Worker) cleanupRetainedSeeds(ctx context.Context) {
seeds := w.retainedSeedSnapshot()
if len(seeds) == 0 {
return
}
reasons := map[string]string{}
now := time.Now()
for _, seed := range seeds {
if !seed.expiresAt.IsZero() && !now.Before(seed.expiresAt) {
reasons[seed.taskID] = "expired"
}
}
if w.cfg.SeedCacheLimit > 0 {
type seedSize struct {
seed retainedSeed
size int64
}
sized := make([]seedSize, 0, len(seeds))
var total int64
for _, seed := range seeds {
if reasons[seed.taskID] != "" {
continue
}
size, err := directorySize(seed.path)
if err != nil {
w.logger.Warn("failed to inspect retained seed size", "task_id", seed.taskID, "path", seed.path, "error", err)
continue
}
total += size
sized = append(sized, seedSize{seed: seed, size: size})
}
sort.Slice(sized, func(i, j int) bool {
return sized[i].seed.retainedAt.Before(sized[j].seed.retainedAt)
})
for _, item := range sized {
if total <= w.cfg.SeedCacheLimit {
break
}
reasons[item.seed.taskID] = "cache_limit"
total -= item.size
}
}
for _, seed := range seeds {
reason := reasons[seed.taskID]
if reason == "" {
continue
}
w.cleanupRetainedSeed(ctx, seed, reason)
}
}
func (w *Worker) retainedSeedSnapshot() []retainedSeed {
w.mu.Lock()
defer w.mu.Unlock()
return append([]retainedSeed(nil), w.retainedSeeds...)
}
func (w *Worker) cleanupRetainedSeed(ctx context.Context, seed retainedSeed, reason string) {
w.logger.Info("cleaning retained bt seed",
"task_id", seed.taskID,
"engine", seed.engine,
"seed_id", seed.seedID,
"path", seed.path,
"reason", reason,
)
if err := seed.cleanup(ctx); err != nil {
w.logger.Warn("failed to clean retained bt seed",
"task_id", seed.taskID,
"engine", seed.engine,
"seed_id", seed.seedID,
"path", seed.path,
"error", err,
)
return
}
w.mu.Lock()
next := w.retainedSeeds[:0]
for _, retained := range w.retainedSeeds {
if retained.taskID != seed.taskID {
next = append(next, retained)
}
}
w.retainedSeeds = next
w.mu.Unlock()
}
downloader/internal/worker/uploader.go
+306
View File
@@ -0,0 +1,306 @@
package worker
import (
"context"
"fmt"
"io"
"log/slog"
"net/http"
"os"
"path"
"strings"
"time"
"github.com/saltbo/zpan/downloader/internal/client"
"github.com/saltbo/zpan/downloader/internal/engine"
)
const maxSingleUploadSize = 5 * 1024 * 1024 * 1024
const defaultMultipartPartSize = 64 * 1024 * 1024
const maxMultipartPartSize = 512 * 1024 * 1024
const maxMultipartParts = 10_000
const presignMultipartPartsBatchSize = 100
type uploadProgress struct {
totalBytes int64
uploaded int64
lastAt time.Time
lastBytes int64
}
func (w *Worker) uploadResult(
ctx context.Context,
log *slog.Logger,
task client.DownloadTask,
result engine.Result,
) (string, error) {
if !result.IsDir {
progress := &uploadProgress{totalBytes: result.Size, lastAt: time.Now()}
return w.uploadSingleFile(ctx, log, task, result.Path, result.Name, result.Size, task.TargetFolder, progress)
}
progress := &uploadProgress{totalBytes: result.Size, lastAt: time.Now()}
log.Info("creating remote folder", "name", result.Name, "size", result.Size, "target_folder", task.TargetFolder)
root, err := w.createFolder(ctx, task.UploadToken, result.Name, task.TargetFolder)
if err != nil {
return "", fmt.Errorf("create remote folder: %w", err)
}
rootPath := joinObjectPath(task.TargetFolder, root.Name)
entries, err := collectDirectoryEntries(result.Path)
if err != nil {
return "", err
}
for _, entry := range entries {
parent := joinObjectPath(rootPath, path.Dir(entry.relativePath))
if entry.isDir {
log.Debug("creating remote subfolder", "name", entry.name, "parent", parent)
if _, err := w.createFolder(ctx, task.UploadToken, entry.name, parent); err != nil {
return "", fmt.Errorf("create remote subfolder %s: %w", entry.relativePath, err)
}
continue
}
if _, err := w.uploadSingleFile(ctx, log, task, entry.path, entry.name, entry.size, parent, progress); err != nil {
return "", err
}
}
return root.ID, nil
}
func (w *Worker) uploadSingleFile(
ctx context.Context,
log *slog.Logger,
task client.DownloadTask,
path string,
name string,
size int64,
parent string,
progress *uploadProgress,
) (string, error) {
log.Info("creating remote object", "name", name, "size", size, "target_folder", parent)
draft, err := w.createObject(ctx, task.UploadToken, name, size, parent)
if err != nil {
return "", fmt.Errorf("create remote object: %w", err)
}
log.Info("uploading file to object storage", "object_id", draft.ID, "path", path)
if size > maxSingleUploadSize {
if err := w.uploadMultipartFile(ctx, log, task, draft.ID, path, size, progress); err != nil {
return "", fmt.Errorf("upload object %s: %w", draft.ID, err)
}
} else {
if err := uploadFile(ctx, draft.UploadURL, path, draft.ContentDisposition, func(written int64) error {
return w.reportUploadProgress(ctx, log, task, progress, written)
}); err != nil {
return "", fmt.Errorf("upload object %s: %w", draft.ID, err)
}
}
log.Info("confirming uploaded object", "object_id", draft.ID)
if err := w.confirmObject(ctx, task.UploadToken, draft.ID); err != nil {
return "", fmt.Errorf("confirm object %s: %w", draft.ID, err)
}
return draft.ID, nil
}
func (w *Worker) uploadMultipartFile(
ctx context.Context,
log *slog.Logger,
task client.DownloadTask,
objectID string,
filePath string,
size int64,
progress *uploadProgress,
) error {
partSize := multipartPartSize(size)
log.Info("creating multipart upload session", "object_id", objectID, "part_size", partSize)
session, err := w.createObjectUploadSession(ctx, task.UploadToken, objectID, partSize)
if err != nil {
return fmt.Errorf("create multipart upload session: %w", err)
}
if session.PartSize <= 0 {
return fmt.Errorf("create multipart upload session: invalid part size %d", session.PartSize)
}
completed := false
defer func() {
if completed {
return
}
abortCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second)
defer cancel()
if abortErr := w.abortObjectUploadSession(abortCtx, task.UploadToken, objectID, session.ID); abortErr != nil {
log.Warn("failed to abort multipart upload session", "object_id", objectID, "upload_session_id", session.ID, "error", abortErr)
}
}()
file, err := os.Open(filePath)
if err != nil {
return err
}
defer file.Close()
totalParts := int((size + session.PartSize - 1) / session.PartSize)
parts := make([]client.CompletedObjectUploadPart, 0, totalParts)
for firstPart := 1; firstPart <= totalParts; firstPart += presignMultipartPartsBatchSize {
lastPart := firstPart + presignMultipartPartsBatchSize - 1
if lastPart > totalParts {
lastPart = totalParts
}
partNumbers := make([]int, 0, lastPart-firstPart+1)
for partNumber := firstPart; partNumber <= lastPart; partNumber++ {
partNumbers = append(partNumbers, partNumber)
}
presignedParts, err := w.presignObjectUploadParts(ctx, task.UploadToken, objectID, session.ID, partNumbers)
if err != nil {
return fmt.Errorf("presign multipart upload parts: %w", err)
}
byNumber := make(map[int]string, len(presignedParts))
for _, part := range presignedParts {
byNumber[part.PartNumber] = part.URL
}
for _, partNumber := range partNumbers {
url, ok := byNumber[partNumber]
if !ok {
return fmt.Errorf("presign multipart upload part %d: missing upload URL", partNumber)
}
offset := int64(partNumber-1) * session.PartSize
length := session.PartSize
if remaining := size - offset; remaining < length {
length = remaining
}
etag, err := uploadFilePart(ctx, url, file, offset, length, func(written int64) error {
return w.reportUploadProgress(ctx, log, task, progress, written)
})
if err != nil {
return fmt.Errorf("upload part %d: %w", partNumber, err)
}
if etag == "" {
return fmt.Errorf("upload part %d: missing ETag", partNumber)
}
parts = append(parts, client.CompletedObjectUploadPart{PartNumber: partNumber, ETag: etag})
}
}
if err := w.completeObjectUploadSession(ctx, task.UploadToken, objectID, session.ID, parts); err != nil {
return fmt.Errorf("complete multipart upload session: %w", err)
}
completed = true
return nil
}
func (w *Worker) reportUploadProgress(
ctx context.Context,
log *slog.Logger,
task client.DownloadTask,
progress *uploadProgress,
written int64,
) error {
progress.uploaded += written
now := time.Now()
if progress.uploaded < progress.totalBytes && now.Sub(progress.lastAt) < time.Second {
return nil
}
elapsed := now.Sub(progress.lastAt).Seconds()
var bps int64
if elapsed > 0 {
bps = int64(float64(progress.uploaded-progress.lastBytes) / elapsed)
}
detail := task.Detail
if detail == nil {
detail = &client.DownloadTaskDetail{}
}
detail.Phase = "uploading"
detail.ETASeconds = uploadETA(progress, bps)
detail.PeerUploadBps = nil
zero := int64(0)
_, err := w.updateTask(ctx, task.ID, client.TaskPatch{
Status: "uploading",
StorageUploadedBytes: &progress.uploaded,
DownloadBps: &zero,
StorageUploadBps: &bps,
Detail: detail,
})
if err != nil {
log.Error("failed to report upload progress", "uploaded_bytes", progress.uploaded, "total_bytes", progress.totalBytes, "bps", bps, "error", err)
return err
}
log.Debug("task upload progress", "uploaded_bytes", progress.uploaded, "total_bytes", progress.totalBytes, "bps", bps)
progress.lastAt = now
progress.lastBytes = progress.uploaded
return nil
}
func uploadFile(ctx context.Context, url, path string, contentDisposition string, progress func(written int64) error) error {
file, err := os.Open(path)
if err != nil {
return err
}
defer file.Close()
stat, err := file.Stat()
if err != nil {
return err
}
reader := io.Reader(file)
if progress != nil {
reader = &uploadProgressReader{reader: file, progress: progress}
}
req, err := http.NewRequestWithContext(ctx, http.MethodPut, url, reader)
if err != nil {
return err
}
req.Header.Set("Content-Type", "application/octet-stream")
if contentDisposition != "" {
req.Header.Set("Content-Disposition", contentDisposition)
}
req.ContentLength = stat.Size()
res, err := http.DefaultClient.Do(req)
if err != nil {
return err
}
defer res.Body.Close()
if res.StatusCode < 200 || res.StatusCode >= 300 {
body, _ := io.ReadAll(io.LimitReader(res.Body, 4096))
if len(body) > 0 {
return fmt.Errorf("upload failed: %s: %s", res.Status, strings.TrimSpace(string(body)))
}
return fmt.Errorf("upload failed: %s", res.Status)
}
return nil
}
func uploadFilePart(ctx context.Context, url string, file *os.File, offset int64, length int64, progress func(written int64) error) (string, error) {
reader := io.NewSectionReader(file, offset, length)
var body io.Reader = reader
if progress != nil {
body = &uploadProgressReader{reader: reader, progress: progress}
}
req, err := http.NewRequestWithContext(ctx, http.MethodPut, url, body)
if err != nil {
return "", err
}
req.ContentLength = length
res, err := http.DefaultClient.Do(req)
if err != nil {
return "", err
}
defer res.Body.Close()
if res.StatusCode < 200 || res.StatusCode >= 300 {
body, _ := io.ReadAll(io.LimitReader(res.Body, 4096))
if len(body) > 0 {
return "", fmt.Errorf("upload failed: %s: %s", res.Status, strings.TrimSpace(string(body)))
}
return "", fmt.Errorf("upload failed: %s", res.Status)
}
return res.Header.Get("ETag"), nil
}
type uploadProgressReader struct {
reader io.Reader
progress func(written int64) error
}
func (r *uploadProgressReader) Read(p []byte) (int, error) {
n, err := r.reader.Read(p)
if n > 0 {
if progressErr := r.progress(int64(n)); progressErr != nil {
return n, progressErr
}
}
return n, err
}
downloader/internal/worker/worker.go
+87 -772
View File
File diff suppressed because it is too large Load Diff
downloader/internal/worker/worker_test.go
+4 -4
View File
@@ -18,7 +18,7 @@ import (
)
func TestResolveEngineRejectsUnknownConfiguredEngine(t *testing.T) {
w := New(config.Config{Engine: "bad-engine"})
w := NewWithAPI(config.Config{Engine: "bad-engine"}, nil)
err := w.resolveEngine(context.Background())
if err == nil {
@@ -215,7 +215,7 @@ func TestTaskErrorMessageTruncatesToSchemaLimit(t *testing.T) {
func TestRetainSeedKeepsDownloadedResult(t *testing.T) {
dir := t.TempDir()
cleaned := false
w := New(config.Config{SeedEnabled: true, SeedDuration: time.Hour})
w := NewWithAPI(config.Config{SeedEnabled: true, SeedDuration: time.Hour}, nil)
retained := w.retainSeed(
clientTask("task-1"),
@@ -250,7 +250,7 @@ func TestRetainSeedKeepsDownloadedResult(t *testing.T) {
func TestCleanupRetainedSeedsRemovesExpiredSeed(t *testing.T) {
dir := t.TempDir()
w := New(config.Config{SeedEnabled: true, SeedDuration: time.Hour})
w := NewWithAPI(config.Config{SeedEnabled: true, SeedDuration: time.Hour}, nil)
cleaned := false
w.retainedSeeds = []retainedSeed{{
taskID: "task-1",
@@ -295,7 +295,7 @@ func TestCleanupRetainedSeedsRemovesOldestWhenCacheLimitExceeded(t *testing.T) {
}
var cleaned []string
w := New(config.Config{SeedEnabled: true, SeedCacheLimit: 3})
w := NewWithAPI(config.Config{SeedEnabled: true, SeedCacheLimit: 3}, nil)
w.retainedSeeds = []retainedSeed{
{
taskID: "old",
downloader/oapi-codegen.yaml
+5
View File
@@ -0,0 +1,5 @@
package: openapi
generate:
models: true
client: true
output: internal/openapi/client.gen.go
package.json
+4 -1
View File
@@ -36,7 +36,10 @@
"e2e:cloud:cf": "node scripts/run-cloud-e2e.mjs --runtime cf",
"e2e:archive": "node scripts/run-cloud-e2e.mjs --local --with-s3-mock --spec archive.spec.ts",
"e2e:archive:cf": "node scripts/run-cloud-e2e.mjs --runtime cf --local --with-s3-mock --spec archive.spec.ts",
"openapi:downloader": "tsx scripts/generate-downloader-openapi.ts"
"openapi:downloader": "tsx scripts/generate-downloader-openapi.ts",
"openapi:downloader:go": "pnpm openapi:downloader && cd downloader && go run github.com/oapi-codegen/oapi-codegen/v2/cmd/oapi-codegen@v2.7.0 -config oapi-codegen.yaml ../docs/openapi/downloader.json",
"openapi:downloader:check": "tsx scripts/check-downloader-openapi.ts",
"openapi:downloader:all": "pnpm openapi:downloader:go"
},
"engines": {
"node": ">=24"
scripts/check-downloader-openapi.ts
+77
View File
@@ -0,0 +1,77 @@
import { execFile as execFileCallback } from 'node:child_process'
import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { promisify } from 'node:util'
import { downloaderOpenAPIDocument } from '../server/openapi/downloader'
const execFile = promisify(execFileCallback)
const root = process.cwd()
async function main() {
const tempDir = await mkdtemp(join(tmpdir(), 'zpan-downloader-openapi-'))
try {
const generatedDocPath = join(tempDir, 'downloader.json')
const generatedClientPath = join(tempDir, 'client.gen.go')
const configPath = join(tempDir, 'oapi-codegen.yaml')
await mkdir(join(root, 'docs/openapi'), { recursive: true })
await writeFile(
generatedDocPath,
`${JSON.stringify(downloaderOpenAPIDocument(), null, 2)}\n`,
'utf8',
)
await writeFile(
configPath,
[
'package: openapi',
'generate:',
' models: true',
' client: true',
`output: ${JSON.stringify(generatedClientPath)}`,
'',
].join('\n'),
'utf8',
)
await execFile(
'go',
[
'run',
'github.com/oapi-codegen/oapi-codegen/v2/cmd/oapi-codegen@v2.7.0',
'-config',
configPath,
generatedDocPath,
],
{ cwd: root },
)
await assertSame(
'docs/openapi/downloader.json',
generatedDocPath,
'Downloader OpenAPI document is stale.',
)
await assertSame(
'downloader/internal/openapi/client.gen.go',
generatedClientPath,
'Downloader Go OpenAPI client is stale.',
)
} finally {
await rm(tempDir, { force: true, recursive: true })
}
}
async function assertSame(path: string, generatedPath: string, message: string) {
const actual = await readFile(join(root, path), 'utf8')
const generated = await readFile(generatedPath, 'utf8')
if (actual !== generated) {
console.error(message)
console.error(`Run: pnpm openapi:downloader:go`)
process.exitCode = 1
}
}
main().catch((err) => {
console.error(err)
process.exit(1)
})
server/openapi/downloader.ts
+14 -83
View File
@@ -1,5 +1,14 @@
import { createRoute, OpenAPIHono, z } from '@hono/zod-openapi'
import { createMatterSchema } from '@shared/schemas'
import {
confirmMatterSchema,
createMatterSchema,
createObjectUploadSessionSchema,
objectDraftSchema,
objectUploadSessionSchema,
patchObjectUploadSessionSchema,
presignObjectUploadPartsResponseSchema,
presignObjectUploadPartsSchema,
} from '@shared/schemas'
import downloadTasks from '../routes/download-tasks'
import downloaders, { downloaderSelfRoute } from '../routes/downloaders'
@@ -40,84 +49,6 @@ const deviceTokenSchema = z
})
.openapi('DeviceToken')
const objectDraftSchema = z
.object({
id: z.string(),
name: z.string(),
uploadUrl: z.string().optional(),
contentDisposition: z.string().optional(),
})
.openapi('ObjectDraft')
const confirmObjectRequestSchema = z
.object({
action: z.enum(['confirm']),
onConflict: z.enum(['fail', 'rename']).optional(),
})
.openapi('ConfirmObjectRequest')
const createObjectUploadSessionRequestSchema = z
.object({
partSize: z
.number()
.int()
.min(5 * 1024 * 1024)
.max(512 * 1024 * 1024)
.optional(),
})
.openapi('CreateObjectUploadSessionRequest')
const objectUploadSessionSchema = z
.object({
id: z.string(),
objectId: z.string(),
uploadId: z.string(),
partSize: z.number().int(),
status: z.enum(['active', 'completed', 'aborted']),
expiresAt: z.string(),
createdAt: z.string(),
updatedAt: z.string(),
})
.openapi('ObjectUploadSession')
const presignObjectUploadPartsRequestSchema = z
.object({
partNumbers: z.array(z.number().int().min(1).max(10_000)).min(1).max(100),
})
.openapi('PresignObjectUploadPartsRequest')
const presignedObjectUploadPartSchema = z
.object({
partNumber: z.number().int(),
url: z.string(),
})
.openapi('PresignedObjectUploadPart')
const presignObjectUploadPartsResponseSchema = z
.object({
uploadId: z.string(),
partSize: z.number().int(),
parts: z.array(presignedObjectUploadPartSchema),
})
.openapi('PresignObjectUploadPartsResponse')
const completeObjectUploadSessionRequestSchema = z
.object({
action: z.literal('complete'),
parts: z.array(z.object({ partNumber: z.number().int(), etag: z.string() })).min(1),
})
.openapi('CompleteObjectUploadSessionRequest')
const abortObjectUploadSessionRequestSchema = z
.object({
action: z.literal('abort'),
})
.openapi('AbortObjectUploadSessionRequest')
const patchObjectUploadSessionRequestSchema = z
.union([completeObjectUploadSessionRequestSchema, abortObjectUploadSessionRequestSchema])
.openapi('PatchObjectUploadSessionRequest')
function jsonResponse(schema: z.ZodType, description: string) {
return {
content: {
@@ -194,7 +125,7 @@ function mountObjectUploadRoutes(app: OpenAPIHono) {
request: {
params: z.object({ id: z.string() }),
body: {
content: { 'application/json': { schema: confirmObjectRequestSchema } },
content: { 'application/json': { schema: confirmMatterSchema } },
required: true,
},
},
@@ -214,7 +145,7 @@ function mountObjectUploadRoutes(app: OpenAPIHono) {
request: {
params: z.object({ id: z.string() }),
body: {
content: { 'application/json': { schema: createObjectUploadSessionRequestSchema } },
content: { 'application/json': { schema: createObjectUploadSessionSchema } },
required: true,
},
},
@@ -235,7 +166,7 @@ function mountObjectUploadRoutes(app: OpenAPIHono) {
request: {
params: z.object({ id: z.string(), uploadSessionId: z.string() }),
body: {
content: { 'application/json': { schema: presignObjectUploadPartsRequestSchema } },
content: { 'application/json': { schema: presignObjectUploadPartsSchema } },
required: true,
},
},
@@ -256,7 +187,7 @@ function mountObjectUploadRoutes(app: OpenAPIHono) {
request: {
params: z.object({ id: z.string(), uploadSessionId: z.string() }),
body: {
content: { 'application/json': { schema: patchObjectUploadSessionRequestSchema } },
content: { 'application/json': { schema: patchObjectUploadSessionSchema } },
required: true,
},
},
server/routes/download-tasks.integration.test.ts
+29
View File
@@ -235,6 +235,18 @@ describe('Download tasks API integration', () => {
expect(runningTask.detail.infoHash).toBe('abc123')
expect(runningTask.detail.trackers[0].url).toBe('udp://tracker.example/announce')
const recoverRunningRes = await app.request('/api/download-tasks?assignedTo=me&status=running', {
headers: { Authorization: `Bearer ${createdDownloader.token}` },
})
expect(recoverRunningRes.status).toBe(200)
const recoverRunning = (await recoverRunningRes.json()) as {
items: Array<{ id: string; uploadToken?: string; status: string }>
}
const recoverRunningTask = recoverRunning.items.find((item) => item.id === createdTask.id)
expect(recoverRunningTask?.status).toBe('running')
expect(recoverRunningTask?.uploadToken).toBeTruthy()
uploadHeaders.Authorization = `Bearer ${recoverRunningTask?.uploadToken}`
const createFolderRes = await app.request('/api/objects', {
method: 'POST',
headers: uploadHeaders,
@@ -335,6 +347,23 @@ describe('Download tasks API integration', () => {
const confirmed = (await confirmRes.json()) as { id: string; status: string }
expect(confirmed.status).toBe('active')
const uploadingRes = await app.request(`/api/download-tasks/${createdTask.id}`, {
method: 'PATCH',
headers: downloaderHeaders,
body: JSON.stringify({ status: 'uploading', downloadedBytes: 10 * 1024 * 1024 }),
})
expect(uploadingRes.status).toBe(200)
const recoverUploadingRes = await app.request('/api/download-tasks?assignedTo=me&status=uploading', {
headers: { Authorization: `Bearer ${createdDownloader.token}` },
})
expect(recoverUploadingRes.status).toBe(200)
const recoverUploading = (await recoverUploadingRes.json()) as {
items: Array<{ id: string; uploadToken?: string; status: string }>
}
const recoverUploadingTask = recoverUploading.items.find((item) => item.id === createdTask.id)
expect(recoverUploadingTask?.status).toBe('uploading')
expect(recoverUploadingTask?.uploadToken).toBeTruthy()
const completeTaskRes = await app.request(`/api/download-tasks/${createdTask.id}`, {
method: 'PATCH',
headers: downloaderHeaders,
server/routes/download-tasks.ts
+2 -48
View File
@@ -2,7 +2,8 @@ import { createRoute, OpenAPIHono, z } from '@hono/zod-openapi'
import {
createDownloadTaskSchema,
downloadTaskActionInputSchema,
downloadTaskDetailSchema,
downloadTaskPageSchema,
downloadTaskSchema,
listDownloadTasksQuerySchema,
updateDownloadTaskSchema,
} from '@shared/schemas'
@@ -19,13 +20,6 @@ import {
} from '../services/downloads'
const errorSchema = z.object({ error: z.string() })
const int64Schema = () => z.number().int().openapi({ type: 'integer', format: 'int64' })
const nullableInt64Schema = () =>
z
.number()
.int()
.nullable()
.openapi({ type: 'integer', format: 'int64', nullable: true } as never)
type OpenAPIContext = Context<Env> & {
req: Context<Env>['req'] & {
@@ -34,46 +28,6 @@ type OpenAPIContext = Context<Env> & {
}
}
const downloadTaskSchema = z.object({
id: z.string(),
sourceType: z.enum(['http', 'magnet', 'torrent_url']),
sourceUri: z.string(),
name: z.string(),
targetFolder: z.string(),
category: z.string().nullable(),
tags: z.array(z.string()),
status: z.enum([
'queued',
'assigned',
'running',
'billing_paused',
'pausing',
'paused',
'uploading',
'canceling',
'completed',
'failed',
'canceled',
]),
downloadedBytes: int64Schema(),
storageUploadedBytes: int64Schema(),
totalBytes: nullableInt64Schema(),
downloadBps: int64Schema(),
storageUploadBps: int64Schema(),
errorMessage: z.string().nullable().optional(),
resultObjectId: z.string().nullable().optional(),
detail: downloadTaskDetailSchema.nullable().optional(),
uploadToken: z.string().optional(),
assignedDownloaderId: z.string().nullable().optional(),
})
const downloadTaskPageSchema = z.object({
items: z.array(downloadTaskSchema),
total: z.number().int(),
page: z.number().int(),
pageSize: z.number().int(),
})
const sseEncoder = new TextEncoder()
const downloadTaskEventIntervalMs = 2000
server/routes/downloaders.ts
+9 -39
View File
@@ -1,5 +1,13 @@
import { createRoute, OpenAPIHono, z } from '@hono/zod-openapi'
import { createDownloaderSchema, downloaderHeartbeatSchema, updateDownloaderSchema } from '@shared/schemas'
import {
createDownloaderResponseSchema,
createDownloaderSchema,
deleteDownloaderResponseSchema,
downloaderHeartbeatSchema,
downloaderListSchema,
downloaderSchema,
updateDownloaderSchema,
} from '@shared/schemas'
import type { Context } from 'hono'
import { requireAdmin, requireDownloader } from '../middleware/auth'
import type { Env } from '../middleware/platform'
@@ -13,7 +21,6 @@ import {
} from '../services/downloads'
const errorSchema = z.object({ error: z.string() })
const int64Schema = () => z.number().int().openapi({ type: 'integer', format: 'int64' })
type OpenAPIContext = Context<Env> & {
req: Context<Env>['req'] & {
@@ -22,43 +29,6 @@ type OpenAPIContext = Context<Env> & {
}
}
const downloaderHeartbeatResponseSchema = z.object({
version: z.string(),
hostname: z.string(),
platform: z.string(),
arch: z.string(),
engine: z.enum(['builtin', 'aria2', 'qbittorrent']),
capabilities: z.array(z.string()),
maxConcurrentTasks: z.number().int(),
currentTasks: z.number().int(),
downloadBps: int64Schema(),
uploadBps: int64Schema(),
freeDiskBytes: int64Schema(),
})
const downloaderSchema = z.object({
id: z.string(),
name: z.string().optional(),
status: z.enum(['online', 'offline', 'disabled']).optional(),
enabled: z.boolean().optional(),
heartbeat: downloaderHeartbeatResponseSchema.optional(),
})
const downloaderListSchema = z.object({
items: z.array(downloaderSchema),
total: z.number().int(),
})
const createDownloaderResponseSchema = z.object({
downloader: downloaderSchema,
token: z.string(),
})
const deleteDownloaderResponseSchema = z.object({
id: z.string(),
deleted: z.boolean(),
})
function jsonResponse(schema: z.ZodType, description: string) {
return { content: { 'application/json': { schema } }, description }
}
server/services/downloads.ts
+2 -750
View File
@@ -1,750 +1,2 @@
import type {
CreateDownloaderInput,
CreateDownloadTaskInput,
DownloaderHeartbeatInput,
DownloadTaskActionInput,
UpdateDownloaderInput,
UpdateDownloadTaskInput,
} from '@shared/schemas'
import type { Downloader, DownloadTask } from '@shared/types'
import { and, asc, count, desc, eq, inArray, like, sql } from 'drizzle-orm'
import { nanoid } from 'nanoid'
import { downloaders, downloadTasks } from '../db/schema'
import type { Platform } from '../platform/interface'
import { hashDownloadToken, signDownloadToken } from './download-tokens'
import { RemoteDownloadBillingBlockedError, reportRemoteDownloadUnit } from './remote-download-usage'
const DEFAULT_REMOTE_DOWNLOAD_UNIT_BYTES = 100 * 1024 * 1024
const UPLOAD_TOKEN_TTL_SECONDS = 24 * 60 * 60
const PAUSABLE_TASK_STATUSES = ['queued', 'assigned', 'running'] as const
const CANCELABLE_TASK_STATUSES = [
'queued',
'assigned',
'running',
'billing_paused',
'paused',
'uploading',
'pausing',
] as const
const TERMINAL_TASK_STATUSES = ['completed', 'failed', 'canceled'] as const
export class DownloadError extends Error {
constructor(
readonly code:
| 'not_found'
| 'forbidden'
| 'no_downloader'
| 'invalid_state'
| 'billing_paused'
| 'unsupported_source',
message: string = code,
) {
super(message)
this.name = 'DownloadError'
}
}
type DownloaderRow = typeof downloaders.$inferSelect
type DownloadTaskRow = typeof downloadTasks.$inferSelect
export async function createDownloader(
platform: Platform,
input: CreateDownloaderInput,
userId: string,
): Promise<{ downloader: Downloader; token: string }> {
const now = new Date()
const id = nanoid()
const jti = nanoid()
const token = await signDownloadToken(platform, {
v: 1,
typ: 'downloader',
downloaderId: id,
jti,
iat: Math.floor(now.getTime() / 1000),
})
await platform.db.insert(downloaders).values({
id,
name: input.name,
tokenHash: await hashDownloadToken(platform, token),
tokenJti: jti,
status: 'offline',
enabled: true,
version: input.heartbeat.version,
hostname: input.heartbeat.hostname,
platform: input.heartbeat.platform,
arch: input.heartbeat.arch,
engine: input.heartbeat.engine,
capabilities: JSON.stringify(input.heartbeat.capabilities),
maxConcurrentTasks: input.heartbeat.maxConcurrentTasks,
currentTasks: input.heartbeat.currentTasks,
downloadBps: input.heartbeat.downloadBps,
uploadBps: input.heartbeat.uploadBps,
freeDiskBytes: input.heartbeat.freeDiskBytes,
remoteDownloadCreditBillingEnabled: false,
remoteDownloadCreditUnitBytes: DEFAULT_REMOTE_DOWNLOAD_UNIT_BYTES,
remoteDownloadCreditPerUnit: 1,
lastHeartbeatAt: null,
createdBy: userId,
createdAt: now,
updatedAt: now,
})
return { downloader: await getDownloader(platform, id), token }
}
export async function listDownloaders(platform: Platform): Promise<Downloader[]> {
const rows = await platform.db.select().from(downloaders).orderBy(desc(downloaders.createdAt))
return rows.map(toDownloader)
}
export async function getDownloader(platform: Platform, id: string): Promise<Downloader> {
const rows = await platform.db.select().from(downloaders).where(eq(downloaders.id, id)).limit(1)
if (!rows[0]) throw new DownloadError('not_found')
return toDownloader(rows[0])
}
export async function updateDownloader(
platform: Platform,
id: string,
input: UpdateDownloaderInput,
): Promise<Downloader> {
const rows = await platform.db.select({ id: downloaders.id }).from(downloaders).where(eq(downloaders.id, id)).limit(1)
if (!rows[0]) throw new DownloadError('not_found')
const now = new Date()
await platform.db
.update(downloaders)
.set({
...(input.name !== undefined ? { name: input.name } : {}),
...(input.enabled !== undefined
? { enabled: input.enabled, status: input.enabled ? 'offline' : 'disabled' }
: {}),
...(input.remoteDownloadCreditBillingEnabled !== undefined
? { remoteDownloadCreditBillingEnabled: input.remoteDownloadCreditBillingEnabled }
: {}),
...(input.remoteDownloadCreditUnitBytes !== undefined
? { remoteDownloadCreditUnitBytes: input.remoteDownloadCreditUnitBytes }
: {}),
...(input.remoteDownloadCreditPerUnit !== undefined
? { remoteDownloadCreditPerUnit: input.remoteDownloadCreditPerUnit }
: {}),
updatedAt: now,
})
.where(eq(downloaders.id, id))
return getDownloader(platform, id)
}
export async function deleteDownloader(platform: Platform, id: string): Promise<{ id: string; deleted: true }> {
const rows = await platform.db.select({ id: downloaders.id }).from(downloaders).where(eq(downloaders.id, id)).limit(1)
if (!rows[0]) throw new DownloadError('not_found')
const now = new Date()
await platform.db
.update(downloadTasks)
.set({
status: 'queued',
assignedDownloaderId: null,
uploadTokenHash: null,
uploadTokenJti: null,
uploadTokenExpiresAt: null,
assignedAt: null,
updatedAt: now,
})
.where(
and(
eq(downloadTasks.assignedDownloaderId, id),
inArray(downloadTasks.status, [
'queued',
'assigned',
'running',
'billing_paused',
'pausing',
'paused',
'uploading',
'canceling',
]),
),
)
await platform.db.delete(downloaders).where(eq(downloaders.id, id))
return { id, deleted: true }
}
export async function recordDownloaderHeartbeat(
platform: Platform,
downloaderId: string,
heartbeat: DownloaderHeartbeatInput,
): Promise<Downloader> {
const rows = await platform.db
.select({ id: downloaders.id, enabled: downloaders.enabled })
.from(downloaders)
.where(eq(downloaders.id, downloaderId))
.limit(1)
if (!rows[0]) throw new DownloadError('not_found')
const now = new Date()
await platform.db
.update(downloaders)
.set({
status: rows[0].enabled ? 'online' : 'disabled',
version: heartbeat.version,
hostname: heartbeat.hostname,
platform: heartbeat.platform,
arch: heartbeat.arch,
engine: heartbeat.engine,
capabilities: JSON.stringify(heartbeat.capabilities),
maxConcurrentTasks: heartbeat.maxConcurrentTasks,
currentTasks: heartbeat.currentTasks,
downloadBps: heartbeat.downloadBps,
uploadBps: heartbeat.uploadBps,
freeDiskBytes: heartbeat.freeDiskBytes,
lastHeartbeatAt: now,
updatedAt: now,
})
.where(eq(downloaders.id, downloaderId))
await assignQueuedTasks(platform)
return getDownloader(platform, downloaderId)
}
export async function createDownloadTask(
platform: Platform,
orgId: string,
userId: string,
input: CreateDownloadTaskInput,
): Promise<DownloadTask> {
const now = new Date()
const id = nanoid()
const assigned = await selectDownloader(platform, input.source.type)
const uploadToken = assigned
? await createTaskUploadToken(platform, {
taskId: id,
downloaderId: assigned.id,
orgId,
targetFolder: input.targetFolder,
createdByUserId: userId,
})
: null
await platform.db.insert(downloadTasks).values({
id,
orgId,
createdByUserId: userId,
sourceType: input.source.type,
sourceUri: input.source.uri,
name: input.name ?? null,
targetFolder: input.targetFolder,
category: input.category ?? null,
tags: JSON.stringify(input.tags ?? []),
assignedDownloaderId: assigned?.id ?? null,
status: assigned ? 'assigned' : 'queued',
uploadTokenHash: uploadToken?.hash ?? null,
uploadTokenJti: uploadToken?.jti ?? null,
uploadTokenExpiresAt: uploadToken?.expiresAt ?? null,
createdAt: now,
updatedAt: now,
assignedAt: assigned ? now : null,
})
return getDownloadTask(platform, orgId, id)
}
export async function listDownloadTasks(
platform: Platform,
opts: {
orgId?: string
downloaderId?: string
status?: string
category?: string
tag?: string
sortBy?: 'createdAt' | 'source' | 'category' | 'tags' | 'status' | 'progress' | 'eta'
sortDir?: 'asc' | 'desc'
page: number
pageSize: number
includeUploadToken?: boolean
},
): Promise<{ items: DownloadTask[]; total: number }> {
const offset = (opts.page - 1) * opts.pageSize
const filters = []
if (opts.orgId) filters.push(eq(downloadTasks.orgId, opts.orgId))
if (opts.downloaderId) filters.push(eq(downloadTasks.assignedDownloaderId, opts.downloaderId))
if (opts.status) filters.push(eq(downloadTasks.status, opts.status))
if (opts.category) filters.push(eq(downloadTasks.category, opts.category))
if (opts.tag) filters.push(like(downloadTasks.tags, `%${JSON.stringify(opts.tag)}%`))
const where = filters.length ? and(...filters) : undefined
const [rows, totalRows] = await Promise.all([
platform.db
.select()
.from(downloadTasks)
.where(where)
.orderBy(downloadTaskOrderBy(opts.sortBy ?? 'createdAt', opts.sortDir ?? 'desc'))
.limit(opts.pageSize)
.offset(offset),
platform.db.select({ count: count() }).from(downloadTasks).where(where),
])
return {
items: await Promise.all(
rows.map((row) =>
toDownloadTaskWithToken(
platform,
row,
(opts.includeUploadToken ?? false) && (row.status === 'assigned' || row.status === 'billing_paused'),
),
),
),
total: totalRows[0]?.count ?? 0,
}
}
function downloadTaskOrderBy(
sortBy: 'createdAt' | 'source' | 'category' | 'tags' | 'status' | 'progress' | 'eta',
sortDir: 'asc' | 'desc',
) {
const direction = sortDir === 'asc' ? asc : desc
if (sortBy === 'source') return direction(downloadTasks.sourceUri)
if (sortBy === 'category') return direction(downloadTasks.category)
if (sortBy === 'tags') return direction(downloadTasks.tags)
if (sortBy === 'status') return direction(downloadTasks.status)
if (sortBy === 'progress') {
return direction(sql<number>`
case
when ${downloadTasks.totalBytes} is null or ${downloadTasks.totalBytes} = 0 then 0
else (${downloadTasks.downloadedBytes} * 1000000 / ${downloadTasks.totalBytes})
end
`)
}
if (sortBy === 'eta') {
return direction(sql<number>`coalesce(json_extract(${downloadTasks.detail}, '$.etaSeconds'), 9223372036854775807)`)
}
return direction(downloadTasks.createdAt)
}
export async function getDownloadTask(platform: Platform, orgId: string, id: string): Promise<DownloadTask> {
const rows = await platform.db
.select()
.from(downloadTasks)
.where(and(eq(downloadTasks.id, id), eq(downloadTasks.orgId, orgId)))
.limit(1)
if (!rows[0]) throw new DownloadError('not_found')
return toDownloadTask(rows[0])
}
export async function updateDownloadTask(
platform: Platform,
id: string,
input: UpdateDownloadTaskInput,
actor: { orgId?: string; downloaderId?: string },
): Promise<DownloadTask> {
const rows = await platform.db.select().from(downloadTasks).where(eq(downloadTasks.id, id)).limit(1)
const task = rows[0]
if (!task) throw new DownloadError('not_found')
if (actor.orgId && task.orgId !== actor.orgId) throw new DownloadError('not_found')
if (actor.downloaderId && task.assignedDownloaderId !== actor.downloaderId) throw new DownloadError('forbidden')
if (actor.downloaderId && task.status === 'pausing' && input.status === 'paused') {
const now = new Date()
await platform.db
.update(downloadTasks)
.set({ status: 'paused', downloadBps: 0, uploadBps: 0, updatedAt: now })
.where(eq(downloadTasks.id, id))
return getDownloadTask(platform, task.orgId, id)
}
if (actor.downloaderId && task.status === 'canceling' && input.status === 'canceled') {
const now = new Date()
await platform.db
.update(downloadTasks)
.set({ status: 'canceled', downloadBps: 0, uploadBps: 0, finishedAt: task.finishedAt ?? now, updatedAt: now })
.where(eq(downloadTasks.id, id))
return getDownloadTask(platform, task.orgId, id)
}
if (actor.downloaderId && ['pausing', 'paused', 'canceling', 'canceled'].includes(task.status)) {
throw new DownloadError('invalid_state', `Task is ${task.status}`)
}
if (actor.orgId && !actor.downloaderId) {
const onlyCancel =
input.status === 'canceled' &&
input.downloadedBytes === undefined &&
input.storageUploadedBytes === undefined &&
input.totalBytes === undefined &&
input.downloadBps === undefined &&
input.storageUploadBps === undefined &&
input.errorMessage === undefined &&
input.resultObjectId === undefined &&
input.detail === undefined
if (!onlyCancel) throw new DownloadError('forbidden')
}
const now = new Date()
let status = input.status ?? task.status
let authorizedBytes = task.authorizedBytes
let billedBytes = task.billedBytes
let billedCredits = task.billedCredits
let billingStatus = task.billingStatus
const downloadedBytes =
actor.downloaderId && input.downloadedBytes !== undefined
? Math.max(input.downloadedBytes, task.downloadedBytes)
: (input.downloadedBytes ?? task.downloadedBytes)
const storageUploadedBytes =
actor.downloaderId && input.storageUploadedBytes !== undefined
? Math.max(input.storageUploadedBytes, task.uploadedBytes)
: (input.storageUploadedBytes ?? task.uploadedBytes)
if (actor.downloaderId && downloadedBytes > task.downloadedBytes) {
const downloader = await loadDownloaderRow(platform, actor.downloaderId)
const targetUnits = Math.ceil(downloadedBytes / downloader.remoteDownloadCreditUnitBytes)
const currentUnits = Math.ceil(task.billedBytes / downloader.remoteDownloadCreditUnitBytes)
try {
for (let unit = currentUnits + 1; unit <= targetUnits; unit += 1) {
await reportRemoteDownloadUnit({
platform,
orgId: task.orgId,
downloaderId: actor.downloaderId,
taskId: task.id,
unitIndex: unit,
unitBytes: downloader.remoteDownloadCreditUnitBytes,
creditsPerUnit: downloader.remoteDownloadCreditPerUnit,
enabled: downloader.remoteDownloadCreditBillingEnabled,
})
billedCredits += downloader.remoteDownloadCreditBillingEnabled ? downloader.remoteDownloadCreditPerUnit : 0
}
if (targetUnits > currentUnits) {
billedBytes = targetUnits * downloader.remoteDownloadCreditUnitBytes
authorizedBytes = billedBytes
billingStatus = 'ok'
}
} catch (error) {
if (error instanceof RemoteDownloadBillingBlockedError) {
status = 'billing_paused'
billingStatus = 'insufficient_credits'
} else {
throw error
}
}
}
const nextFinishedAt =
task.finishedAt ?? (input.status !== undefined && ['completed', 'failed', 'canceled'].includes(status) ? now : null)
await platform.db
.update(downloadTasks)
.set({
status,
downloadedBytes,
uploadedBytes: storageUploadedBytes,
totalBytes: input.totalBytes === undefined ? task.totalBytes : input.totalBytes,
authorizedBytes,
billedBytes,
billedCredits,
billingStatus,
downloadBps: input.downloadBps ?? task.downloadBps,
uploadBps: input.storageUploadBps ?? task.uploadBps,
errorMessage: input.errorMessage === undefined ? task.errorMessage : input.errorMessage,
resultObjectId: input.resultObjectId === undefined ? task.resultObjectId : input.resultObjectId,
detail: input.detail === undefined ? task.detail : JSON.stringify(input.detail),
startedAt: task.startedAt ?? (status === 'running' ? now : null),
finishedAt: nextFinishedAt,
updatedAt: now,
})
.where(eq(downloadTasks.id, id))
return getDownloadTask(platform, task.orgId, id)
}
export async function performDownloadTaskAction(
platform: Platform,
orgId: string,
id: string,
action: DownloadTaskActionInput['action'],
): Promise<DownloadTask | { id: string; deleted: true }> {
const rows = await platform.db
.select()
.from(downloadTasks)
.where(and(eq(downloadTasks.id, id), eq(downloadTasks.orgId, orgId)))
.limit(1)
const task = rows[0]
if (!task) throw new DownloadError('not_found')
if (action === 'delete') {
if (!TERMINAL_TASK_STATUSES.includes(task.status as (typeof TERMINAL_TASK_STATUSES)[number])) {
throw new DownloadError('invalid_state', 'Only completed, failed, or canceled tasks can be deleted')
}
await platform.db.delete(downloadTasks).where(eq(downloadTasks.id, id))
return { id, deleted: true }
}
const now = new Date()
if (action === 'pause') {
if (task.status === 'paused') return toDownloadTask(task)
if (!PAUSABLE_TASK_STATUSES.includes(task.status as (typeof PAUSABLE_TASK_STATUSES)[number])) {
throw new DownloadError('invalid_state', 'Only queued, assigned, or running tasks can be paused')
}
const status = task.status === 'running' ? 'pausing' : 'paused'
await platform.db
.update(downloadTasks)
.set({ status, downloadBps: 0, uploadBps: 0, updatedAt: now })
.where(eq(downloadTasks.id, id))
return getDownloadTask(platform, orgId, id)
}
if (action === 'resume') {
if (task.status !== 'paused') throw new DownloadError('invalid_state', 'Only paused tasks can be resumed')
await platform.db
.update(downloadTasks)
.set({
status: 'queued',
assignedDownloaderId: null,
uploadTokenHash: null,
uploadTokenJti: null,
uploadTokenExpiresAt: null,
assignedAt: null,
downloadBps: 0,
uploadBps: 0,
updatedAt: now,
})
.where(eq(downloadTasks.id, id))
await assignQueuedTasks(platform)
return getDownloadTask(platform, orgId, id)
}
if (action === 'cancel') {
if (task.status === 'canceled') return toDownloadTask(task)
if (!CANCELABLE_TASK_STATUSES.includes(task.status as (typeof CANCELABLE_TASK_STATUSES)[number])) {
throw new DownloadError('invalid_state', 'Only active or paused tasks can be canceled')
}
const status =
task.assignedDownloaderId && ['assigned', 'running', 'uploading', 'pausing'].includes(task.status)
? 'canceling'
: 'canceled'
await platform.db
.update(downloadTasks)
.set({
status,
downloadBps: 0,
uploadBps: 0,
finishedAt: status === 'canceled' ? (task.finishedAt ?? now) : task.finishedAt,
updatedAt: now,
})
.where(eq(downloadTasks.id, id))
return getDownloadTask(platform, orgId, id)
}
if (action === 'retry') {
if (!['failed', 'canceled'].includes(task.status)) {
throw new DownloadError('invalid_state', 'Only failed or canceled tasks can be retried')
}
await platform.db
.update(downloadTasks)
.set({
status: 'queued',
assignedDownloaderId: null,
uploadTokenHash: null,
uploadTokenJti: null,
uploadTokenExpiresAt: null,
downloadedBytes: 0,
uploadedBytes: 0,
totalBytes: null,
downloadBps: 0,
uploadBps: 0,
errorMessage: null,
resultObjectId: null,
detail: null,
assignedAt: null,
startedAt: null,
finishedAt: null,
updatedAt: now,
})
.where(eq(downloadTasks.id, id))
await assignQueuedTasks(platform)
return getDownloadTask(platform, orgId, id)
}
throw new DownloadError('invalid_state')
}
export async function assertTaskUploadAllowed(platform: Platform, params: { taskId: string; downloaderId: string }) {
const rows = await platform.db.select().from(downloadTasks).where(eq(downloadTasks.id, params.taskId)).limit(1)
const task = rows[0]
if (!task || task.assignedDownloaderId !== params.downloaderId) throw new DownloadError('forbidden')
if (!['assigned', 'running', 'uploading'].includes(task.status)) throw new DownloadError('invalid_state')
return task
}
async function assignQueuedTasks(platform: Platform): Promise<void> {
const tasks = await platform.db
.select()
.from(downloadTasks)
.where(eq(downloadTasks.status, 'queued'))
.orderBy(asc(downloadTasks.createdAt))
.limit(20)
for (const task of tasks) {
const downloader = await selectDownloader(platform, task.sourceType)
if (!downloader) continue
const token = await createTaskUploadToken(platform, {
taskId: task.id,
downloaderId: downloader.id,
orgId: task.orgId,
targetFolder: task.targetFolder,
createdByUserId: task.createdByUserId,
})
const now = new Date()
await platform.db
.update(downloadTasks)
.set({
status: 'assigned',
assignedDownloaderId: downloader.id,
uploadTokenHash: token.hash,
uploadTokenJti: token.jti,
uploadTokenExpiresAt: token.expiresAt,
assignedAt: now,
updatedAt: now,
})
.where(eq(downloadTasks.id, task.id))
}
}
async function selectDownloader(platform: Platform, sourceType: string): Promise<DownloaderRow | null> {
const needed = sourceType === 'http' ? ['http'] : ['magnet', 'torrent']
const rows = await platform.db
.select()
.from(downloaders)
.where(and(eq(downloaders.enabled, true), eq(downloaders.status, 'online')))
.orderBy(asc(downloaders.currentTasks), asc(downloaders.downloadBps))
return (
rows.find((row) => {
const capabilities = parseCapabilities(row.capabilities)
return needed.some((capability) => capabilities.includes(capability))
}) ?? null
)
}
async function createTaskUploadToken(
platform: Platform,
params: { taskId: string; downloaderId: string; orgId: string; targetFolder: string; createdByUserId: string },
) {
const now = Math.floor(Date.now() / 1000)
const jti = nanoid()
const expiresAt = new Date((now + UPLOAD_TOKEN_TTL_SECONDS) * 1000)
const token = await signDownloadToken(platform, {
v: 1,
typ: 'download-task-upload',
taskId: params.taskId,
downloaderId: params.downloaderId,
orgId: params.orgId,
targetFolder: params.targetFolder,
createdByUserId: params.createdByUserId,
scopes: ['objects:create', 'objects:upload', 'objects:confirm'],
jti,
iat: now,
exp: now + UPLOAD_TOKEN_TTL_SECONDS,
})
return { token, hash: await hashDownloadToken(platform, token), jti, expiresAt }
}
async function toDownloadTaskWithToken(platform: Platform, row: DownloadTaskRow, includeUploadToken: boolean) {
const task = toDownloadTask(row)
if (includeUploadToken && row.assignedDownloaderId && row.uploadTokenJti) {
const token = await createTaskUploadToken(platform, {
taskId: row.id,
downloaderId: row.assignedDownloaderId,
orgId: row.orgId,
targetFolder: row.targetFolder,
createdByUserId: row.createdByUserId,
})
await platform.db
.update(downloadTasks)
.set({ uploadTokenHash: token.hash, uploadTokenJti: token.jti, uploadTokenExpiresAt: token.expiresAt })
.where(eq(downloadTasks.id, row.id))
task.uploadToken = token.token
}
return task
}
async function loadDownloaderRow(platform: Platform, id: string): Promise<DownloaderRow> {
const rows = await platform.db.select().from(downloaders).where(eq(downloaders.id, id)).limit(1)
if (!rows[0]) throw new DownloadError('not_found')
return rows[0]
}
function toDownloader(row: DownloaderRow): Downloader {
return {
id: row.id,
name: row.name,
status: row.enabled ? (row.status as Downloader['status']) : 'disabled',
enabled: row.enabled,
version: row.version,
hostname: row.hostname,
platform: row.platform,
arch: row.arch,
engine: row.engine as Downloader['engine'],
capabilities: parseCapabilities(row.capabilities),
maxConcurrentTasks: row.maxConcurrentTasks,
currentTasks: row.currentTasks,
downloadBps: row.downloadBps,
uploadBps: row.uploadBps,
freeDiskBytes: row.freeDiskBytes,
remoteDownloadCreditBillingEnabled: row.remoteDownloadCreditBillingEnabled,
remoteDownloadCreditUnitBytes: row.remoteDownloadCreditUnitBytes,
remoteDownloadCreditPerUnit: row.remoteDownloadCreditPerUnit,
lastHeartbeatAt: row.lastHeartbeatAt?.toISOString() ?? null,
createdBy: row.createdBy,
createdAt: row.createdAt.toISOString(),
updatedAt: row.updatedAt.toISOString(),
}
}
function toDownloadTask(row: DownloadTaskRow): DownloadTask {
return {
id: row.id,
orgId: row.orgId,
createdByUserId: row.createdByUserId,
sourceType: row.sourceType as DownloadTask['sourceType'],
sourceUri: row.sourceUri,
name: row.name,
targetFolder: row.targetFolder,
category: row.category,
tags: parseTaskTags(row.tags),
assignedDownloaderId: row.assignedDownloaderId,
status: row.status as DownloadTask['status'],
downloadedBytes: row.downloadedBytes,
storageUploadedBytes: row.uploadedBytes,
totalBytes: row.totalBytes,
authorizedBytes: row.authorizedBytes,
billedBytes: row.billedBytes,
billedCredits: row.billedCredits,
billingStatus: row.billingStatus,
downloadBps: row.downloadBps,
storageUploadBps: row.uploadBps,
errorMessage: row.errorMessage,
resultObjectId: row.resultObjectId,
detail: parseTaskDetail(row.detail),
createdAt: row.createdAt.toISOString(),
updatedAt: row.updatedAt.toISOString(),
assignedAt: row.assignedAt?.toISOString() ?? null,
startedAt: row.startedAt?.toISOString() ?? null,
finishedAt: row.finishedAt?.toISOString() ?? null,
}
}
function parseTaskDetail(value: string | null): DownloadTask['detail'] {
if (!value) return null
try {
const detail = JSON.parse(value) as DownloadTask['detail'] & { uploadedBytes?: number }
if (detail.peerUploadedBytes === undefined && detail.uploadedBytes !== undefined) {
detail.peerUploadedBytes = detail.uploadedBytes
}
delete detail.uploadedBytes
return detail
} catch {
return null
}
}
function parseTaskTags(value: string): string[] {
try {
const parsed = JSON.parse(value)
return Array.isArray(parsed) ? parsed.filter((item): item is string => typeof item === 'string') : []
} catch {
return []
}
}
function parseCapabilities(value: string): string[] {
try {
const parsed = JSON.parse(value)
return Array.isArray(parsed) ? parsed.filter((item): item is string => typeof item === 'string') : []
} catch {
return []
}
}
export * from './downloads/core'
export * from './downloads/types'
server/services/downloads/core.ts
+643
View File
@@ -0,0 +1,643 @@
import type {
CreateDownloaderInput,
CreateDownloadTaskInput,
DownloaderHeartbeatInput,
DownloadTaskActionInput,
UpdateDownloaderInput,
UpdateDownloadTaskInput,
} from '@shared/schemas'
import type { Downloader, DownloadTask } from '@shared/types'
import { and, asc, count, desc, eq, inArray, like, sql } from 'drizzle-orm'
import { nanoid } from 'nanoid'
import { downloaders, downloadTasks } from '../../db/schema'
import type { Platform } from '../../platform/interface'
import { hashDownloadToken, signDownloadToken } from '../download-tokens'
import { RemoteDownloadBillingBlockedError, reportRemoteDownloadUnit } from '../remote-download-usage'
import { parseCapabilities, toDownloader, toDownloadTask } from './mappers'
import { DownloadError, type DownloaderRow, type DownloadTaskRow } from './types'
const DEFAULT_REMOTE_DOWNLOAD_UNIT_BYTES = 100 * 1024 * 1024
const UPLOAD_TOKEN_TTL_SECONDS = 24 * 60 * 60
const PAUSABLE_TASK_STATUSES = ['queued', 'assigned', 'running'] as const
const CANCELABLE_TASK_STATUSES = [
'queued',
'assigned',
'running',
'billing_paused',
'paused',
'uploading',
'pausing',
] as const
const TERMINAL_TASK_STATUSES = ['completed', 'failed', 'canceled'] as const
const DOWNLOADER_TOKEN_TASK_STATUSES = ['assigned', 'running', 'uploading', 'billing_paused'] as const
export async function createDownloader(
platform: Platform,
input: CreateDownloaderInput,
userId: string,
): Promise<{ downloader: Downloader; token: string }> {
const now = new Date()
const id = nanoid()
const jti = nanoid()
const token = await signDownloadToken(platform, {
v: 1,
typ: 'downloader',
downloaderId: id,
jti,
iat: Math.floor(now.getTime() / 1000),
})
await platform.db.insert(downloaders).values({
id,
name: input.name,
tokenHash: await hashDownloadToken(platform, token),
tokenJti: jti,
status: 'offline',
enabled: true,
version: input.heartbeat.version,
hostname: input.heartbeat.hostname,
platform: input.heartbeat.platform,
arch: input.heartbeat.arch,
engine: input.heartbeat.engine,
capabilities: JSON.stringify(input.heartbeat.capabilities),
maxConcurrentTasks: input.heartbeat.maxConcurrentTasks,
currentTasks: input.heartbeat.currentTasks,
downloadBps: input.heartbeat.downloadBps,
uploadBps: input.heartbeat.uploadBps,
freeDiskBytes: input.heartbeat.freeDiskBytes,
remoteDownloadCreditBillingEnabled: false,
remoteDownloadCreditUnitBytes: DEFAULT_REMOTE_DOWNLOAD_UNIT_BYTES,
remoteDownloadCreditPerUnit: 1,
lastHeartbeatAt: null,
createdBy: userId,
createdAt: now,
updatedAt: now,
})
return { downloader: await getDownloader(platform, id), token }
}
export async function listDownloaders(platform: Platform): Promise<Downloader[]> {
const rows = await platform.db.select().from(downloaders).orderBy(desc(downloaders.createdAt))
return rows.map(toDownloader)
}
export async function getDownloader(platform: Platform, id: string): Promise<Downloader> {
const rows = await platform.db.select().from(downloaders).where(eq(downloaders.id, id)).limit(1)
if (!rows[0]) throw new DownloadError('not_found')
return toDownloader(rows[0])
}
export async function updateDownloader(
platform: Platform,
id: string,
input: UpdateDownloaderInput,
): Promise<Downloader> {
const rows = await platform.db.select({ id: downloaders.id }).from(downloaders).where(eq(downloaders.id, id)).limit(1)
if (!rows[0]) throw new DownloadError('not_found')
const now = new Date()
await platform.db
.update(downloaders)
.set({
...(input.name !== undefined ? { name: input.name } : {}),
...(input.enabled !== undefined
? { enabled: input.enabled, status: input.enabled ? 'offline' : 'disabled' }
: {}),
...(input.remoteDownloadCreditBillingEnabled !== undefined
? { remoteDownloadCreditBillingEnabled: input.remoteDownloadCreditBillingEnabled }
: {}),
...(input.remoteDownloadCreditUnitBytes !== undefined
? { remoteDownloadCreditUnitBytes: input.remoteDownloadCreditUnitBytes }
: {}),
...(input.remoteDownloadCreditPerUnit !== undefined
? { remoteDownloadCreditPerUnit: input.remoteDownloadCreditPerUnit }
: {}),
updatedAt: now,
})
.where(eq(downloaders.id, id))
return getDownloader(platform, id)
}
export async function deleteDownloader(platform: Platform, id: string): Promise<{ id: string; deleted: true }> {
const rows = await platform.db.select({ id: downloaders.id }).from(downloaders).where(eq(downloaders.id, id)).limit(1)
if (!rows[0]) throw new DownloadError('not_found')
const now = new Date()
await platform.db
.update(downloadTasks)
.set({
status: 'queued',
assignedDownloaderId: null,
uploadTokenHash: null,
uploadTokenJti: null,
uploadTokenExpiresAt: null,
assignedAt: null,
updatedAt: now,
})
.where(
and(
eq(downloadTasks.assignedDownloaderId, id),
inArray(downloadTasks.status, [
'queued',
'assigned',
'running',
'billing_paused',
'pausing',
'paused',
'uploading',
'canceling',
]),
),
)
await platform.db.delete(downloaders).where(eq(downloaders.id, id))
return { id, deleted: true }
}
export async function recordDownloaderHeartbeat(
platform: Platform,
downloaderId: string,
heartbeat: DownloaderHeartbeatInput,
): Promise<Downloader> {
const rows = await platform.db
.select({ id: downloaders.id, enabled: downloaders.enabled })
.from(downloaders)
.where(eq(downloaders.id, downloaderId))
.limit(1)
if (!rows[0]) throw new DownloadError('not_found')
const now = new Date()
await platform.db
.update(downloaders)
.set({
status: rows[0].enabled ? 'online' : 'disabled',
version: heartbeat.version,
hostname: heartbeat.hostname,
platform: heartbeat.platform,
arch: heartbeat.arch,
engine: heartbeat.engine,
capabilities: JSON.stringify(heartbeat.capabilities),
maxConcurrentTasks: heartbeat.maxConcurrentTasks,
currentTasks: heartbeat.currentTasks,
downloadBps: heartbeat.downloadBps,
uploadBps: heartbeat.uploadBps,
freeDiskBytes: heartbeat.freeDiskBytes,
lastHeartbeatAt: now,
updatedAt: now,
})
.where(eq(downloaders.id, downloaderId))
await assignQueuedTasks(platform)
return getDownloader(platform, downloaderId)
}
export async function createDownloadTask(
platform: Platform,
orgId: string,
userId: string,
input: CreateDownloadTaskInput,
): Promise<DownloadTask> {
const now = new Date()
const id = nanoid()
const assigned = await selectDownloader(platform, input.source.type)
const uploadToken = assigned
? await createTaskUploadToken(platform, {
taskId: id,
downloaderId: assigned.id,
orgId,
targetFolder: input.targetFolder,
createdByUserId: userId,
})
: null
await platform.db.insert(downloadTasks).values({
id,
orgId,
createdByUserId: userId,
sourceType: input.source.type,
sourceUri: input.source.uri,
name: input.name ?? null,
targetFolder: input.targetFolder,
category: input.category ?? null,
tags: JSON.stringify(input.tags ?? []),
assignedDownloaderId: assigned?.id ?? null,
status: assigned ? 'assigned' : 'queued',
uploadTokenHash: uploadToken?.hash ?? null,
uploadTokenJti: uploadToken?.jti ?? null,
uploadTokenExpiresAt: uploadToken?.expiresAt ?? null,
createdAt: now,
updatedAt: now,
assignedAt: assigned ? now : null,
})
return getDownloadTask(platform, orgId, id)
}
export async function listDownloadTasks(
platform: Platform,
opts: {
orgId?: string
downloaderId?: string
status?: string
category?: string
tag?: string
sortBy?: 'createdAt' | 'source' | 'category' | 'tags' | 'status' | 'progress' | 'eta'
sortDir?: 'asc' | 'desc'
page: number
pageSize: number
includeUploadToken?: boolean
},
): Promise<{ items: DownloadTask[]; total: number }> {
const offset = (opts.page - 1) * opts.pageSize
const filters = []
if (opts.orgId) filters.push(eq(downloadTasks.orgId, opts.orgId))
if (opts.downloaderId) filters.push(eq(downloadTasks.assignedDownloaderId, opts.downloaderId))
if (opts.status) filters.push(eq(downloadTasks.status, opts.status))
if (opts.category) filters.push(eq(downloadTasks.category, opts.category))
if (opts.tag) filters.push(like(downloadTasks.tags, `%${JSON.stringify(opts.tag)}%`))
const where = filters.length ? and(...filters) : undefined
const [rows, totalRows] = await Promise.all([
platform.db
.select()
.from(downloadTasks)
.where(where)
.orderBy(downloadTaskOrderBy(opts.sortBy ?? 'createdAt', opts.sortDir ?? 'desc'))
.limit(opts.pageSize)
.offset(offset),
platform.db.select({ count: count() }).from(downloadTasks).where(where),
])
return {
items: await Promise.all(
rows.map((row) =>
toDownloadTaskWithToken(
platform,
row,
(opts.includeUploadToken ?? false) &&
DOWNLOADER_TOKEN_TASK_STATUSES.includes(row.status as (typeof DOWNLOADER_TOKEN_TASK_STATUSES)[number]),
),
),
),
total: totalRows[0]?.count ?? 0,
}
}
function downloadTaskOrderBy(
sortBy: 'createdAt' | 'source' | 'category' | 'tags' | 'status' | 'progress' | 'eta',
sortDir: 'asc' | 'desc',
) {
const direction = sortDir === 'asc' ? asc : desc
if (sortBy === 'source') return direction(downloadTasks.sourceUri)
if (sortBy === 'category') return direction(downloadTasks.category)
if (sortBy === 'tags') return direction(downloadTasks.tags)
if (sortBy === 'status') return direction(downloadTasks.status)
if (sortBy === 'progress') {
return direction(sql<number>`
case
when ${downloadTasks.totalBytes} is null or ${downloadTasks.totalBytes} = 0 then 0
else (${downloadTasks.downloadedBytes} * 1000000 / ${downloadTasks.totalBytes})
end
`)
}
if (sortBy === 'eta') {
return direction(sql<number>`coalesce(json_extract(${downloadTasks.detail}, '$.etaSeconds'), 9223372036854775807)`)
}
return direction(downloadTasks.createdAt)
}
export async function getDownloadTask(platform: Platform, orgId: string, id: string): Promise<DownloadTask> {
const rows = await platform.db
.select()
.from(downloadTasks)
.where(and(eq(downloadTasks.id, id), eq(downloadTasks.orgId, orgId)))
.limit(1)
if (!rows[0]) throw new DownloadError('not_found')
return toDownloadTask(rows[0])
}
export async function updateDownloadTask(
platform: Platform,
id: string,
input: UpdateDownloadTaskInput,
actor: { orgId?: string; downloaderId?: string },
): Promise<DownloadTask> {
const rows = await platform.db.select().from(downloadTasks).where(eq(downloadTasks.id, id)).limit(1)
const task = rows[0]
if (!task) throw new DownloadError('not_found')
if (actor.orgId && task.orgId !== actor.orgId) throw new DownloadError('not_found')
if (actor.downloaderId && task.assignedDownloaderId !== actor.downloaderId) throw new DownloadError('forbidden')
if (actor.downloaderId && task.status === 'pausing' && input.status === 'paused') {
const now = new Date()
await platform.db
.update(downloadTasks)
.set({ status: 'paused', downloadBps: 0, uploadBps: 0, updatedAt: now })
.where(eq(downloadTasks.id, id))
return getDownloadTask(platform, task.orgId, id)
}
if (actor.downloaderId && task.status === 'canceling' && input.status === 'canceled') {
const now = new Date()
await platform.db
.update(downloadTasks)
.set({ status: 'canceled', downloadBps: 0, uploadBps: 0, finishedAt: task.finishedAt ?? now, updatedAt: now })
.where(eq(downloadTasks.id, id))
return getDownloadTask(platform, task.orgId, id)
}
if (actor.downloaderId && ['pausing', 'paused', 'canceling', 'canceled'].includes(task.status)) {
throw new DownloadError('invalid_state', `Task is ${task.status}`)
}
if (actor.orgId && !actor.downloaderId) {
const onlyCancel =
input.status === 'canceled' &&
input.downloadedBytes === undefined &&
input.storageUploadedBytes === undefined &&
input.totalBytes === undefined &&
input.downloadBps === undefined &&
input.storageUploadBps === undefined &&
input.errorMessage === undefined &&
input.resultObjectId === undefined &&
input.detail === undefined
if (!onlyCancel) throw new DownloadError('forbidden')
}
const now = new Date()
let status = input.status ?? task.status
let authorizedBytes = task.authorizedBytes
let billedBytes = task.billedBytes
let billedCredits = task.billedCredits
let billingStatus = task.billingStatus
const downloadedBytes =
actor.downloaderId && input.downloadedBytes !== undefined
? Math.max(input.downloadedBytes, task.downloadedBytes)
: (input.downloadedBytes ?? task.downloadedBytes)
const storageUploadedBytes =
actor.downloaderId && input.storageUploadedBytes !== undefined
? Math.max(input.storageUploadedBytes, task.uploadedBytes)
: (input.storageUploadedBytes ?? task.uploadedBytes)
if (actor.downloaderId && downloadedBytes > task.downloadedBytes) {
const downloader = await loadDownloaderRow(platform, actor.downloaderId)
const targetUnits = Math.ceil(downloadedBytes / downloader.remoteDownloadCreditUnitBytes)
const currentUnits = Math.ceil(task.billedBytes / downloader.remoteDownloadCreditUnitBytes)
try {
for (let unit = currentUnits + 1; unit <= targetUnits; unit += 1) {
await reportRemoteDownloadUnit({
platform,
orgId: task.orgId,
downloaderId: actor.downloaderId,
taskId: task.id,
unitIndex: unit,
unitBytes: downloader.remoteDownloadCreditUnitBytes,
creditsPerUnit: downloader.remoteDownloadCreditPerUnit,
enabled: downloader.remoteDownloadCreditBillingEnabled,
})
billedCredits += downloader.remoteDownloadCreditBillingEnabled ? downloader.remoteDownloadCreditPerUnit : 0
}
if (targetUnits > currentUnits) {
billedBytes = targetUnits * downloader.remoteDownloadCreditUnitBytes
authorizedBytes = billedBytes
billingStatus = 'ok'
}
} catch (error) {
if (error instanceof RemoteDownloadBillingBlockedError) {
status = 'billing_paused'
billingStatus = 'insufficient_credits'
} else {
throw error
}
}
}
const nextFinishedAt =
task.finishedAt ?? (input.status !== undefined && ['completed', 'failed', 'canceled'].includes(status) ? now : null)
await platform.db
.update(downloadTasks)
.set({
status,
downloadedBytes,
uploadedBytes: storageUploadedBytes,
totalBytes: input.totalBytes === undefined ? task.totalBytes : input.totalBytes,
authorizedBytes,
billedBytes,
billedCredits,
billingStatus,
downloadBps: input.downloadBps ?? task.downloadBps,
uploadBps: input.storageUploadBps ?? task.uploadBps,
errorMessage: input.errorMessage === undefined ? task.errorMessage : input.errorMessage,
resultObjectId: input.resultObjectId === undefined ? task.resultObjectId : input.resultObjectId,
detail: input.detail === undefined ? task.detail : JSON.stringify(input.detail),
startedAt: task.startedAt ?? (status === 'running' ? now : null),
finishedAt: nextFinishedAt,
updatedAt: now,
})
.where(eq(downloadTasks.id, id))
return getDownloadTask(platform, task.orgId, id)
}
export async function performDownloadTaskAction(
platform: Platform,
orgId: string,
id: string,
action: DownloadTaskActionInput['action'],
): Promise<DownloadTask | { id: string; deleted: true }> {
const rows = await platform.db
.select()
.from(downloadTasks)
.where(and(eq(downloadTasks.id, id), eq(downloadTasks.orgId, orgId)))
.limit(1)
const task = rows[0]
if (!task) throw new DownloadError('not_found')
if (action === 'delete') {
if (!TERMINAL_TASK_STATUSES.includes(task.status as (typeof TERMINAL_TASK_STATUSES)[number])) {
throw new DownloadError('invalid_state', 'Only completed, failed, or canceled tasks can be deleted')
}
await platform.db.delete(downloadTasks).where(eq(downloadTasks.id, id))
return { id, deleted: true }
}
const now = new Date()
if (action === 'pause') {
if (task.status === 'paused') return toDownloadTask(task)
if (!PAUSABLE_TASK_STATUSES.includes(task.status as (typeof PAUSABLE_TASK_STATUSES)[number])) {
throw new DownloadError('invalid_state', 'Only queued, assigned, or running tasks can be paused')
}
const status = task.status === 'running' ? 'pausing' : 'paused'
await platform.db
.update(downloadTasks)
.set({ status, downloadBps: 0, uploadBps: 0, updatedAt: now })
.where(eq(downloadTasks.id, id))
return getDownloadTask(platform, orgId, id)
}
if (action === 'resume') {
if (task.status !== 'paused') throw new DownloadError('invalid_state', 'Only paused tasks can be resumed')
await platform.db
.update(downloadTasks)
.set({
status: 'queued',
assignedDownloaderId: null,
uploadTokenHash: null,
uploadTokenJti: null,
uploadTokenExpiresAt: null,
assignedAt: null,
downloadBps: 0,
uploadBps: 0,
updatedAt: now,
})
.where(eq(downloadTasks.id, id))
await assignQueuedTasks(platform)
return getDownloadTask(platform, orgId, id)
}
if (action === 'cancel') {
if (task.status === 'canceled') return toDownloadTask(task)
if (!CANCELABLE_TASK_STATUSES.includes(task.status as (typeof CANCELABLE_TASK_STATUSES)[number])) {
throw new DownloadError('invalid_state', 'Only active or paused tasks can be canceled')
}
const status =
task.assignedDownloaderId && ['assigned', 'running', 'uploading', 'pausing'].includes(task.status)
? 'canceling'
: 'canceled'
await platform.db
.update(downloadTasks)
.set({
status,
downloadBps: 0,
uploadBps: 0,
finishedAt: status === 'canceled' ? (task.finishedAt ?? now) : task.finishedAt,
updatedAt: now,
})
.where(eq(downloadTasks.id, id))
return getDownloadTask(platform, orgId, id)
}
if (action === 'retry') {
if (!['failed', 'canceled'].includes(task.status)) {
throw new DownloadError('invalid_state', 'Only failed or canceled tasks can be retried')
}
await platform.db
.update(downloadTasks)
.set({
status: 'queued',
assignedDownloaderId: null,
uploadTokenHash: null,
uploadTokenJti: null,
uploadTokenExpiresAt: null,
downloadedBytes: 0,
uploadedBytes: 0,
totalBytes: null,
downloadBps: 0,
uploadBps: 0,
errorMessage: null,
resultObjectId: null,
detail: null,
assignedAt: null,
startedAt: null,
finishedAt: null,
updatedAt: now,
})
.where(eq(downloadTasks.id, id))
await assignQueuedTasks(platform)
return getDownloadTask(platform, orgId, id)
}
throw new DownloadError('invalid_state')
}
export async function assertTaskUploadAllowed(platform: Platform, params: { taskId: string; downloaderId: string }) {
const rows = await platform.db.select().from(downloadTasks).where(eq(downloadTasks.id, params.taskId)).limit(1)
const task = rows[0]
if (!task || task.assignedDownloaderId !== params.downloaderId) throw new DownloadError('forbidden')
if (!['assigned', 'running', 'uploading'].includes(task.status)) throw new DownloadError('invalid_state')
return task
}
async function assignQueuedTasks(platform: Platform): Promise<void> {
const tasks = await platform.db
.select()
.from(downloadTasks)
.where(eq(downloadTasks.status, 'queued'))
.orderBy(asc(downloadTasks.createdAt))
.limit(20)
for (const task of tasks) {
const downloader = await selectDownloader(platform, task.sourceType)
if (!downloader) continue
const token = await createTaskUploadToken(platform, {
taskId: task.id,
downloaderId: downloader.id,
orgId: task.orgId,
targetFolder: task.targetFolder,
createdByUserId: task.createdByUserId,
})
const now = new Date()
await platform.db
.update(downloadTasks)
.set({
status: 'assigned',
assignedDownloaderId: downloader.id,
uploadTokenHash: token.hash,
uploadTokenJti: token.jti,
uploadTokenExpiresAt: token.expiresAt,
assignedAt: now,
updatedAt: now,
})
.where(eq(downloadTasks.id, task.id))
}
}
async function selectDownloader(platform: Platform, sourceType: string): Promise<DownloaderRow | null> {
const needed = sourceType === 'http' ? ['http'] : ['magnet', 'torrent']
const rows = await platform.db
.select()
.from(downloaders)
.where(and(eq(downloaders.enabled, true), eq(downloaders.status, 'online')))
.orderBy(asc(downloaders.currentTasks), asc(downloaders.downloadBps))
return (
rows.find((row) => {
const capabilities = parseCapabilities(row.capabilities)
return needed.some((capability) => capabilities.includes(capability))
}) ?? null
)
}
async function createTaskUploadToken(
platform: Platform,
params: { taskId: string; downloaderId: string; orgId: string; targetFolder: string; createdByUserId: string },
) {
const now = Math.floor(Date.now() / 1000)
const jti = nanoid()
const expiresAt = new Date((now + UPLOAD_TOKEN_TTL_SECONDS) * 1000)
const token = await signDownloadToken(platform, {
v: 1,
typ: 'download-task-upload',
taskId: params.taskId,
downloaderId: params.downloaderId,
orgId: params.orgId,
targetFolder: params.targetFolder,
createdByUserId: params.createdByUserId,
scopes: ['objects:create', 'objects:upload', 'objects:confirm'],
jti,
iat: now,
exp: now + UPLOAD_TOKEN_TTL_SECONDS,
})
return { token, hash: await hashDownloadToken(platform, token), jti, expiresAt }
}
async function toDownloadTaskWithToken(platform: Platform, row: DownloadTaskRow, includeUploadToken: boolean) {
const task = toDownloadTask(row)
if (includeUploadToken && row.assignedDownloaderId) {
const token = await createTaskUploadToken(platform, {
taskId: row.id,
downloaderId: row.assignedDownloaderId,
orgId: row.orgId,
targetFolder: row.targetFolder,
createdByUserId: row.createdByUserId,
})
await platform.db
.update(downloadTasks)
.set({ uploadTokenHash: token.hash, uploadTokenJti: token.jti, uploadTokenExpiresAt: token.expiresAt })
.where(eq(downloadTasks.id, row.id))
task.uploadToken = token.token
}
return task
}
async function loadDownloaderRow(platform: Platform, id: string): Promise<DownloaderRow> {
const rows = await platform.db.select().from(downloaders).where(eq(downloaders.id, id)).limit(1)
if (!rows[0]) throw new DownloadError('not_found')
return rows[0]
}
server/services/downloads/mappers.ts
+93
View File
@@ -0,0 +1,93 @@
import type { Downloader, DownloadTask } from '@shared/types'
import type { DownloaderRow, DownloadTaskRow } from './types'
export function toDownloader(row: DownloaderRow): Downloader {
return {
id: row.id,
name: row.name,
status: row.enabled ? (row.status as Downloader['status']) : 'disabled',
enabled: row.enabled,
version: row.version,
hostname: row.hostname,
platform: row.platform,
arch: row.arch,
engine: row.engine as Downloader['engine'],
capabilities: parseCapabilities(row.capabilities),
maxConcurrentTasks: row.maxConcurrentTasks,
currentTasks: row.currentTasks,
downloadBps: row.downloadBps,
uploadBps: row.uploadBps,
freeDiskBytes: row.freeDiskBytes,
remoteDownloadCreditBillingEnabled: row.remoteDownloadCreditBillingEnabled,
remoteDownloadCreditUnitBytes: row.remoteDownloadCreditUnitBytes,
remoteDownloadCreditPerUnit: row.remoteDownloadCreditPerUnit,
lastHeartbeatAt: row.lastHeartbeatAt?.toISOString() ?? null,
createdBy: row.createdBy,
createdAt: row.createdAt.toISOString(),
updatedAt: row.updatedAt.toISOString(),
}
}
export function toDownloadTask(row: DownloadTaskRow): DownloadTask {
return {
id: row.id,
orgId: row.orgId,
createdByUserId: row.createdByUserId,
sourceType: row.sourceType as DownloadTask['sourceType'],
sourceUri: row.sourceUri,
name: row.name,
targetFolder: row.targetFolder,
category: row.category,
tags: parseTaskTags(row.tags),
assignedDownloaderId: row.assignedDownloaderId,
status: row.status as DownloadTask['status'],
downloadedBytes: row.downloadedBytes,
storageUploadedBytes: row.uploadedBytes,
totalBytes: row.totalBytes,
authorizedBytes: row.authorizedBytes,
billedBytes: row.billedBytes,
billedCredits: row.billedCredits,
billingStatus: row.billingStatus,
downloadBps: row.downloadBps,
storageUploadBps: row.uploadBps,
errorMessage: row.errorMessage,
resultObjectId: row.resultObjectId,
detail: parseTaskDetail(row.detail),
createdAt: row.createdAt.toISOString(),
updatedAt: row.updatedAt.toISOString(),
assignedAt: row.assignedAt?.toISOString() ?? null,
startedAt: row.startedAt?.toISOString() ?? null,
finishedAt: row.finishedAt?.toISOString() ?? null,
}
}
export function parseCapabilities(value: string): string[] {
return parseStringArray(value)
}
function parseTaskDetail(value: string | null): DownloadTask['detail'] {
if (!value) return null
try {
const detail = JSON.parse(value) as DownloadTask['detail'] & { uploadedBytes?: number }
if (detail.peerUploadedBytes === undefined && detail.uploadedBytes !== undefined) {
detail.peerUploadedBytes = detail.uploadedBytes
}
delete detail.uploadedBytes
return detail
} catch {
return null
}
}
function parseTaskTags(value: string): string[] {
return parseStringArray(value)
}
function parseStringArray(value: string): string[] {
try {
const parsed = JSON.parse(value)
return Array.isArray(parsed) ? parsed.filter((item): item is string => typeof item === 'string') : []
} catch {
return []
}
}
server/services/downloads/types.ts
+20
View File
@@ -0,0 +1,20 @@
import type { downloaders, downloadTasks } from '../../db/schema'
export class DownloadError extends Error {
constructor(
readonly code:
| 'not_found'
| 'forbidden'
| 'no_downloader'
| 'invalid_state'
| 'billing_paused'
| 'unsupported_source',
message: string = code,
) {
super(message)
this.name = 'DownloadError'
}
}
export type DownloaderRow = typeof downloaders.$inferSelect
export type DownloadTaskRow = typeof downloadTasks.$inferSelect
shared/schemas/downloads.ts
+90 -15
View File
@@ -1,4 +1,4 @@
import { z } from 'zod'
import { z } from '@hono/zod-openapi'
export const downloaderStatusSchema = z.enum(['online', 'offline', 'disabled'])
export const downloaderEngineSchema = z.enum(['builtin', 'aria2', 'qbittorrent'])
@@ -19,6 +19,15 @@ export const downloadTaskActionSchema = z.enum(['pause', 'resume', 'cancel', 're
export const downloadSourceTypeSchema = z.enum(['http', 'magnet', 'torrent_url'])
export const downloadTaskPhaseSchema = z.enum(['metadata', 'downloading', 'uploading', 'seeding', 'completed', 'error'])
const int64Schema = () => z.number().int().min(0).openapi({ type: 'integer', format: 'int64' })
const nullableInt64Schema = () =>
z
.number()
.int()
.min(0)
.nullable()
.openapi({ type: 'integer', format: 'int64', nullable: true } as never)
const downloadTaskTrackerSchema = z.object({
url: z.string().max(1024),
status: z.string().max(80).optional(),
@@ -32,14 +41,14 @@ const downloadTaskPeerSchema = z.object({
address: z.string().max(160),
client: z.string().max(160).optional(),
progress: z.number().min(0).max(1).optional(),
downloadBps: z.number().int().min(0).optional(),
uploadBps: z.number().int().min(0).optional(),
downloadBps: int64Schema().optional(),
uploadBps: int64Schema().optional(),
})
const downloadTaskFileSchema = z.object({
path: z.string().max(1024),
size: z.number().int().min(0),
completedBytes: z.number().int().min(0).optional(),
size: int64Schema(),
completedBytes: int64Schema().optional(),
selected: z.boolean().optional(),
})
@@ -55,13 +64,41 @@ export const downloadTaskDetailSchema = z.object({
seeders: z.number().int().min(0).optional(),
leechers: z.number().int().min(0).optional(),
peers: z.number().int().min(0).optional(),
peerUploadedBytes: z.number().int().min(0).optional(),
peerUploadBps: z.number().int().min(0).optional(),
peerUploadedBytes: int64Schema().optional(),
peerUploadBps: int64Schema().optional(),
trackers: z.array(downloadTaskTrackerSchema).max(20).optional(),
peerSamples: z.array(downloadTaskPeerSchema).max(20).optional(),
files: z.array(downloadTaskFileSchema).max(50).optional(),
})
export const downloadTaskSchema = z.object({
id: z.string(),
sourceType: downloadSourceTypeSchema,
sourceUri: z.string(),
name: z.string(),
targetFolder: z.string(),
category: z.string().nullable(),
tags: z.array(z.string()),
status: downloadTaskStatusSchema,
downloadedBytes: int64Schema(),
storageUploadedBytes: int64Schema(),
totalBytes: nullableInt64Schema(),
downloadBps: int64Schema(),
storageUploadBps: int64Schema(),
errorMessage: z.string().nullable().optional(),
resultObjectId: z.string().nullable().optional(),
detail: downloadTaskDetailSchema.nullable().optional(),
uploadToken: z.string().optional(),
assignedDownloaderId: z.string().nullable().optional(),
})
export const downloadTaskPageSchema = z.object({
items: z.array(downloadTaskSchema),
total: z.number().int(),
page: z.number().int(),
pageSize: z.number().int(),
})
export const downloaderHeartbeatSchema = z.object({
version: z.string().min(1).max(80),
hostname: z.string().min(1).max(160),
@@ -71,9 +108,46 @@ export const downloaderHeartbeatSchema = z.object({
capabilities: z.array(z.string().min(1).max(80)).max(32),
maxConcurrentTasks: z.number().int().min(1).max(100),
currentTasks: z.number().int().min(0).max(100),
downloadBps: z.number().int().min(0).default(0),
uploadBps: z.number().int().min(0).default(0),
freeDiskBytes: z.number().int().min(0).default(0),
downloadBps: int64Schema().default(0),
uploadBps: int64Schema().default(0),
freeDiskBytes: int64Schema().default(0),
})
export const downloaderHeartbeatResponseSchema = z.object({
version: z.string(),
hostname: z.string(),
platform: z.string(),
arch: z.string(),
engine: downloaderEngineSchema,
capabilities: z.array(z.string()),
maxConcurrentTasks: z.number().int(),
currentTasks: z.number().int(),
downloadBps: int64Schema(),
uploadBps: int64Schema(),
freeDiskBytes: int64Schema(),
})
export const downloaderSchema = z.object({
id: z.string(),
name: z.string().optional(),
status: downloaderStatusSchema.optional(),
enabled: z.boolean().optional(),
heartbeat: downloaderHeartbeatResponseSchema.optional(),
})
export const downloaderListSchema = z.object({
items: z.array(downloaderSchema),
total: z.number().int(),
})
export const createDownloaderResponseSchema = z.object({
downloader: downloaderSchema,
token: z.string(),
})
export const deleteDownloaderResponseSchema = z.object({
id: z.string(),
deleted: z.boolean(),
})
export const updateDownloaderSchema = z.object({
@@ -118,11 +192,11 @@ export const createDownloadTaskSchema = z.object({
export const updateDownloadTaskSchema = z.object({
status: downloadTaskStatusSchema.optional(),
downloadedBytes: z.number().int().min(0).optional(),
storageUploadedBytes: z.number().int().min(0).optional(),
totalBytes: z.number().int().min(0).nullable().optional(),
downloadBps: z.number().int().min(0).optional(),
storageUploadBps: z.number().int().min(0).optional(),
downloadedBytes: int64Schema().optional(),
storageUploadedBytes: int64Schema().optional(),
totalBytes: nullableInt64Schema().optional(),
downloadBps: int64Schema().optional(),
storageUploadBps: int64Schema().optional(),
errorMessage: z.string().max(1000).nullable().optional(),
resultObjectId: z.string().min(1).nullable().optional(),
detail: downloadTaskDetailSchema.nullable().optional(),
@@ -185,6 +259,7 @@ export type UpdateDownloadTaskInput = z.infer<typeof updateDownloadTaskSchema>
export type DownloadTaskActionInput = z.infer<typeof downloadTaskActionInputSchema>
export type ListDownloadTasksQuery = z.infer<typeof listDownloadTasksQuerySchema>
export type DownloadTaskDetail = z.infer<typeof downloadTaskDetailSchema>
export type DownloadTaskSchema = z.infer<typeof downloadTaskSchema>
export type CreateObjectUploadSessionInput = z.infer<typeof createObjectUploadSessionSchema>
export type PresignObjectUploadPartsInput = z.infer<typeof presignObjectUploadPartsSchema>
export type PatchObjectUploadSessionInput = z.infer<typeof patchObjectUploadSessionSchema>
shared/schemas/index.ts
+39 -2
View File
@@ -78,6 +78,7 @@ export type {
DownloaderHeartbeatInput,
DownloadTaskActionInput,
DownloadTaskDetail,
DownloadTaskSchema,
ListDownloadTasksQuery,
PatchObjectUploadSessionInput,
PresignObjectUploadPartsInput,
@@ -85,16 +86,23 @@ export type {
UpdateDownloadTaskInput,
} from './downloads'
export {
createDownloaderResponseSchema,
createDownloaderSchema,
createDownloadTaskSchema,
createObjectUploadSessionSchema,
deleteDownloaderResponseSchema,
downloaderEngineSchema,
downloaderHeartbeatResponseSchema,
downloaderHeartbeatSchema,
downloaderListSchema,
downloaderSchema,
downloaderStatusSchema,
downloadSourceTypeSchema,
downloadTaskActionInputSchema,
downloadTaskActionSchema,
downloadTaskDetailSchema,
downloadTaskPageSchema,
downloadTaskSchema,
downloadTaskStatusSchema,
listDownloadTasksQuerySchema,
patchObjectUploadSessionSchema,
@@ -132,9 +140,9 @@ export type ConflictStrategy = z.infer<typeof conflictStrategySchema>
export const createMatterSchema = z.object({
name: z.string().min(1),
type: z.string().min(1),
size: z.number().optional(),
size: z.number().int().min(0).optional(),
parent: z.string().default(''),
dirtype: z.number().default(0),
dirtype: z.number().int().default(0),
onConflict: conflictStrategySchema.optional(),
})
@@ -154,6 +162,35 @@ export const confirmMatterSchema = z.object({
onConflict: conflictStrategySchema.optional(),
})
export const objectDraftSchema = z.object({
id: z.string(),
name: z.string(),
uploadUrl: z.string().optional(),
contentDisposition: z.string().optional(),
})
export const objectUploadSessionSchema = z.object({
id: z.string(),
objectId: z.string(),
uploadId: z.string(),
partSize: z.number().int(),
status: z.enum(['active', 'completed', 'aborted']),
expiresAt: z.string(),
createdAt: z.string(),
updatedAt: z.string(),
})
export const presignedObjectUploadPartSchema = z.object({
partNumber: z.number().int(),
url: z.string(),
})
export const presignObjectUploadPartsResponseSchema = z.object({
uploadId: z.string(),
partSize: z.number().int(),
parts: z.array(presignedObjectUploadPartSchema),
})
export const trashMatterSchema = z.object({
action: z.literal('trash'),
})