Files

712 lines
18 KiB
Python

"""Typed request and response models for the doclib public interface."""
from __future__ import annotations
from typing import Any, Literal
from pydantic import BaseModel, Field
from ..parser.page_range import PAGE_RANGE_DESCRIPTION
from ..types import Tier
ParseSubmitStatus = Literal["pending", "done"]
ParseStatus = Literal["pending", "done", "parsing", "failed", "superseded"]
FileStatus = Literal["active", "deleted", "unreachable"]
ScanStatus = Literal["pending", "running", "done", "failed"]
ScanKind = Literal["manual", "watch"]
ScanSource = Literal["unknown", "cli", "sdk", "api", "watch", "system"]
RuleType = Literal["exclude", "parsing_rule"]
WatchStatus = Literal["active", "unreachable"]
TelemetryConsentState = Literal["unset", "enabled", "disabled"]
TelemetryAction = Literal["enable", "disable", "flush"]
InvalidateTarget = Literal["parses"]
ForgetMatchedAs = Literal["file", "directory", "none"]
ConfigSource = Literal["default", "override", "environment"]
RemoteAccessLevel = Literal["anonymous", "registered"]
ContentFormat = Literal["markdown", "image"]
ImageFormat = Literal["jpeg", "png", "webp"]
PARSE_STATUS_PENDING: ParseSubmitStatus = "pending"
PARSE_STATUS_DONE: ParseSubmitStatus = "done"
PARSE_STATUS_PARSING: ParseStatus = "parsing"
PARSE_STATUS_FAILED: ParseStatus = "failed"
PARSE_STATUS_SUPERSEDED: ParseStatus = "superseded"
FILE_STATUS_ACTIVE: FileStatus = "active"
FILE_STATUS_DELETED: FileStatus = "deleted"
FILE_STATUS_UNREACHABLE: FileStatus = "unreachable"
SCAN_STATUS_PENDING: ScanStatus = "pending"
SCAN_STATUS_RUNNING: ScanStatus = "running"
SCAN_STATUS_DONE: ScanStatus = "done"
SCAN_STATUS_FAILED: ScanStatus = "failed"
SCAN_KIND_MANUAL: ScanKind = "manual"
SCAN_KIND_WATCH: ScanKind = "watch"
SCAN_SOURCE_UNKNOWN: ScanSource = "unknown"
SCAN_SOURCE_CLI: ScanSource = "cli"
SCAN_SOURCE_SDK: ScanSource = "sdk"
SCAN_SOURCE_API: ScanSource = "api"
SCAN_SOURCE_WATCH: ScanSource = "watch"
SCAN_SOURCE_SYSTEM: ScanSource = "system"
RULE_TYPE_EXCLUDE: RuleType = "exclude"
RULE_TYPE_PARSING_RULE: RuleType = "parsing_rule"
WATCH_STATUS_ACTIVE: WatchStatus = "active"
WATCH_STATUS_UNREACHABLE: WatchStatus = "unreachable"
class DoclibModel(BaseModel):
"""Base model for doclib interface schemas."""
class ErrorInfo(DoclibModel):
type: str
code: str
message: str
param: str | None = None
class ErrorResponse(DoclibModel):
error: ErrorInfo
class ShutdownResponse(DoclibModel):
accepted: bool
message: str
class RemoveWatchResponse(DoclibModel):
watch_id: int
removed: bool
class RemoveExcludeRuleResponse(DoclibModel):
rule_id: int
removed: bool
class RemoveParsingRuleResponse(DoclibModel):
rule_id: int
removed: bool
class ParseRequest(DoclibModel):
path: str
tier: Tier | None = None
page_range: str | None = Field(default=None, description=PAGE_RANGE_DESCRIPTION)
force: bool = False
remote: bool = False
class ParseResponse(DoclibModel):
sha256: str
short_id: str | None = None
tier: Tier
page_range: str = Field(description="Resolved 1-based inclusive page ranges, e.g. 1-5,8.")
status: ParseSubmitStatus
cache_hit: bool = False
wait_parse_ids: list[int] = Field(default_factory=list)
created_parse_ids: list[int] = Field(default_factory=list)
reused_parse_ids: list[int] = Field(default_factory=list)
tip: str | None = None
class ParseCoverage(DoclibModel):
done_page_range: str = Field(description="Resolved 1-based page ranges; an empty string denotes no pages.")
active_page_range: str = Field(description="Resolved 1-based page ranges; an empty string denotes no pages.")
missing_page_range: str = Field(description="Resolved 1-based page ranges; an empty string denotes no pages.")
class ParseInfo(DoclibModel):
id: int
sha256: str
short_id: str
tier: Tier
page_range: str = Field(description="Resolved 1-based inclusive page ranges, e.g. 1-5,8.")
status: ParseStatus
priority: int = 0
privacy: str
via: str | None = None
coverage: ParseCoverage | None = None
created_at: int
updated_at: int
done_at: int | None = None
error_code: str | None = None
error_msg: str | None = None
class ListParsesResponse(DoclibModel):
parses: list[ParseInfo] = Field(default_factory=list)
coverage: ParseCoverage | None = None
total: int
limit: int
offset: int = 0
class InvalidateRequest(DoclibModel):
target: InvalidateTarget = "parses"
path: str | None = None
doc_ref: str | None = None
tier: Tier | None = None
class InvalidateResponse(DoclibModel):
target: str
sha256: str
short_id: str
tier: Tier | None = None
invalidated_count: int
class ForgetPathRequest(DoclibModel):
path: str
dry_run: bool = True
class ForgetPathResponse(DoclibModel):
path: str
matched_as: ForgetMatchedAs
forgotten_files: int
dry_run: bool
warnings: list[str] = Field(default_factory=list)
class ScanRequest(DoclibModel):
path: str
kind: ScanKind = "manual"
source: ScanSource = "unknown"
watch_id: int | None = None
class ScanInfo(DoclibModel):
id: int
path: str
kind: ScanKind
source: ScanSource = "unknown"
watch_id: int | None = None
status: ScanStatus
files_seen: int = 0
files_refreshed: int = 0
files_new: int = 0
files_changed: int = 0
files_deleted: int = 0
files_unreachable: int = 0
files_error: int = 0
files_unsupported: int = 0
files_excluded: int = 0
error_code: str | None = None
error_msg: str | None = None
started_at: int | None = None
finished_at: int | None = None
created_at: int
updated_at: int
class ScanListResponse(DoclibModel):
scans: list[ScanInfo] = Field(default_factory=list)
total: int
limit: int
offset: int = 0
class FileInfo(DoclibModel):
filename: str
path: str
ext: str
size_bytes: int
mtime_ms: int
sha256: str | None = None
short_id: str | None = None
watch_id: int | None = None
status: FileStatus
error_code: str | None = None
error_msg: str | None = None
deleted_at: int | None = None
first_seen_at: int
updated_at: int
class ListFilesResponse(DoclibModel):
files: list[FileInfo] = Field(default_factory=list)
total: int
limit: int
offset: int = 0
class DocInfo(DoclibModel):
sha256: str
short_id: str
size_bytes: int
file_type: str | None = None
title: str | None = None
author: str | None = None
subject: str | None = None
keywords: str | None = None
page_count: int | None = None
language: str | None = None
is_image_based: bool = False
meta_tier: Tier | None = None
error_code: str | None = None
error_msg: str | None = None
first_seen_at: int
updated_at: int
files: list[FileInfo] | None = None
class ListDocsResponse(DoclibModel):
docs: list[DocInfo] = Field(default_factory=list)
total: int
limit: int
offset: int = 0
class DocContentResponse(DoclibModel):
sha256: str
short_id: str
tier: Tier
format: ContentFormat = "markdown"
content: str
request_scope: "ContentRequestScope"
content_ranges: list["ContentRange"] = Field(default_factory=list)
truncated: bool = False
next_request: "ContentNextRequest | None" = None
asset: "ContentAsset | None" = None
class ContentRequestScope(DoclibModel):
page_range: str | None = Field(default=None, description=PAGE_RANGE_DESCRIPTION)
after: str | None = None
limit: int = 30000
locator: str | None = None
context: int = 0
image_format: ImageFormat | None = None
class ContentRange(DoclibModel):
page_range: str | None = Field(default=None, description=PAGE_RANGE_DESCRIPTION)
start: str
end: str
class ContentNextRequest(DoclibModel):
page_range: str | None = Field(default=None, description=PAGE_RANGE_DESCRIPTION)
after: str | None = None
locator: str | None = None
class ContentAsset(DoclibModel):
path: str
mime_type: str
size_bytes: int | None = None
width: int | None = None
height: int | None = None
class DocContentExportRequest(DoclibModel):
tier: Tier
page_range: str | None = Field(default=None, description=PAGE_RANGE_DESCRIPTION)
format: str = "markdown"
output: str
no_marker: bool = False
class DocContentExportResponse(DoclibModel):
sha256: str
short_id: str
tier: Tier
output: str
class SearchFile(DoclibModel):
path: str
filename: str
ext: str
status: FileStatus
class SearchResult(DoclibModel):
sha256: str
short_id: str
title: str | None = None
author: str | None = None
size_bytes: int | None = None
page_count: int | None = None
tier: Tier | None = None
snippet: str
files: list[SearchFile] = Field(default_factory=list)
class FindResult(DoclibModel):
filename: str
ext: str | None = None
size_bytes: int | None = None
page_count: int | None = None
paths: list[str] = Field(default_factory=list)
class SearchResponse(DoclibModel):
results: list[SearchResult] = Field(default_factory=list)
total: int
query: str
class FindResponse(DoclibModel):
results: list[FindResult] = Field(default_factory=list)
total: int
query: str
class FileInfoResponse(DoclibModel):
file: FileInfo
doc: DocInfo | None = None
parsed_tiers: list["TierParseInfo"] = Field(default_factory=list)
active_parses: list[ParseInfo] = Field(default_factory=list)
class TierParseInfo(DoclibModel):
tier: Tier
status: ParseStatus
page_range: str = Field(description="Resolved 1-based inclusive page ranges, e.g. 1-5,8.")
done_at: int | None = None
class ConfigValueRequest(DoclibModel):
value: str
class ConfigResponse(DoclibModel):
config: dict[str, str] = Field(default_factory=dict)
sources: dict[str, ConfigSource] = Field(default_factory=dict)
class ConfigValueResponse(DoclibModel):
key: str
value: str
source: ConfigSource
class ConfigSetRequest(ConfigValueRequest):
pass
class ConfigSetResponse(ConfigValueResponse):
pass
class ConfigUnsetResponse(ConfigValueResponse):
removed: bool = True
class RemoteUsageBillingPeriod(DoclibModel):
start: str
end: str | None = None
class RemoteUsageCurrent(DoclibModel):
pages_processed: int = 0
files_processed: int = 0
jobs_created: int = 0
class RemoteUsageLimits(DoclibModel):
max_pages_per_file: int
max_file_size_bytes: int
max_files_per_job: int
max_concurrent_jobs: int
max_file_retention_days: int | None = None
class RemoteUsageResponse(DoclibModel):
object: Literal["usage"] = "usage"
access_level: RemoteAccessLevel
billing_period: RemoteUsageBillingPeriod
current: RemoteUsageCurrent
limits: RemoteUsageLimits
class WatchRequest(DoclibModel):
path: str
removable: bool = False
label: str | None = None
class WatchInfo(DoclibModel):
id: int
path: str
label: str | None = None
removable: bool = False
enabled: bool = True
recursive: bool = False
status: WatchStatus
unreachable_at: int | None = None
last_scan_at: int | None = None
last_scan_files: int = 0
created_at: int = 0
updated_at: int = 0
class WatchListResponse(DoclibModel):
watches: list[WatchInfo] = Field(default_factory=list)
class ExcludeRuleRequest(DoclibModel):
pattern: str
name: str | None = None
priority: int = 0
class ParsingRuleRequest(DoclibModel):
pattern: str
name: str | None = None
tier: Tier | None = None
page_range: str | None = Field(default=None, description=PAGE_RANGE_DESCRIPTION)
remote: bool = False
priority: int = 0
class ExcludeRuleInfo(DoclibModel):
id: int
pattern: str
name: str | None = None
enabled: bool = True
priority: int = 0
hit_count: int = 0
created_at: int = 0
updated_at: int = 0
class ParsingRuleInfo(DoclibModel):
id: int
pattern: str
name: str | None = None
tier: Tier | None = None
page_range: str | None = Field(default=None, description=PAGE_RANGE_DESCRIPTION)
remote: bool = False
enabled: bool = True
priority: int = 0
hit_count: int = 0
created_at: int = 0
updated_at: int = 0
class ExcludeRuleListResponse(DoclibModel):
rules: list[ExcludeRuleInfo] = Field(default_factory=list)
class ParsingRuleListResponse(DoclibModel):
rules: list[ParsingRuleInfo] = Field(default_factory=list)
class LocalParseServerStatus(DoclibModel):
mode: str | None = None
healthy: bool = False
starting: bool = False
started_at: float | int | None = None
url: str | None = None
port: int | None = None
managed_pid: int | None = None
managed_running: bool = False
managed_tier: Tier | None = None
self_hosted_url: str | None = None
restart_count: int = 0
max_restart_attempts: int = 0
last_probe_at: int | None = None
last_success_at: int | None = None
last_failure_at: int | None = None
supported_tiers: list[Tier] = Field(default_factory=list)
error_code: str | None = None
error_msg: str | None = None
class RemoteParseServerStatus(DoclibModel):
healthy: bool = False
url: str | None = None
port: int | None = None
last_probe_at: int | None = None
last_success_at: int | None = None
last_failure_at: int | None = None
supported_tiers: list[Tier] = Field(default_factory=list)
error_code: str | None = None
error_msg: str | None = None
class ParseServerStatus(DoclibModel):
local: LocalParseServerStatus = Field(default_factory=LocalParseServerStatus)
remote: RemoteParseServerStatus = Field(default_factory=RemoteParseServerStatus)
class TCPServerStatus(DoclibModel):
enabled: bool = False
host: str | None = None
port: int | None = None
class WorkerStatus(DoclibModel):
watch_running: bool = False
scan_running: bool = False
scan_workers: int = 0
ingest_running: bool = False
ingest_workers: int = 0
parse_running: bool = False
parse_workers: int = 0
device_monitor_running: bool = False
compaction_running: bool = False
health_check_running: bool = False
class WatchStats(DoclibModel):
watch_id: int
path: str
label: str | None = None
removable: bool = False
status: WatchStatus
total_files: int = 0
active_files: int = 0
deleted_files: int = 0
unreachable_files: int = 0
pending_ingest_files: int = 0
file_error_count: int = 0
doc_count: int = 0
parse_pending_count: int = 0
parse_parsing_count: int = 0
parse_failed_count: int = 0
parse_done_count: int = 0
last_scan_at: int | None = None
last_scan_files: int = 0
class ErrorBucket(DoclibModel):
code: str
count: int
class ErrorSummary(DoclibModel):
file_errors: list[ErrorBucket] = Field(default_factory=list)
doc_errors: list[ErrorBucket] = Field(default_factory=list)
parse_errors: list[ErrorBucket] = Field(default_factory=list)
class ServerStatusResponse(DoclibModel):
running: bool
pid: int | None = None
server_id: str = ""
uptime_seconds: float | None = None
mineru_home: str = ""
version: str = ""
python_version: str = ""
socket_path: str
data_dir: str
sqlite_path: str
log_path: str
access_log_path: str = ""
stdout_log_path: str = ""
stderr_log_path: str = ""
tcp: TCPServerStatus = Field(default_factory=TCPServerStatus)
active_scan_count: int = 0
last_scan_at: int | None = None
sqlite_journal_mode: str | None = None
sqlite_size_bytes: int | None = None
sqlite_wal_size_bytes: int | None = None
workers: WorkerStatus = Field(default_factory=WorkerStatus)
files_total: int = 0
docs_total: int = 0
parse_queue_length: int = 0
ingest_queue_length: int = 0
parse_server: ParseServerStatus | None = None
watch_count: int = 0
watches: list[WatchInfo] = Field(default_factory=list)
watch_stats: list[WatchStats] = Field(default_factory=list)
recent_scans: list[ScanInfo] = Field(default_factory=list)
error_summary: ErrorSummary | None = None
recent_logs: list[str] = Field(default_factory=list)
app_logs: list[str] = Field(default_factory=list)
access_logs: list[str] = Field(default_factory=list)
stdout_logs: list[str] = Field(default_factory=list)
stderr_logs: list[str] = Field(default_factory=list)
parse_server_stdout_logs: list[str] = Field(default_factory=list)
parse_server_stderr_logs: list[str] = Field(default_factory=list)
class CleanupDeletedRequest(DoclibModel):
dry_run: bool = True
class CleanupDeletedResponse(DoclibModel):
deleted_files: int
dry_run: bool
class CleanupOrphansRequest(DoclibModel):
dry_run: bool = True
class CleanupOrphansResponse(DoclibModel):
orphan_docs: int
dry_run: bool
class CleanupTempRequest(DoclibModel):
older_than_days: int = 7
class CleanupTempResponse(DoclibModel):
temp_files_removed: int
class TelemetryMetric(DoclibModel):
name: str
value: int = 1
dimensions: dict[str, str] = Field(default_factory=dict)
class TelemetryPayload(DoclibModel):
api_version: str
batch_id: str
schema_version: str
installation_id: str
period_start: str
period_end: str
context: dict[str, str] = Field(default_factory=dict)
metrics: list[dict[str, Any]] = Field(default_factory=list)
class TelemetryStatusResponse(DoclibModel):
state: TelemetryConsentState
installation_id: str
pending_periods: int = 0
pending_metrics: int = 0
last_flush_at: int | None = None
class TelemetryPreviewResponse(DoclibModel):
body: TelemetryPayload
class TelemetryActionResponse(DoclibModel):
action: TelemetryAction
state: TelemetryConsentState
installation_id: str
accepted: bool = True
executed: bool = True
reason: str | None = None
flush_result: dict[str, Any] | None = None
class TelemetryObservation(DoclibModel):
metric_name: str
value: int = 1
duration_ms: int | None = None
parse_ids: list[int] = Field(default_factory=list)
dimensions: dict[str, str] = Field(default_factory=dict)
class TelemetryObservationsRequest(DoclibModel):
observations: list[TelemetryObservation] = Field(default_factory=list)
class TelemetryObservationsResponse(DoclibModel):
accepted: int