mirror of
https://github.com/Tencent/WeKnora.git
synced 2026-11-01 12:42:19 +08:00
wizardchen
7b1bb1054f
Large scanned PDFs (hundreds of pages) were slow and fragile end-to-end.
This change addresses the parse, transport, and task-scheduling layers:
docreader (parse + transport):
- Parallelize per-page scanned rendering across processes (forkserver/fork),
with serial fallback. ~4-7x faster on large scanned PDFs; pdfium is not
thread-safe so we fan out across processes. Configurable via
DOCREADER_PDF_RENDER_PARALLELISM.
- Add server-streaming ReadStream RPC: emit one meta frame then one frame per
image, so documents with many page images are no longer capped by the unary
gRPC message-size limit (a 874-page PDF produced ~193MiB of images, far over
the 50MB cap) and memory is bounded on both ends. Unary Read is kept for
backward compatibility; the Go production reader switches to ReadStream.
VLM:
- Make the VLM HTTP timeout configurable (VLM_HTTP_TIMEOUT_SECONDS) and raise
the default 90s -> 180s so dense scanned-page OCR does not time out with
"context deadline exceeded".
Async task queues:
- Isolate high-volume, model-heavy fan-out tasks into dedicated asynq queues so
a single large document cannot saturate the shared worker pool and block
user-facing document parsing:
image:multimodal -> "multimodal"
chunk:extract -> "graph"
question:generation -> "question"
- Register the new queues in the server weight map and the cancel inspector's
scanned-queue set (so cancelling a knowledge still purges its pending tasks).
87 lines
2.4 KiB
Protocol Buffer
87 lines
2.4 KiB
Protocol Buffer
syntax = "proto3";
|
|
|
|
package docreader;
|
|
|
|
option go_package = "github.com/Tencent/WeKnora/internal/docreader/proto";
|
|
|
|
service DocReader {
|
|
rpc Read(ReadRequest) returns (ReadResponse) {}
|
|
// ReadStream is the streaming counterpart of Read. It first emits one
|
|
// ReadStreamResponse carrying the parse metadata (markdown / metadata /
|
|
// error), then emits one message per image. This keeps every gRPC message
|
|
// small so large scanned PDFs (hundreds of page images, far exceeding the
|
|
// unary message-size cap) can be returned without RESOURCE_EXHAUSTED and
|
|
// with bounded memory on both ends.
|
|
rpc ReadStream(ReadRequest) returns (stream ReadStreamResponse) {}
|
|
rpc ListEngines(ListEnginesRequest) returns (ListEnginesResponse) {}
|
|
}
|
|
|
|
message ReadConfig {
|
|
string parser_engine = 1;
|
|
map<string, string> parser_engine_overrides = 2;
|
|
// image_storage removed: image persistence is now handled entirely by the Go App.
|
|
// Field number 3 is reserved for backward compatibility.
|
|
reserved 3;
|
|
}
|
|
|
|
// Unified read request: set file_content for file mode, url for URL mode.
|
|
message ReadRequest {
|
|
bytes file_content = 1;
|
|
string file_name = 2;
|
|
string file_type = 3;
|
|
string url = 4;
|
|
string title = 5;
|
|
ReadConfig config = 6;
|
|
string request_id = 7;
|
|
}
|
|
|
|
message ImageRef {
|
|
string filename = 1;
|
|
string original_ref = 2;
|
|
string mime_type = 3;
|
|
string storage_key = 4; // download URL from shared storage
|
|
bytes image_data = 5; // inline bytes fallback
|
|
}
|
|
|
|
message ReadResponse {
|
|
string markdown_content = 1;
|
|
repeated ImageRef image_refs = 2;
|
|
string image_dir_path = 3;
|
|
map<string, string> metadata = 4;
|
|
string error = 5;
|
|
}
|
|
|
|
// Metadata header for a streamed read. Sent exactly once, before any image.
|
|
message ReadStreamMeta {
|
|
string markdown_content = 1;
|
|
string image_dir_path = 2;
|
|
map<string, string> metadata = 3;
|
|
string error = 4;
|
|
uint32 image_count = 5; // best-effort total image count (0 if unknown)
|
|
}
|
|
|
|
// One frame of a ReadStream. The first frame carries `meta`; every subsequent
|
|
// frame carries a single `image`.
|
|
message ReadStreamResponse {
|
|
oneof payload {
|
|
ReadStreamMeta meta = 1;
|
|
ImageRef image = 2;
|
|
}
|
|
}
|
|
|
|
message ListEnginesRequest {
|
|
map<string, string> config_overrides = 1;
|
|
}
|
|
|
|
message ParserEngineInfo {
|
|
string name = 1;
|
|
string description = 2;
|
|
repeated string file_types = 3;
|
|
bool available = 4;
|
|
string unavailable_reason = 5;
|
|
}
|
|
|
|
message ListEnginesResponse {
|
|
repeated ParserEngineInfo engines = 1;
|
|
}
|