mirror of
https://github.com/Tencent/WeKnora.git
synced 2026-11-02 05:03:00 +08:00
wizardchen
7b1bb1054f
Large scanned PDFs (hundreds of pages) were slow and fragile end-to-end.
This change addresses the parse, transport, and task-scheduling layers:
docreader (parse + transport):
- Parallelize per-page scanned rendering across processes (forkserver/fork),
with serial fallback. ~4-7x faster on large scanned PDFs; pdfium is not
thread-safe so we fan out across processes. Configurable via
DOCREADER_PDF_RENDER_PARALLELISM.
- Add server-streaming ReadStream RPC: emit one meta frame then one frame per
image, so documents with many page images are no longer capped by the unary
gRPC message-size limit (a 874-page PDF produced ~193MiB of images, far over
the 50MB cap) and memory is bounded on both ends. Unary Read is kept for
backward compatibility; the Go production reader switches to ReadStream.
VLM:
- Make the VLM HTTP timeout configurable (VLM_HTTP_TIMEOUT_SECONDS) and raise
the default 90s -> 180s so dense scanned-page OCR does not time out with
"context deadline exceeded".
Async task queues:
- Isolate high-volume, model-heavy fan-out tasks into dedicated asynq queues so
a single large document cannot saturate the shared worker pool and block
user-facing document parsing:
image:multimodal -> "multimodal"
chunk:extract -> "graph"
question:generation -> "question"
- Register the new queues in the server weight map and the cancel inspector's
scanned-queue set (so cancelling a knowledge still purges its pending tasks).
72 lines
6.1 KiB
Python
72 lines
6.1 KiB
Python
# -*- coding: utf-8 -*-
|
|
# Generated by the protocol buffer compiler. DO NOT EDIT!
|
|
# NO CHECKED-IN PROTOBUF GENCODE
|
|
# source: docreader.proto
|
|
# Protobuf Python Version: 6.31.1
|
|
"""Generated protocol buffer code."""
|
|
from google.protobuf import descriptor as _descriptor
|
|
from google.protobuf import descriptor_pool as _descriptor_pool
|
|
from google.protobuf import runtime_version as _runtime_version
|
|
from google.protobuf import symbol_database as _symbol_database
|
|
from google.protobuf.internal import builder as _builder
|
|
_runtime_version.ValidateProtobufRuntimeVersion(
|
|
_runtime_version.Domain.PUBLIC,
|
|
6,
|
|
31,
|
|
1,
|
|
'',
|
|
'docreader.proto'
|
|
)
|
|
# @@protoc_insertion_point(imports)
|
|
|
|
_sym_db = _symbol_database.Default()
|
|
|
|
|
|
|
|
|
|
DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0f\x64ocreader.proto\x12\tdocreader\"\xba\x01\n\nReadConfig\x12\x15\n\rparser_engine\x18\x01 \x01(\t\x12Q\n\x17parser_engine_overrides\x18\x02 \x03(\x0b\x32\x30.docreader.ReadConfig.ParserEngineOverridesEntry\x1a<\n\x1aParserEngineOverridesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01J\x04\x08\x03\x10\x04\"\xa0\x01\n\x0bReadRequest\x12\x14\n\x0c\x66ile_content\x18\x01 \x01(\x0c\x12\x11\n\tfile_name\x18\x02 \x01(\t\x12\x11\n\tfile_type\x18\x03 \x01(\t\x12\x0b\n\x03url\x18\x04 \x01(\t\x12\r\n\x05title\x18\x05 \x01(\t\x12%\n\x06\x63onfig\x18\x06 \x01(\x0b\x32\x15.docreader.ReadConfig\x12\x12\n\nrequest_id\x18\x07 \x01(\t\"n\n\x08ImageRef\x12\x10\n\x08\x66ilename\x18\x01 \x01(\t\x12\x14\n\x0coriginal_ref\x18\x02 \x01(\t\x12\x11\n\tmime_type\x18\x03 \x01(\t\x12\x13\n\x0bstorage_key\x18\x04 \x01(\t\x12\x12\n\nimage_data\x18\x05 \x01(\x0c\"\xe2\x01\n\x0cReadResponse\x12\x18\n\x10markdown_content\x18\x01 \x01(\t\x12\'\n\nimage_refs\x18\x02 \x03(\x0b\x32\x13.docreader.ImageRef\x12\x16\n\x0eimage_dir_path\x18\x03 \x01(\t\x12\x37\n\x08metadata\x18\x04 \x03(\x0b\x32%.docreader.ReadResponse.MetadataEntry\x12\r\n\x05\x65rror\x18\x05 \x01(\t\x1a/\n\rMetadataEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\xd2\x01\n\x0eReadStreamMeta\x12\x18\n\x10markdown_content\x18\x01 \x01(\t\x12\x16\n\x0eimage_dir_path\x18\x02 \x01(\t\x12\x39\n\x08metadata\x18\x03 \x03(\x0b\x32\'.docreader.ReadStreamMeta.MetadataEntry\x12\r\n\x05\x65rror\x18\x04 \x01(\t\x12\x13\n\x0bimage_count\x18\x05 \x01(\r\x1a/\n\rMetadataEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"p\n\x12ReadStreamResponse\x12)\n\x04meta\x18\x01 \x01(\x0b\x32\x19.docreader.ReadStreamMetaH\x00\x12$\n\x05image\x18\x02 \x01(\x0b\x32\x13.docreader.ImageRefH\x00\x42\t\n\x07payload\"\x9a\x01\n\x12ListEnginesRequest\x12L\n\x10\x63onfig_overrides\x18\x01 \x03(\x0b\x32\x32.docreader.ListEnginesRequest.ConfigOverridesEntry\x1a\x36\n\x14\x43onfigOverridesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"x\n\x10ParserEngineInfo\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x13\n\x0b\x64\x65scription\x18\x02 \x01(\t\x12\x12\n\nfile_types\x18\x03 \x03(\t\x12\x11\n\tavailable\x18\x04 \x01(\x08\x12\x1a\n\x12unavailable_reason\x18\x05 \x01(\t\"C\n\x13ListEnginesResponse\x12,\n\x07\x65ngines\x18\x01 \x03(\x0b\x32\x1b.docreader.ParserEngineInfo2\xdf\x01\n\tDocReader\x12\x39\n\x04Read\x12\x16.docreader.ReadRequest\x1a\x17.docreader.ReadResponse\"\x00\x12G\n\nReadStream\x12\x16.docreader.ReadRequest\x1a\x1d.docreader.ReadStreamResponse\"\x00\x30\x01\x12N\n\x0bListEngines\x12\x1d.docreader.ListEnginesRequest\x1a\x1e.docreader.ListEnginesResponse\"\x00\x42\x35Z3github.com/Tencent/WeKnora/internal/docreader/protob\x06proto3')
|
|
|
|
_globals = globals()
|
|
_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals)
|
|
_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'docreader_pb2', _globals)
|
|
if not _descriptor._USE_C_DESCRIPTORS:
|
|
_globals['DESCRIPTOR']._loaded_options = None
|
|
_globals['DESCRIPTOR']._serialized_options = b'Z3github.com/Tencent/WeKnora/internal/docreader/proto'
|
|
_globals['_READCONFIG_PARSERENGINEOVERRIDESENTRY']._loaded_options = None
|
|
_globals['_READCONFIG_PARSERENGINEOVERRIDESENTRY']._serialized_options = b'8\001'
|
|
_globals['_READRESPONSE_METADATAENTRY']._loaded_options = None
|
|
_globals['_READRESPONSE_METADATAENTRY']._serialized_options = b'8\001'
|
|
_globals['_READSTREAMMETA_METADATAENTRY']._loaded_options = None
|
|
_globals['_READSTREAMMETA_METADATAENTRY']._serialized_options = b'8\001'
|
|
_globals['_LISTENGINESREQUEST_CONFIGOVERRIDESENTRY']._loaded_options = None
|
|
_globals['_LISTENGINESREQUEST_CONFIGOVERRIDESENTRY']._serialized_options = b'8\001'
|
|
_globals['_READCONFIG']._serialized_start=31
|
|
_globals['_READCONFIG']._serialized_end=217
|
|
_globals['_READCONFIG_PARSERENGINEOVERRIDESENTRY']._serialized_start=151
|
|
_globals['_READCONFIG_PARSERENGINEOVERRIDESENTRY']._serialized_end=211
|
|
_globals['_READREQUEST']._serialized_start=220
|
|
_globals['_READREQUEST']._serialized_end=380
|
|
_globals['_IMAGEREF']._serialized_start=382
|
|
_globals['_IMAGEREF']._serialized_end=492
|
|
_globals['_READRESPONSE']._serialized_start=495
|
|
_globals['_READRESPONSE']._serialized_end=721
|
|
_globals['_READRESPONSE_METADATAENTRY']._serialized_start=674
|
|
_globals['_READRESPONSE_METADATAENTRY']._serialized_end=721
|
|
_globals['_READSTREAMMETA']._serialized_start=724
|
|
_globals['_READSTREAMMETA']._serialized_end=934
|
|
_globals['_READSTREAMMETA_METADATAENTRY']._serialized_start=674
|
|
_globals['_READSTREAMMETA_METADATAENTRY']._serialized_end=721
|
|
_globals['_READSTREAMRESPONSE']._serialized_start=936
|
|
_globals['_READSTREAMRESPONSE']._serialized_end=1048
|
|
_globals['_LISTENGINESREQUEST']._serialized_start=1051
|
|
_globals['_LISTENGINESREQUEST']._serialized_end=1205
|
|
_globals['_LISTENGINESREQUEST_CONFIGOVERRIDESENTRY']._serialized_start=1151
|
|
_globals['_LISTENGINESREQUEST_CONFIGOVERRIDESENTRY']._serialized_end=1205
|
|
_globals['_PARSERENGINEINFO']._serialized_start=1207
|
|
_globals['_PARSERENGINEINFO']._serialized_end=1327
|
|
_globals['_LISTENGINESRESPONSE']._serialized_start=1329
|
|
_globals['_LISTENGINESRESPONSE']._serialized_end=1396
|
|
_globals['_DOCREADER']._serialized_start=1399
|
|
_globals['_DOCREADER']._serialized_end=1622
|
|
# @@protoc_insertion_point(module_scope)
|