mirror of
https://github.com/Tencent/WeKnora.git
synced 2026-10-15 10:58:21 +08:00
lyingbug
1bf717a54d
* feat(chat): open cited documents at the cited passage Clicking a citation used to show only the parsed chunk. It now opens the original file in the references panel, jumps to the cited place and highlights it, narrowed to the sentence the citation supports. Parsing records where each chunk sits in the original file (chunks.source_locators): PDF page and paragraph boxes from the docreader, MinerU and PaddleOCR-VL layouts; Word paragraphs, slides, sheet rows and EPUB chapters aligned in Go against any parser engine's Markdown; text offsets for Markdown/TXT; segment times for audio. Pipeline text rewrites are followed with a line-diff remapper. Scanned PDF pages get their OCR text as quotes at search time so a sentence can pick its page. The frontend renders PDFs with pdf.js and every other previewable type with the existing preview, then highlights with the CSS Custom Highlight API in a highlighter yellow that stays visible against the brand green. * fix(docreader): serialize source blocks outside main so tests import it test_docx_tables stubs docreader.parser in sys.modules, so importing docreader.main from a later test failed in the full unittest discover run. The wire conversion now lives in docreader.source_wire, which needs only the proto. * fix(chat): tighten source locators from review - Remap rewritten-line-for-line stretches by line so scanned PDF pages no longer drift onto the wrong page. - Keep the source preview mounted for citations in the file already open. - Wait for in-flight PDF page renders before looking for the text layer. - Count image rune offsets in one pass; stream xlsx rows up to maxUnits. - Start text locators after a UTF-8 BOM, as viewers decode without it. - Only re-query image children for scanned-page results. - Seek audio citations without autoplay and drop stale seek listeners. - Fix the SQLite migration's cross-reference to versioned/000113. * fix(migrations): renumber chunk source locators after main's chunk_images main took versioned 000113 and sqlite 000032 for chunk_images, so the source_locators column moves to versioned 000114 and sqlite 000033.
98 lines
3.0 KiB
Python
98 lines
3.0 KiB
Python
"""Chunk document schema."""
|
|
|
|
import json
|
|
from typing import Any, Dict, List
|
|
|
|
from pydantic import BaseModel, Field
|
|
|
|
|
|
class Chunk(BaseModel):
|
|
"""Document Chunk including chunk content, chunk metadata."""
|
|
|
|
content: str = Field(default="", description="chunk text content")
|
|
seq: int = Field(default=0, description="Chunk sequence number")
|
|
start: int = Field(default=0, description="Chunk start position")
|
|
end: int = Field(description="Chunk end position")
|
|
images: List[Dict[str, Any]] = Field(
|
|
default_factory=list, description="Images in the chunk"
|
|
)
|
|
|
|
metadata: Dict[str, Any] = Field(
|
|
default_factory=dict,
|
|
description="metadata fields",
|
|
)
|
|
|
|
def to_dict(self, **kwargs: Any) -> Dict[str, Any]:
|
|
"""Convert Chunk to dict."""
|
|
|
|
data = self.model_dump()
|
|
data.update(kwargs)
|
|
data["class_name"] = self.__class__.__name__
|
|
return data
|
|
|
|
def to_json(self, **kwargs: Any) -> str:
|
|
"""Convert Chunk to json."""
|
|
data = self.to_dict(**kwargs)
|
|
return json.dumps(data)
|
|
|
|
def __hash__(self):
|
|
"""Hash function."""
|
|
return hash((self.content,))
|
|
|
|
def __eq__(self, other):
|
|
"""Equal function."""
|
|
return self.content == other.content
|
|
|
|
@classmethod
|
|
def from_dict(cls, data: Dict[str, Any], **kwargs: Any): # type: ignore
|
|
"""Create Chunk from dict."""
|
|
if isinstance(kwargs, dict):
|
|
data.update(kwargs)
|
|
|
|
data.pop("class_name", None)
|
|
return cls(**data)
|
|
|
|
@classmethod
|
|
def from_json(cls, data_str: str, **kwargs: Any): # type: ignore
|
|
"""Create Chunk from json."""
|
|
data = json.loads(data_str)
|
|
return cls.from_dict(data, **kwargs)
|
|
|
|
|
|
class Document(BaseModel):
|
|
"""Document including document content, document metadata."""
|
|
|
|
model_config = {"arbitrary_types_allowed": True}
|
|
|
|
content: str = Field(default="", description="document text content")
|
|
images: Dict[str, str] = Field(
|
|
default_factory=dict, description="Images in the document"
|
|
)
|
|
|
|
chunks: List[Chunk] = Field(default_factory=list, description="document chunks")
|
|
source_blocks: List[Dict[str, Any]] = Field(
|
|
default_factory=list,
|
|
description=(
|
|
"Ranges of content mapped to positions in the original file: "
|
|
'{"start": int, "end": int, "locator": {"type": "pdf", "page": 1, '
|
|
'"bbox": [x0, y0, x1, y1]}}. Offsets are code point indices into '
|
|
"content, end exclusive; ordinals are 1-based; bbox is fractions of "
|
|
"the page with the origin at the top-left corner."
|
|
),
|
|
)
|
|
metadata: Dict[str, Any] = Field(
|
|
default_factory=dict,
|
|
description="metadata fields",
|
|
)
|
|
|
|
def set_content(self, content: str) -> None:
|
|
"""Set document content."""
|
|
self.content = content
|
|
|
|
def get_content(self) -> str:
|
|
"""Get document content."""
|
|
return self.content
|
|
|
|
def is_valid(self) -> bool:
|
|
return self.content != ""
|