mirror of
https://github.com/Tencent/WeKnora.git
synced 2026-10-14 02:07:53 +08:00
wizardchen
b0767453e4
Plugins can contribute parser engines (contributes.parsers, with the
fileTypes they handle). Each registers under its qualified ID
(acme.ocr/ocr) next to the builtin engines and the docreader's, kept in a
separate store so the builtin plugin catalog never describes it as a
builtin. A knowledge base picks it in its parser rules like any engine;
uploads of its file types are accepted.
The engine calls POST /v1/parsers/{id}/parse with the file as base64 and
takes back Markdown plus the images it references, which the pipeline
stores and chunks as usual. A retryable plugin error (unavailable, rate
limited) fails the call so the task retries; any other error fails the
document for good. Other engines' tenant credentials are never passed to
a plugin. A plugin engine that is gone no longer falls through to the
Python docreader; the parse fails saying the plugin is not running. The
engine list honours the workspace's plugin switches and shows plugin
engines by their localized names.
SDK: p.Parser(id, ...), openapi.yaml and a conformance check.
examples/plugins/subtitles parses SRT and WebVTT into a transcript and
counts parsed files per workspace through the Host API key-value store.
56 lines
1.7 KiB
Go
56 lines
1.7 KiB
Go
package pluginsdk
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"net/http"
|
|
|
|
"github.com/Tencent/WeKnora/pluginsdk/pluginapi"
|
|
)
|
|
|
|
// Parser turns a document into Markdown. Return a non-retryable error (such
|
|
// as pluginapi.CodeInvalidConfig or CodeNotFound) for a document that will
|
|
// never parse; retryable ones (CodeUnavailable, CodeRateLimited) make
|
|
// WeKnora try again later.
|
|
type Parser interface {
|
|
Parse(ctx context.Context, call *Call, in pluginapi.ParseInput) (*pluginapi.ParseOutput, error)
|
|
}
|
|
|
|
// ParserFunc adapts a function to Parser.
|
|
type ParserFunc func(ctx context.Context, call *Call, in pluginapi.ParseInput) (*pluginapi.ParseOutput, error)
|
|
|
|
// Parse implements Parser.
|
|
func (f ParserFunc) Parse(ctx context.Context, call *Call, in pluginapi.ParseInput) (*pluginapi.ParseOutput, error) {
|
|
return f(ctx, call, in)
|
|
}
|
|
|
|
// Parser registers parser id (contributes.parsers[].id).
|
|
func (p *Plugin) Parser(id string, parser Parser) { p.parsers[id] = parser }
|
|
|
|
func (p *Plugin) routeParsers(mux *http.ServeMux) {
|
|
mux.HandleFunc("POST /v1/parsers/{id}/parse", func(w http.ResponseWriter, r *http.Request) {
|
|
parser, ok := p.parsers[r.PathValue("id")]
|
|
if !ok {
|
|
writeError(w, pluginapi.Errorf(pluginapi.CodeNotFound, "no parser %q", r.PathValue("id")))
|
|
return
|
|
}
|
|
p.unary(func(ctx context.Context, call *Call, raw json.RawMessage) (any, error) {
|
|
in, err := decodeInput[pluginapi.ParseInput](raw)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if len(in.Content) == 0 && in.URL == "" {
|
|
return nil, pluginapi.Errorf(pluginapi.CodeBadRequest, "content or url is required")
|
|
}
|
|
out, err := parser.Parse(ctx, call, in)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if out == nil {
|
|
out = &pluginapi.ParseOutput{}
|
|
}
|
|
return out, nil
|
|
})(w, r)
|
|
})
|
|
}
|