mirror of
https://github.com/Tencent/WeKnora.git
synced 2026-09-21 13:52:09 +08:00
feat: Update web search providers and enhance knowledge base management
- Added DuckDuckGo as a new web search provider in the configuration. - Removed deprecated web search providers (Baidu, Bing, Bocha, Exa, Google, Kuaisou, Searxng, Tavily, Zhipu) to streamline the codebase. - Enhanced knowledge base management by introducing temporary knowledge base handling in the session service. - Updated the web search service to support RAG-based compression for improved search result processing. - Refactored session and knowledge service interfaces to accommodate new functionalities.
This commit is contained in:
wizardchen
+5
-46
@@ -618,57 +618,16 @@ agent:
|
||||
web_search:
|
||||
# 可用搜索引擎列表
|
||||
providers:
|
||||
- id: "kuaisou"
|
||||
name: "快搜"
|
||||
free: false
|
||||
requires_api_key: true
|
||||
description: "高性能实时网页搜索"
|
||||
api_url: "https://platform.kuaisou.com/api/v1/search"
|
||||
- id: "google"
|
||||
name: "Google"
|
||||
- id: "duckduckgo"
|
||||
name: "DuckDuckGo"
|
||||
free: true
|
||||
requires_api_key: false
|
||||
description: "Google搜索（免费）"
|
||||
api_url: "https://www.googleapis.com/customsearch/v1"
|
||||
- id: "baidu"
|
||||
name: "百度"
|
||||
free: true
|
||||
requires_api_key: false
|
||||
description: "百度搜索（免费）"
|
||||
- id: "bing"
|
||||
name: "Bing"
|
||||
free: true
|
||||
requires_api_key: false
|
||||
description: "Bing搜索（免费）"
|
||||
- id: "bocha"
|
||||
name: "博查"
|
||||
free: false
|
||||
requires_api_key: true
|
||||
description: "专为AI应用设计的搜索API"
|
||||
- id: "zhipu"
|
||||
name: "智谱"
|
||||
free: false
|
||||
requires_api_key: true
|
||||
description: "智谱搜索API"
|
||||
- id: "tavily"
|
||||
name: "Tavily"
|
||||
free: false
|
||||
requires_api_key: true
|
||||
description: "Tavily AI搜索API"
|
||||
- id: "searxng"
|
||||
name: "Searxng"
|
||||
free: true
|
||||
requires_api_key: false
|
||||
description: "Searxng开源元搜索引擎"
|
||||
- id: "exa"
|
||||
name: "Exa"
|
||||
free: false
|
||||
requires_api_key: true
|
||||
description: "Exa AI搜索API"
|
||||
description: "DuckDuckGo API"
|
||||
|
||||
|
||||
# 默认配置
|
||||
default:
|
||||
provider: "kuaisou"
|
||||
provider: "duckduckgo"
|
||||
max_results: 5
|
||||
include_date: true
|
||||
compression_method: "none"
|
||||
|
||||
@@ -44,7 +44,7 @@
|
||||
</div>
|
||||
|
||||
<!-- Answer Event -->
|
||||
<div v-else-if="event.type === 'answer'" class="answer-event">
|
||||
<div v-else-if="event.type === 'answer' && event.content && event.content.trim()" class="answer-event">
|
||||
<div
|
||||
class="answer-content-wrapper"
|
||||
:class="{
|
||||
|
||||
@@ -32,9 +32,6 @@
|
||||
<div class="provider-option">
|
||||
<span class="provider-name">{{ provider.name }}</span>
|
||||
</div>
|
||||
<div v-if="provider.description" class="provider-desc">
|
||||
{{ provider.description }}
|
||||
</div>
|
||||
</div>
|
||||
</t-option>
|
||||
</t-select>
|
||||
|
||||
@@ -5,6 +5,7 @@ go 1.24.0
|
||||
toolchain go1.24.2
|
||||
|
||||
require (
|
||||
github.com/PuerkitoBio/goquery v1.10.3
|
||||
github.com/elastic/go-elasticsearch/v7 v7.17.10
|
||||
github.com/elastic/go-elasticsearch/v8 v8.18.0
|
||||
github.com/gin-contrib/cors v1.7.5
|
||||
@@ -45,6 +46,7 @@ require (
|
||||
|
||||
require (
|
||||
github.com/andybalholm/brotli v1.1.0 // indirect
|
||||
github.com/andybalholm/cascadia v1.3.3 // indirect
|
||||
github.com/bahlo/generic-list-go v0.2.0 // indirect
|
||||
github.com/buger/jsonparser v1.1.1 // indirect
|
||||
github.com/bytedance/sonic v1.13.2 // indirect
|
||||
|
||||
@@ -4,9 +4,13 @@ github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 h1:L/gRVlceqvL25
|
||||
github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E=
|
||||
github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY=
|
||||
github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU=
|
||||
github.com/PuerkitoBio/goquery v1.10.3 h1:pFYcNSqHxBD06Fpj/KsbStFRsgRATgnf3LeXiUkhzPo=
|
||||
github.com/PuerkitoBio/goquery v1.10.3/go.mod h1:tMUX0zDMHXYlAQk6p35XxQMqMweEKB7iK7iLNd4RH4Y=
|
||||
github.com/QcloudApi/qcloud_sign_golang v0.0.0-20141224014652-e4130a326409/go.mod h1:1pk82RBxDY/JZnPQrtqHlUFfCctgdorsd9M06fMynOM=
|
||||
github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M=
|
||||
github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY=
|
||||
github.com/andybalholm/cascadia v1.3.3 h1:AG2YHrzJIm4BZ19iwJ/DAua6Btl3IwJX+VI4kktS1LM=
|
||||
github.com/andybalholm/cascadia v1.3.3/go.mod h1:xNd9bqTn98Ln4DwST8/nG+H0yuB8Hmgu1YHNnWw0GeA=
|
||||
github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPnH1Wvgk=
|
||||
github.com/bahlo/generic-list-go v0.2.0/go.mod h1:2KvAjgMlE5NNynlg/5iLrrCCZ2+5xWbdbCW3pNTGyYg=
|
||||
github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs=
|
||||
@@ -103,6 +107,7 @@ github.com/golang-migrate/migrate/v4 v4.19.0/go.mod h1:9dyEcu+hO+G9hPSw8AIg50yg6
|
||||
github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
|
||||
github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
|
||||
github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
|
||||
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
|
||||
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
|
||||
github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
|
||||
@@ -285,6 +290,7 @@ github.com/yanyiwu/gojieba v1.4.5 h1:VyZogGtdFSnJbACHvDRvDreXPPVPCg8axKFUdblU/JI
|
||||
github.com/yanyiwu/gojieba v1.4.5/go.mod h1:JUq4DddFVGdHXJHxxepxRmhrKlDpaBxR8O28v6fKYLY=
|
||||
github.com/yosida95/uritemplate/v3 v3.0.2 h1:Ed3Oyj9yrmi9087+NczuL5BwkIc4wvTb5zIM+UJPGz4=
|
||||
github.com/yosida95/uritemplate/v3 v3.0.2/go.mod h1:ILOh0sOhIJR3+L/8afwt/kE++YT040gmv5BQTMR2HP4=
|
||||
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
|
||||
go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA=
|
||||
go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A=
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0 h1:TT4fX+nBOA/+LUkobKGW1ydGcn+G3vRw9+g5HwCphpk=
|
||||
@@ -315,22 +321,85 @@ go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
|
||||
go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
|
||||
golang.org/x/arch v0.15.0 h1:QtOrQd0bTUnhNVNndMpLHNWrDmYzZ2KDqSrEymqInZw=
|
||||
golang.org/x/arch v0.15.0/go.mod h1:JmwW7aLIoRUKgaTzhkiEFxvcEiQGyOg9BMonBJUS7EE=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
|
||||
golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc=
|
||||
golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
|
||||
golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
|
||||
golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
|
||||
golang.org/x/crypto v0.42.0 h1:chiH31gIWm57EkTXpwnqf8qeuMUi0yekh6mT2AvFlqI=
|
||||
golang.org/x/crypto v0.42.0/go.mod h1:4+rDnOTJhQCx2q7/j6rAN5XDw8kPjeaXEUR2eL94ix8=
|
||||
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
|
||||
golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
|
||||
golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
|
||||
golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
|
||||
golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
|
||||
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
|
||||
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
|
||||
golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
|
||||
golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
|
||||
golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk=
|
||||
golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
|
||||
golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
|
||||
golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4=
|
||||
golang.org/x/net v0.43.0 h1:lat02VYK2j4aLzMzecihNvTlJNQUq316m2Mr9rnM6YE=
|
||||
golang.org/x/net v0.43.0/go.mod h1:vhO1fvI4dGsIjh73sWfUVjj3N7CA9WkKJNQm2svM6Jg=
|
||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
|
||||
golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
|
||||
golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
|
||||
golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
|
||||
golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug=
|
||||
golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.36.0 h1:KVRy2GtZBrk1cBYA7MKu5bEZFxQk4NIDV6RLVcC8o0k=
|
||||
golang.org/x/sys v0.36.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
|
||||
golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
|
||||
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
|
||||
golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
|
||||
golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU=
|
||||
golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
|
||||
golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY=
|
||||
golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM=
|
||||
golang.org/x/term v0.35.0 h1:bZBVKBudEyhRcajGcNc3jIfWPqV4y/Kt2XcoigOWtDQ=
|
||||
golang.org/x/term v0.35.0/go.mod h1:TPGtkTLesOwf2DE8CgVYiZinHAOuy5AYUYT1lENIZnA=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
|
||||
golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
|
||||
golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
|
||||
golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
|
||||
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
|
||||
golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
|
||||
golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
|
||||
golang.org/x/text v0.29.0 h1:1neNs90w9YzJ9BocxfsQNHKuAT4pkghyXc4nhZ6sJvk=
|
||||
golang.org/x/text v0.29.0/go.mod h1:7MhJOA9CD2qZyOKYazxdYMF85OwPdEr9jTtBpO7ydH4=
|
||||
golang.org/x/time v0.13.0 h1:eUlYslOIt32DgYD6utsuUeHs4d7AsEYLuIAdg7FlYgI=
|
||||
golang.org/x/time v0.13.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4=
|
||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
|
||||
golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
|
||||
golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58=
|
||||
golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=
|
||||
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20250603155806-513f23925822 h1:oWVWY3NzT7KJppx2UKhKmzPq4SRe0LdCijVRwvGeikY=
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20250603155806-513f23925822/go.mod h1:h3c4v36UTKzUiuaOKQ6gr3S+0hovBtUrXzTG/i3+XEc=
|
||||
|
||||
@@ -52,7 +52,7 @@ func (r *knowledgeBaseRepository) ListKnowledgeBasesByTenantID(
|
||||
ctx context.Context, tenantID uint,
|
||||
) ([]*types.KnowledgeBase, error) {
|
||||
var kbs []*types.KnowledgeBase
|
||||
if err := r.db.WithContext(ctx).Where("tenant_id = ?", tenantID).
|
||||
if err := r.db.WithContext(ctx).Where("tenant_id = ? AND is_temporary = ?", tenantID, false).
|
||||
Order("created_at DESC").Find(&kbs).Error; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
@@ -2,6 +2,7 @@ package chatpipline
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
@@ -11,30 +12,37 @@ import (
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/Tencent/WeKnora/internal/types/interfaces"
|
||||
"github.com/redis/go-redis/v9"
|
||||
)
|
||||
|
||||
// PluginSearch implements search functionality for chat pipeline
|
||||
type PluginSearch struct {
|
||||
knowledgeBaseService interfaces.KnowledgeBaseService
|
||||
knowledgeService interfaces.KnowledgeService
|
||||
modelService interfaces.ModelService
|
||||
config *config.Config
|
||||
webSearchService interfaces.WebSearchService
|
||||
tenantService interfaces.TenantService
|
||||
redisClient *redis.Client
|
||||
}
|
||||
|
||||
func NewPluginSearch(eventManager *EventManager,
|
||||
knowledgeBaseService interfaces.KnowledgeBaseService,
|
||||
knowledgeService interfaces.KnowledgeService,
|
||||
modelService interfaces.ModelService,
|
||||
config *config.Config,
|
||||
webSearchService interfaces.WebSearchService,
|
||||
tenantService interfaces.TenantService,
|
||||
redisClient *redis.Client,
|
||||
) *PluginSearch {
|
||||
res := &PluginSearch{
|
||||
knowledgeBaseService: knowledgeBaseService,
|
||||
knowledgeService: knowledgeService,
|
||||
modelService: modelService,
|
||||
config: config,
|
||||
webSearchService: webSearchService,
|
||||
tenantService: tenantService,
|
||||
redisClient: redisClient,
|
||||
}
|
||||
eventManager.Register(res)
|
||||
return res
|
||||
@@ -62,44 +70,37 @@ func (p *PluginSearch) OnEvent(ctx context.Context,
|
||||
return ErrSearch.WithError(nil)
|
||||
}
|
||||
|
||||
// Run KB search and web search concurrently
|
||||
logger.Infof(ctx, "Searching across %d knowledge base(s): %v", len(knowledgeBaseIDs), knowledgeBaseIDs)
|
||||
|
||||
// Prepare search parameters
|
||||
searchParams := types.SearchParams{
|
||||
QueryText: strings.TrimSpace(chatManage.RewriteQuery),
|
||||
VectorThreshold: chatManage.VectorThreshold,
|
||||
KeywordThreshold: chatManage.KeywordThreshold,
|
||||
MatchCount: chatManage.EmbeddingTopK,
|
||||
}
|
||||
logger.Infof(ctx, "Search parameters: %v", searchParams)
|
||||
|
||||
// Parallel search across multiple knowledge bases
|
||||
var wg sync.WaitGroup
|
||||
var mu sync.Mutex
|
||||
var allResults []*types.SearchResult
|
||||
|
||||
for _, kbID := range knowledgeBaseIDs {
|
||||
wg.Add(1)
|
||||
go func(knowledgeBaseID string) {
|
||||
defer wg.Done()
|
||||
|
||||
results, err := p.knowledgeBaseService.HybridSearch(ctx, knowledgeBaseID, searchParams)
|
||||
if err != nil {
|
||||
logger.Errorf(ctx, "Failed to search KB %s: %v", knowledgeBaseID, err)
|
||||
return
|
||||
}
|
||||
|
||||
logger.Infof(ctx, "KB %s search results count: %d", knowledgeBaseID, len(results))
|
||||
allResults := make([]*types.SearchResult, 0)
|
||||
|
||||
wg.Add(2)
|
||||
// Goroutine 1: Knowledge base search (rewrite + processed)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
kbResults := p.searchKnowledgeBases(ctx, knowledgeBaseIDs, chatManage)
|
||||
if len(kbResults) > 0 {
|
||||
mu.Lock()
|
||||
allResults = append(allResults, results...)
|
||||
allResults = append(allResults, kbResults...)
|
||||
mu.Unlock()
|
||||
}(kbID)
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
// Goroutine 2: Web search (if enabled)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
webResults := p.searchWebIfEnabled(ctx, chatManage)
|
||||
if len(webResults) > 0 {
|
||||
mu.Lock()
|
||||
allResults = append(allResults, webResults...)
|
||||
mu.Unlock()
|
||||
}
|
||||
}()
|
||||
|
||||
wg.Wait()
|
||||
|
||||
logger.Infof(ctx, "Total search results from all KBs: %d", len(allResults))
|
||||
chatManage.SearchResult = allResults
|
||||
|
||||
// Add relevant results from chat history
|
||||
@@ -109,67 +110,6 @@ func (p *PluginSearch) OnEvent(ctx context.Context,
|
||||
chatManage.SearchResult = append(chatManage.SearchResult, historyResult...)
|
||||
}
|
||||
|
||||
// Try search with processed query if different from rewrite query
|
||||
if chatManage.RewriteQuery != chatManage.ProcessedQuery {
|
||||
searchParams.QueryText = strings.TrimSpace(chatManage.ProcessedQuery)
|
||||
logger.Infof(ctx, "Searching with processed query: %s", searchParams.QueryText)
|
||||
|
||||
var wg2 sync.WaitGroup
|
||||
var mu2 sync.Mutex
|
||||
var processedResults []*types.SearchResult
|
||||
|
||||
for _, kbID := range knowledgeBaseIDs {
|
||||
wg2.Add(1)
|
||||
go func(knowledgeBaseID string) {
|
||||
defer wg2.Done()
|
||||
|
||||
results, err := p.knowledgeBaseService.HybridSearch(ctx, knowledgeBaseID, searchParams)
|
||||
if err != nil {
|
||||
logger.Errorf(ctx, "Failed to search KB %s with processed query: %v", knowledgeBaseID, err)
|
||||
return
|
||||
}
|
||||
|
||||
logger.Infof(ctx, "KB %s processed query results count: %d", knowledgeBaseID, len(results))
|
||||
|
||||
mu2.Lock()
|
||||
processedResults = append(processedResults, results...)
|
||||
mu2.Unlock()
|
||||
}(kbID)
|
||||
}
|
||||
|
||||
wg2.Wait()
|
||||
|
||||
logger.Infof(ctx, "Total processed query results from all KBs: %d", len(processedResults))
|
||||
chatManage.SearchResult = append(chatManage.SearchResult, processedResults...)
|
||||
}
|
||||
|
||||
// Perform web search if enabled and merge results with KB search results
|
||||
if chatManage.WebSearchEnabled && p.webSearchService != nil && p.tenantService != nil && chatManage.TenantID > 0 {
|
||||
// Get tenant to retrieve web search config
|
||||
tenant, err := p.tenantService.GetTenantByID(ctx, chatManage.TenantID)
|
||||
if err != nil {
|
||||
logger.Warnf(ctx, "Failed to get tenant for web search: %v", err)
|
||||
} else if tenant != nil && tenant.WebSearchConfig != nil && tenant.WebSearchConfig.Provider != "" {
|
||||
// Perform web search in parallel with KB search (already completed)
|
||||
logger.Infof(ctx, "Performing web search with provider: %s", tenant.WebSearchConfig.Provider)
|
||||
webResults, err := p.webSearchService.Search(ctx, tenant.WebSearchConfig, chatManage.RewriteQuery)
|
||||
if err != nil {
|
||||
logger.Warnf(ctx, "Web search failed: %v", err)
|
||||
} else {
|
||||
// Convert web search results to SearchResult
|
||||
webSearchResults := convertWebSearchResults(webResults)
|
||||
logger.Infof(ctx, "Web search returned %d results", len(webSearchResults))
|
||||
// Merge web search results with KB search results
|
||||
if len(webSearchResults) > 0 {
|
||||
chatManage.SearchResult = append(chatManage.SearchResult, webSearchResults...)
|
||||
logger.Infof(ctx, "Merged web search results, total results: %d", len(chatManage.SearchResult))
|
||||
}
|
||||
}
|
||||
} else {
|
||||
logger.Warnf(ctx, "Web search enabled but no valid configuration found for tenant %d", chatManage.TenantID)
|
||||
}
|
||||
}
|
||||
|
||||
// Remove duplicate results
|
||||
chatManage.SearchResult = removeDuplicateResults(chatManage.SearchResult)
|
||||
|
||||
@@ -216,6 +156,136 @@ func removeDuplicateResults(results []*types.SearchResult) []*types.SearchResult
|
||||
return uniqueResults
|
||||
}
|
||||
|
||||
// searchKnowledgeBases performs KB searches for rewrite and processed queries across KB IDs
|
||||
func (p *PluginSearch) searchKnowledgeBases(ctx context.Context, knowledgeBaseIDs []string, chatManage *types.ChatManage) []*types.SearchResult {
|
||||
// Build base params for rewrite query
|
||||
baseParams := types.SearchParams{
|
||||
QueryText: strings.TrimSpace(chatManage.RewriteQuery),
|
||||
VectorThreshold: chatManage.VectorThreshold,
|
||||
KeywordThreshold: chatManage.KeywordThreshold,
|
||||
MatchCount: chatManage.EmbeddingTopK,
|
||||
}
|
||||
|
||||
var wg sync.WaitGroup
|
||||
var mu sync.Mutex
|
||||
var results []*types.SearchResult
|
||||
|
||||
// Search with rewrite query
|
||||
for _, kbID := range knowledgeBaseIDs {
|
||||
wg.Add(1)
|
||||
go func(knowledgeBaseID string) {
|
||||
defer wg.Done()
|
||||
res, err := p.knowledgeBaseService.HybridSearch(ctx, knowledgeBaseID, baseParams)
|
||||
if err != nil {
|
||||
logger.Errorf(ctx, "Failed to search KB %s: %v", knowledgeBaseID, err)
|
||||
return
|
||||
}
|
||||
logger.Infof(ctx, "KB %s search results count: %d", knowledgeBaseID, len(res))
|
||||
mu.Lock()
|
||||
results = append(results, res...)
|
||||
mu.Unlock()
|
||||
}(kbID)
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
|
||||
// If processed query differs, search again
|
||||
if chatManage.RewriteQuery != chatManage.ProcessedQuery {
|
||||
paramsProcessed := baseParams
|
||||
paramsProcessed.QueryText = strings.TrimSpace(chatManage.ProcessedQuery)
|
||||
logger.Infof(ctx, "Searching with processed query: %s", paramsProcessed.QueryText)
|
||||
|
||||
wg = sync.WaitGroup{}
|
||||
for _, kbID := range knowledgeBaseIDs {
|
||||
wg.Add(1)
|
||||
go func(knowledgeBaseID string) {
|
||||
defer wg.Done()
|
||||
res, err := p.knowledgeBaseService.HybridSearch(ctx, knowledgeBaseID, paramsProcessed)
|
||||
if err != nil {
|
||||
logger.Errorf(ctx, "Failed to search KB %s with processed query: %v", knowledgeBaseID, err)
|
||||
return
|
||||
}
|
||||
logger.Infof(ctx, "KB %s processed query results count: %d", knowledgeBaseID, len(res))
|
||||
mu.Lock()
|
||||
results = append(results, res...)
|
||||
mu.Unlock()
|
||||
}(kbID)
|
||||
}
|
||||
wg.Wait()
|
||||
}
|
||||
|
||||
logger.Infof(ctx, "Total KB results (rewrite + processed): %d", len(results))
|
||||
return results
|
||||
}
|
||||
|
||||
// searchWebIfEnabled executes web search when enabled and returns converted results
|
||||
func (p *PluginSearch) searchWebIfEnabled(ctx context.Context, chatManage *types.ChatManage) []*types.SearchResult {
|
||||
if !(chatManage.WebSearchEnabled && p.webSearchService != nil && p.tenantService != nil && chatManage.TenantID > 0) {
|
||||
return nil
|
||||
}
|
||||
tenant := ctx.Value(types.TenantInfoContextKey).(*types.Tenant)
|
||||
if tenant == nil || tenant.WebSearchConfig == nil || tenant.WebSearchConfig.Provider == "" {
|
||||
logger.Warnf(ctx, "Web search enabled but no valid configuration found for tenant %d", chatManage.TenantID)
|
||||
return nil
|
||||
}
|
||||
|
||||
logger.Infof(ctx, "Performing web search with provider: %s", tenant.WebSearchConfig.Provider)
|
||||
webResults, err := p.webSearchService.Search(ctx, tenant.WebSearchConfig, chatManage.RewriteQuery)
|
||||
if err != nil {
|
||||
logger.Warnf(ctx, "Web search failed: %v", err)
|
||||
return nil
|
||||
}
|
||||
// Build questions (rewrite + processed if different)
|
||||
questions := []string{strings.TrimSpace(chatManage.RewriteQuery)}
|
||||
if chatManage.ProcessedQuery != "" && chatManage.ProcessedQuery != chatManage.RewriteQuery {
|
||||
questions = append(questions, strings.TrimSpace(chatManage.ProcessedQuery))
|
||||
}
|
||||
// Load session-scoped temp KB state from Redis
|
||||
var tempKBID string
|
||||
seen := map[string]bool{}
|
||||
ids := []string{}
|
||||
stateKey := fmt.Sprintf("tempkb:%s", chatManage.SessionID)
|
||||
if raw, getErr := p.redisClient.Get(ctx, stateKey).Bytes(); getErr == nil && len(raw) > 0 {
|
||||
var state struct {
|
||||
KBID string `json:"kbID"`
|
||||
KnowledgeIDs []string `json:"knowledgeIDs"`
|
||||
SeenURLs map[string]bool `json:"seenURLs"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &state); err == nil {
|
||||
tempKBID = state.KBID
|
||||
ids = state.KnowledgeIDs
|
||||
if state.SeenURLs != nil {
|
||||
seen = state.SeenURLs
|
||||
}
|
||||
}
|
||||
}
|
||||
compressed, kbID, newSeen, newIDs, err := p.webSearchService.CompressWithRAG(
|
||||
ctx, chatManage.SessionID, tempKBID, questions, webResults, tenant.WebSearchConfig,
|
||||
p.knowledgeBaseService, p.knowledgeService, seen, ids,
|
||||
)
|
||||
if err != nil {
|
||||
logger.Warnf(ctx, "RAG compression failed, falling back to raw: %v", err)
|
||||
} else {
|
||||
webResults = compressed
|
||||
// Persist temp KB state back into Redis
|
||||
state := struct {
|
||||
KBID string `json:"kbID"`
|
||||
KnowledgeIDs []string `json:"knowledgeIDs"`
|
||||
SeenURLs map[string]bool `json:"seenURLs"`
|
||||
}{
|
||||
KBID: kbID,
|
||||
KnowledgeIDs: newIDs,
|
||||
SeenURLs: newSeen,
|
||||
}
|
||||
if b, mErr := json.Marshal(state); mErr == nil {
|
||||
_ = p.redisClient.Set(ctx, stateKey, b, 0).Err()
|
||||
}
|
||||
}
|
||||
res := convertWebSearchResults(webResults)
|
||||
logger.Infof(ctx, "Web search returned %d results", len(res))
|
||||
return res
|
||||
}
|
||||
|
||||
// convertWebSearchResults converts WebSearchResult to SearchResult
|
||||
// This is a duplicate of the function in service/web_search.go to avoid circular imports
|
||||
func convertWebSearchResults(webResults []*types.WebSearchResult) []*types.SearchResult {
|
||||
|
||||
@@ -351,7 +351,26 @@ func (s *knowledgeService) CreateKnowledgeFromURL(ctx context.Context,
|
||||
func (s *knowledgeService) CreateKnowledgeFromPassage(ctx context.Context,
|
||||
kbID string, passage []string,
|
||||
) (*types.Knowledge, error) {
|
||||
logger.Info(ctx, "Start creating knowledge from passage")
|
||||
return s.createKnowledgeFromPassageInternal(ctx, kbID, passage, false)
|
||||
}
|
||||
|
||||
// CreateKnowledgeFromPassageSync creates a knowledge entry from text passages and waits for indexing to complete.
|
||||
func (s *knowledgeService) CreateKnowledgeFromPassageSync(ctx context.Context,
|
||||
kbID string, passage []string,
|
||||
) (*types.Knowledge, error) {
|
||||
return s.createKnowledgeFromPassageInternal(ctx, kbID, passage, true)
|
||||
}
|
||||
|
||||
// createKnowledgeFromPassageInternal consolidates the common logic for creating knowledge from passages.
|
||||
// When syncMode is true, chunk processing is performed synchronously; otherwise, it's processed asynchronously.
|
||||
func (s *knowledgeService) createKnowledgeFromPassageInternal(ctx context.Context,
|
||||
kbID string, passage []string, syncMode bool,
|
||||
) (*types.Knowledge, error) {
|
||||
if syncMode {
|
||||
logger.Info(ctx, "Start creating knowledge from passage (sync)")
|
||||
} else {
|
||||
logger.Info(ctx, "Start creating knowledge from passage")
|
||||
}
|
||||
logger.Infof(ctx, "Knowledge base ID: %s, passage count: %d", kbID, len(passage))
|
||||
|
||||
// 验证段落内容安全性
|
||||
@@ -374,7 +393,11 @@ func (s *knowledgeService) CreateKnowledgeFromPassage(ctx context.Context,
|
||||
}
|
||||
|
||||
// Create knowledge record
|
||||
logger.Info(ctx, "Creating knowledge record")
|
||||
if syncMode {
|
||||
logger.Info(ctx, "Creating knowledge record (sync)")
|
||||
} else {
|
||||
logger.Info(ctx, "Creating knowledge record")
|
||||
}
|
||||
knowledge := &types.Knowledge{
|
||||
ID: uuid.New().String(),
|
||||
TenantID: ctx.Value(types.TenantIDContextKey).(uint),
|
||||
@@ -394,11 +417,16 @@ func (s *knowledgeService) CreateKnowledgeFromPassage(ctx context.Context,
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Process passages asynchronously
|
||||
logger.Info(ctx, "Starting asynchronous passage processing")
|
||||
go s.processDocumentFromPassage(ctx, kb, knowledge, safePassages)
|
||||
|
||||
logger.Infof(ctx, "Knowledge from passage created successfully, ID: %s", knowledge.ID)
|
||||
// Process passages
|
||||
if syncMode {
|
||||
logger.Info(ctx, "Processing passage synchronously")
|
||||
s.processDocumentFromPassage(ctx, kb, knowledge, safePassages)
|
||||
logger.Infof(ctx, "Knowledge from passage created successfully (sync), ID: %s", knowledge.ID)
|
||||
} else {
|
||||
logger.Info(ctx, "Starting asynchronous passage processing")
|
||||
go s.processDocumentFromPassage(ctx, kb, knowledge, safePassages)
|
||||
logger.Infof(ctx, "Knowledge from passage created successfully, ID: %s", knowledge.ID)
|
||||
}
|
||||
return knowledge, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -2,6 +2,7 @@ package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
@@ -16,6 +17,7 @@ import (
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/Tencent/WeKnora/internal/types/interfaces"
|
||||
"github.com/google/uuid"
|
||||
"github.com/redis/go-redis/v9"
|
||||
"go.opentelemetry.io/otel/attribute"
|
||||
"go.opentelemetry.io/otel/codes"
|
||||
)
|
||||
@@ -36,6 +38,8 @@ type sessionService struct {
|
||||
eventManager *chatpipline.EventManager // Event manager for chat pipeline
|
||||
agentService interfaces.AgentService // Service for agent operations
|
||||
sessionStorage llmcontext.ContextStorage // Session storage
|
||||
knowledgeService interfaces.KnowledgeService // Service for knowledge operations
|
||||
redisClient *redis.Client // Redis client for temp KB state
|
||||
}
|
||||
|
||||
// NewSessionService creates a new session service instance with all required dependencies
|
||||
@@ -43,22 +47,26 @@ func NewSessionService(cfg *config.Config,
|
||||
sessionRepo interfaces.SessionRepository,
|
||||
messageRepo interfaces.MessageRepository,
|
||||
knowledgeBaseService interfaces.KnowledgeBaseService,
|
||||
knowledgeService interfaces.KnowledgeService,
|
||||
modelService interfaces.ModelService,
|
||||
tenantService interfaces.TenantService,
|
||||
eventManager *chatpipline.EventManager,
|
||||
agentService interfaces.AgentService,
|
||||
sessionStorage llmcontext.ContextStorage,
|
||||
redisClient *redis.Client,
|
||||
) interfaces.SessionService {
|
||||
return &sessionService{
|
||||
cfg: cfg,
|
||||
sessionRepo: sessionRepo,
|
||||
messageRepo: messageRepo,
|
||||
knowledgeBaseService: knowledgeBaseService,
|
||||
knowledgeService: knowledgeService,
|
||||
modelService: modelService,
|
||||
tenantService: tenantService,
|
||||
eventManager: eventManager,
|
||||
agentService: agentService,
|
||||
sessionStorage: sessionStorage,
|
||||
redisClient: redisClient,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -202,6 +210,30 @@ func (s *sessionService) DeleteSession(ctx context.Context, id string) error {
|
||||
tenantID := ctx.Value(types.TenantIDContextKey).(uint)
|
||||
logger.Infof(ctx, "Deleting session, ID: %s, tenant ID: %d", id, tenantID)
|
||||
|
||||
// Cleanup temporary KB stored in Redis for this session
|
||||
if s.redisClient != nil {
|
||||
stateKey := fmt.Sprintf("tempkb:%s", id)
|
||||
if raw, getErr := s.redisClient.Get(ctx, stateKey).Bytes(); getErr == nil && len(raw) > 0 {
|
||||
var state struct {
|
||||
KBID string `json:"kbID"`
|
||||
KnowledgeIDs []string `json:"knowledgeIDs"`
|
||||
SeenURLs map[string]bool `json:"seenURLs"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &state); err == nil && strings.TrimSpace(state.KBID) != "" {
|
||||
logger.Infof(ctx, "Cleaning temporary KB for session %s: %s", id, state.KBID)
|
||||
for _, kid := range state.KnowledgeIDs {
|
||||
if delErr := s.knowledgeService.DeleteKnowledge(ctx, kid); delErr != nil {
|
||||
logger.Warnf(ctx, "Failed to delete temp knowledge %s: %v", kid, delErr)
|
||||
}
|
||||
}
|
||||
if delErr := s.knowledgeBaseService.DeleteKnowledgeBase(ctx, state.KBID); delErr != nil {
|
||||
logger.Warnf(ctx, "Failed to delete temp knowledge base %s: %v", state.KBID, delErr)
|
||||
}
|
||||
_ = s.redisClient.Del(ctx, stateKey).Err()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Delete session from repository
|
||||
err := s.sessionRepo.Delete(ctx, tenantID, id)
|
||||
if err != nil {
|
||||
|
||||
@@ -7,23 +7,238 @@ import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/application/service/web_search"
|
||||
"github.com/Tencent/WeKnora/internal/config"
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/Tencent/WeKnora/internal/types/interfaces"
|
||||
)
|
||||
|
||||
// WebSearchProvider defines the interface for web search providers
|
||||
type WebSearchProvider interface {
|
||||
Search(ctx context.Context, query string, maxResults int, includeDate bool) ([]*types.WebSearchResult, error)
|
||||
Name() string
|
||||
}
|
||||
|
||||
// WebSearchService provides web search functionality
|
||||
type WebSearchService struct {
|
||||
providers map[string]WebSearchProvider
|
||||
providers map[string]interfaces.WebSearchProvider
|
||||
config *config.WebSearchConfig
|
||||
}
|
||||
|
||||
// CompressWithRAG performs RAG-based compression using a temporary, hidden knowledge base.
|
||||
// The temporary knowledge base is deleted after use. The UI will not list it due to repo filtering.
|
||||
func (s *WebSearchService) CompressWithRAG(
|
||||
ctx context.Context, sessionID string, tempKBID string, questions []string,
|
||||
webSearchResults []*types.WebSearchResult, cfg *types.WebSearchConfig,
|
||||
kbSvc interfaces.KnowledgeBaseService, knowSvc interfaces.KnowledgeService,
|
||||
seenURLs map[string]bool, knowledgeIDs []string,
|
||||
) (compressed []*types.WebSearchResult, kbID string, newSeen map[string]bool, newIDs []string, err error) {
|
||||
if len(webSearchResults) == 0 || len(questions) == 0 {
|
||||
return
|
||||
}
|
||||
if cfg == nil {
|
||||
return nil, tempKBID, seenURLs, knowledgeIDs, fmt.Errorf("web search config is required for RAG compression")
|
||||
}
|
||||
if cfg.EmbeddingModelID == "" {
|
||||
return nil, tempKBID, seenURLs, knowledgeIDs, fmt.Errorf("embedding_model_id is required for RAG compression")
|
||||
}
|
||||
var createdKB *types.KnowledgeBase
|
||||
// reuse or create temp KB
|
||||
if strings.TrimSpace(tempKBID) != "" {
|
||||
createdKB, err = kbSvc.GetKnowledgeBaseByID(ctx, tempKBID)
|
||||
if err != nil {
|
||||
logger.Warnf(ctx, "Temp KB %s not available, recreating: %v", tempKBID, err)
|
||||
createdKB = nil
|
||||
}
|
||||
}
|
||||
if createdKB == nil {
|
||||
kb := &types.KnowledgeBase{
|
||||
Name: fmt.Sprintf("tmp-websearch-%d", time.Now().UnixNano()),
|
||||
Description: "Ephemeral search compression KB",
|
||||
IsTemporary: true,
|
||||
EmbeddingModelID: cfg.EmbeddingModelID,
|
||||
RerankModelID: cfg.RerankModelID,
|
||||
}
|
||||
createdKB, err = kbSvc.CreateKnowledgeBase(ctx, kb)
|
||||
if err != nil {
|
||||
return nil, tempKBID, seenURLs, knowledgeIDs, fmt.Errorf("failed to create temporary knowledge base: %w", err)
|
||||
}
|
||||
tempKBID = createdKB.ID
|
||||
}
|
||||
|
||||
// Ingest all web results as passages synchronously
|
||||
// dedupe by URL across queries within the same temp KB for this request/session
|
||||
if seenURLs == nil {
|
||||
seenURLs = map[string]bool{}
|
||||
}
|
||||
for _, r := range webSearchResults {
|
||||
sourceURL := r.URL
|
||||
title := strings.TrimSpace(r.Title)
|
||||
snippet := strings.TrimSpace(r.Snippet)
|
||||
body := strings.TrimSpace(r.Content)
|
||||
// skip if already ingested for this KB
|
||||
if sourceURL != "" && seenURLs[sourceURL] {
|
||||
continue
|
||||
}
|
||||
contentLines := make([]string, 0, 4)
|
||||
contentLines = append(contentLines, fmt.Sprintf("[sourceUrl]: %s", sourceURL))
|
||||
if title != "" {
|
||||
contentLines = append(contentLines, title)
|
||||
}
|
||||
if snippet != "" {
|
||||
contentLines = append(contentLines, snippet)
|
||||
}
|
||||
if body != "" {
|
||||
contentLines = append(contentLines, body)
|
||||
}
|
||||
knowledge, err := knowSvc.CreateKnowledgeFromPassageSync(ctx, createdKB.ID, contentLines)
|
||||
if err != nil {
|
||||
logger.Warnf(ctx, "failed to ingest passage into temp KB: %v", err)
|
||||
continue
|
||||
}
|
||||
if sourceURL != "" {
|
||||
seenURLs[sourceURL] = true
|
||||
}
|
||||
knowledgeIDs = append(knowledgeIDs, knowledge.ID)
|
||||
}
|
||||
|
||||
// Retrieve references for questions
|
||||
matchCount := cfg.DocumentFragments
|
||||
if matchCount <= 0 {
|
||||
matchCount = 3
|
||||
}
|
||||
var allRefs []*types.SearchResult
|
||||
for _, q := range questions {
|
||||
params := types.SearchParams{
|
||||
QueryText: q,
|
||||
VectorThreshold: 0.5,
|
||||
KeywordThreshold: 0.5,
|
||||
MatchCount: matchCount,
|
||||
}
|
||||
results, err := kbSvc.HybridSearch(ctx, tempKBID, params)
|
||||
if err != nil {
|
||||
logger.Warnf(ctx, "hybrid search failed for temp KB: %v", err)
|
||||
continue
|
||||
}
|
||||
allRefs = append(allRefs, results...)
|
||||
}
|
||||
|
||||
// Round-robin select references across the original results by source URL
|
||||
selected := s.selectReferencesRoundRobin(webSearchResults, allRefs, matchCount*len(webSearchResults))
|
||||
// Consolidate by URL back into the web results
|
||||
compressedResults := s.consolidateReferencesByURL(webSearchResults, selected)
|
||||
return compressedResults, tempKBID, seenURLs, knowledgeIDs, nil
|
||||
}
|
||||
|
||||
// selectReferencesRoundRobin selects up to limit references, distributing fairly across source URLs.
|
||||
func (s *WebSearchService) selectReferencesRoundRobin(
|
||||
raw []*types.WebSearchResult,
|
||||
refs []*types.SearchResult,
|
||||
limit int,
|
||||
) []*types.SearchResult {
|
||||
if limit <= 0 || len(refs) == 0 {
|
||||
return nil
|
||||
}
|
||||
// group refs by url marker in content
|
||||
urlToRefs := map[string][]*types.SearchResult{}
|
||||
for _, r := range refs {
|
||||
url := extractSourceURLFromContent(r.Content)
|
||||
if url == "" {
|
||||
continue
|
||||
}
|
||||
urlToRefs[url] = append(urlToRefs[url], r)
|
||||
}
|
||||
// preserve order based on raw results
|
||||
order := make([]string, 0, len(raw))
|
||||
seen := map[string]bool{}
|
||||
for _, r := range raw {
|
||||
if r.URL != "" && !seen[r.URL] {
|
||||
order = append(order, r.URL)
|
||||
seen[r.URL] = true
|
||||
}
|
||||
}
|
||||
var out []*types.SearchResult
|
||||
for len(out) < limit {
|
||||
progress := false
|
||||
for _, url := range order {
|
||||
if len(out) >= limit {
|
||||
break
|
||||
}
|
||||
list := urlToRefs[url]
|
||||
if len(list) == 0 {
|
||||
continue
|
||||
}
|
||||
out = append(out, list[0])
|
||||
urlToRefs[url] = list[1:]
|
||||
progress = true
|
||||
}
|
||||
if !progress {
|
||||
break
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// consolidateReferencesByURL merges selected references back into the original results grouped by URL.
|
||||
func (s *WebSearchService) consolidateReferencesByURL(
|
||||
raw []*types.WebSearchResult,
|
||||
selected []*types.SearchResult,
|
||||
) []*types.WebSearchResult {
|
||||
if len(selected) == 0 {
|
||||
return raw
|
||||
}
|
||||
agg := map[string][]string{}
|
||||
for _, ref := range selected {
|
||||
url := extractSourceURLFromContent(ref.Content)
|
||||
if url == "" {
|
||||
continue
|
||||
}
|
||||
// strip the first marker line to avoid duplication
|
||||
agg[url] = append(agg[url], stripMarker(ref.Content))
|
||||
}
|
||||
// build outputs, preserving raw ordering and metadata
|
||||
out := make([]*types.WebSearchResult, 0, len(raw))
|
||||
for _, r := range raw {
|
||||
parts := agg[r.URL]
|
||||
if len(parts) == 0 {
|
||||
out = append(out, r)
|
||||
continue
|
||||
}
|
||||
merged := strings.Join(parts, "\n---\n")
|
||||
out = append(out, &types.WebSearchResult{
|
||||
Title: r.Title,
|
||||
URL: r.URL,
|
||||
Snippet: r.Snippet,
|
||||
Content: merged,
|
||||
Source: r.Source,
|
||||
PublishedAt: r.PublishedAt,
|
||||
})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func extractSourceURLFromContent(content string) string {
|
||||
if content == "" {
|
||||
return ""
|
||||
}
|
||||
lines := strings.Split(content, "\n")
|
||||
if len(lines) == 0 {
|
||||
return ""
|
||||
}
|
||||
first := strings.TrimSpace(lines[0])
|
||||
const prefix = "[sourceUrl]: "
|
||||
if strings.HasPrefix(first, prefix) {
|
||||
return strings.TrimSpace(strings.TrimPrefix(first, prefix))
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func stripMarker(content string) string {
|
||||
lines := strings.Split(content, "\n")
|
||||
if len(lines) == 0 {
|
||||
return content
|
||||
}
|
||||
if strings.HasPrefix(strings.TrimSpace(lines[0]), "[sourceUrl]: ") {
|
||||
return strings.Join(lines[1:], "\n")
|
||||
}
|
||||
return content
|
||||
}
|
||||
|
||||
// Search performs web search using the specified provider
|
||||
// This method implements the interface expected by PluginSearch
|
||||
func (s *WebSearchService) Search(ctx context.Context, config *types.WebSearchConfig, query string) ([]*types.WebSearchResult, error) {
|
||||
@@ -36,11 +251,6 @@ func (s *WebSearchService) Search(ctx context.Context, config *types.WebSearchCo
|
||||
return nil, fmt.Errorf("web search provider %s is not available", config.Provider)
|
||||
}
|
||||
|
||||
// Set API key for providers that need it
|
||||
if config.APIKey != "" {
|
||||
s.setProviderAPIKey(config.Provider, provider, config)
|
||||
}
|
||||
|
||||
// Set timeout
|
||||
timeout := time.Duration(s.config.Timeout) * time.Second
|
||||
if timeout == 0 {
|
||||
@@ -75,44 +285,24 @@ func NewWebSearchService(cfg *config.Config) (*WebSearchService, error) {
|
||||
}
|
||||
|
||||
service := &WebSearchService{
|
||||
providers: make(map[string]WebSearchProvider),
|
||||
providers: make(map[string]interfaces.WebSearchProvider),
|
||||
config: cfg.WebSearch,
|
||||
}
|
||||
|
||||
// Initialize providers based on config
|
||||
for _, providerConfig := range cfg.WebSearch.Providers {
|
||||
var provider WebSearchProvider
|
||||
var provider interfaces.WebSearchProvider
|
||||
var err error
|
||||
|
||||
switch providerConfig.ID {
|
||||
case "kuaisou":
|
||||
provider, err = NewKuaisouProvider(providerConfig)
|
||||
case "baidu":
|
||||
provider, err = NewBaiduProvider(providerConfig)
|
||||
case "google":
|
||||
provider, err = NewGoogleProvider(providerConfig)
|
||||
case "bing":
|
||||
provider, err = NewBingProvider(providerConfig)
|
||||
case "bocha":
|
||||
provider, err = NewBochaProvider(providerConfig)
|
||||
case "zhipu":
|
||||
provider, err = NewZhipuProvider(providerConfig)
|
||||
case "tavily":
|
||||
provider, err = NewTavilyProvider(providerConfig)
|
||||
case "searxng":
|
||||
provider, err = NewSearxngProvider(providerConfig)
|
||||
case "exa":
|
||||
provider, err = NewExaProvider(providerConfig)
|
||||
case "duckduckgo":
|
||||
provider, err = web_search.NewDuckDuckGoProvider(providerConfig)
|
||||
default:
|
||||
logger.Warnf(context.Background(), "Unknown web search provider: %s", providerConfig.ID)
|
||||
continue
|
||||
return nil, fmt.Errorf("unknown web search provider: %s", providerConfig.ID)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
logger.Warnf(context.Background(), "Failed to initialize provider %s: %v", providerConfig.ID, err)
|
||||
continue
|
||||
return nil, fmt.Errorf("failed to initialize provider %s: %v", providerConfig.ID, err)
|
||||
}
|
||||
|
||||
service.providers[providerConfig.ID] = provider
|
||||
logger.Infof(context.Background(), "Initialized web search provider: %s", providerConfig.ID)
|
||||
}
|
||||
@@ -120,27 +310,6 @@ func NewWebSearchService(cfg *config.Config) (*WebSearchService, error) {
|
||||
return service, nil
|
||||
}
|
||||
|
||||
// setProviderAPIKey sets the API key for a provider based on its type
|
||||
func (s *WebSearchService) setProviderAPIKey(providerID string, provider WebSearchProvider, config *types.WebSearchConfig) {
|
||||
switch p := provider.(type) {
|
||||
case *KuaisouProvider:
|
||||
p.SetAPIKey(config.APIKey)
|
||||
case *BochaProvider:
|
||||
p.SetAPIKey(config.APIKey)
|
||||
case *ZhipuProvider:
|
||||
p.SetAPIKey(config.APIKey)
|
||||
case *TavilyProvider:
|
||||
p.SetAPIKey(config.APIKey)
|
||||
case *ExaProvider:
|
||||
p.SetAPIKey(config.APIKey)
|
||||
case *GoogleProvider:
|
||||
// Google needs both API key and search engine ID
|
||||
// For now, we'll use API key as search engine ID if not provided separately
|
||||
// This can be extended later to support separate fields
|
||||
p.SetAPIKey(config.APIKey, config.APIKey) // TODO: Add search engine ID to config
|
||||
}
|
||||
}
|
||||
|
||||
// filterBlacklist filters results based on blacklist rules
|
||||
func (s *WebSearchService) filterBlacklist(results []*types.WebSearchResult, blacklist []string) []*types.WebSearchResult {
|
||||
if len(blacklist) == 0 {
|
||||
|
||||
@@ -1,32 +0,0 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/config"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
)
|
||||
|
||||
// BaiduProvider implements web search using Baidu API
|
||||
type BaiduProvider struct {
|
||||
apiURL string
|
||||
}
|
||||
|
||||
// NewBaiduProvider creates a new Baidu provider
|
||||
func NewBaiduProvider(cfg config.WebSearchProviderConfig) (WebSearchProvider, error) {
|
||||
return &BaiduProvider{
|
||||
apiURL: cfg.APIURL,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Name returns the provider name
|
||||
func (p *BaiduProvider) Name() string {
|
||||
return "baidu"
|
||||
}
|
||||
|
||||
// Search performs a web search using Baidu API
|
||||
func (p *BaiduProvider) Search(ctx context.Context, query string, maxResults int, includeDate bool) ([]*types.WebSearchResult, error) {
|
||||
// TODO: Implement Baidu search API
|
||||
return nil, fmt.Errorf("baidu search provider is not yet implemented")
|
||||
}
|
||||
@@ -1,32 +0,0 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/config"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
)
|
||||
|
||||
// BingProvider implements web search using Bing Search API
|
||||
type BingProvider struct {
|
||||
apiURL string
|
||||
}
|
||||
|
||||
// NewBingProvider creates a new Bing provider
|
||||
func NewBingProvider(cfg config.WebSearchProviderConfig) (WebSearchProvider, error) {
|
||||
return &BingProvider{
|
||||
apiURL: cfg.APIURL,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Name returns the provider name
|
||||
func (p *BingProvider) Name() string {
|
||||
return "bing"
|
||||
}
|
||||
|
||||
// Search performs a web search using Bing Search API
|
||||
func (p *BingProvider) Search(ctx context.Context, query string, maxResults int, includeDate bool) ([]*types.WebSearchResult, error) {
|
||||
// TODO: Implement Bing Search API
|
||||
return nil, fmt.Errorf("bing search provider is not yet implemented")
|
||||
}
|
||||
@@ -1,38 +0,0 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/config"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
)
|
||||
|
||||
// BochaProvider implements web search using Bocha API
|
||||
type BochaProvider struct {
|
||||
apiKey string
|
||||
apiURL string
|
||||
}
|
||||
|
||||
// NewBochaProvider creates a new Bocha provider
|
||||
func NewBochaProvider(cfg config.WebSearchProviderConfig) (WebSearchProvider, error) {
|
||||
return &BochaProvider{
|
||||
apiURL: cfg.APIURL,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Name returns the provider name
|
||||
func (p *BochaProvider) Name() string {
|
||||
return "bocha"
|
||||
}
|
||||
|
||||
// Search performs a web search using Bocha API
|
||||
func (p *BochaProvider) Search(ctx context.Context, query string, maxResults int, includeDate bool) ([]*types.WebSearchResult, error) {
|
||||
// TODO: Implement Bocha search API
|
||||
return nil, fmt.Errorf("bocha search provider is not yet implemented")
|
||||
}
|
||||
|
||||
// SetAPIKey sets the API key for the provider
|
||||
func (p *BochaProvider) SetAPIKey(apiKey string) {
|
||||
p.apiKey = apiKey
|
||||
}
|
||||
@@ -0,0 +1,230 @@
|
||||
package web_search
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/PuerkitoBio/goquery"
|
||||
"github.com/Tencent/WeKnora/internal/config"
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/Tencent/WeKnora/internal/types/interfaces"
|
||||
)
|
||||
|
||||
// DuckDuckGoProvider implements web search using DuckDuckGo (HTML first, API fallback)
|
||||
type DuckDuckGoProvider struct {
|
||||
client *http.Client
|
||||
}
|
||||
|
||||
// NewDuckDuckGoProvider creates a new DuckDuckGo provider
|
||||
func NewDuckDuckGoProvider(_ config.WebSearchProviderConfig) (interfaces.WebSearchProvider, error) {
|
||||
return &DuckDuckGoProvider{
|
||||
client: &http.Client{
|
||||
Timeout: 30 * time.Second,
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Name returns the provider name
|
||||
func (p *DuckDuckGoProvider) Name() string {
|
||||
return "duckduckgo"
|
||||
}
|
||||
|
||||
// Search performs a web search using DuckDuckGo HTML endpoint with API fallback
|
||||
func (p *DuckDuckGoProvider) Search(ctx context.Context, query string, maxResults int, includeDate bool) ([]*types.WebSearchResult, error) {
|
||||
if maxResults <= 0 {
|
||||
maxResults = 5
|
||||
}
|
||||
// Try HTML scraping first (more reliable for general results)
|
||||
htmlResults, err := p.searchHTML(ctx, query, maxResults)
|
||||
if err == nil && len(htmlResults) > 0 {
|
||||
return htmlResults, nil
|
||||
}
|
||||
// Fallback to Instant Answer API
|
||||
apiResults, apiErr := p.searchAPI(ctx, query, maxResults)
|
||||
if apiErr == nil && len(apiResults) > 0 {
|
||||
return apiResults, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("duckduckgo HTML search failed: %w", err)
|
||||
}
|
||||
return nil, fmt.Errorf("duckduckgo API search failed: %w", apiErr)
|
||||
}
|
||||
|
||||
func (p *DuckDuckGoProvider) searchHTML(ctx context.Context, query string, maxResults int) ([]*types.WebSearchResult, error) {
|
||||
baseURL := "https://html.duckduckgo.com/html/"
|
||||
params := url.Values{}
|
||||
params.Set("q", query)
|
||||
// Prefer Chinese results if applicable; otherwise DDG will auto-detect
|
||||
params.Set("kl", "cn-zh")
|
||||
|
||||
reqURL := baseURL + "?" + params.Encode()
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", reqURL, nil)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create request: %w", err)
|
||||
}
|
||||
// Use a realistic UA to avoid blocks
|
||||
req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36")
|
||||
|
||||
resp, err := p.client.Do(req)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to perform request: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusAccepted {
|
||||
return nil, fmt.Errorf("duckduckgo HTML returned status %d", resp.StatusCode)
|
||||
}
|
||||
|
||||
doc, err := goquery.NewDocumentFromReader(resp.Body)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to parse HTML: %w", err)
|
||||
}
|
||||
|
||||
results := make([]*types.WebSearchResult, 0, maxResults)
|
||||
// Structure based on DDG HTML page
|
||||
doc.Find(".web-result").Each(func(i int, s *goquery.Selection) {
|
||||
if len(results) >= maxResults {
|
||||
return
|
||||
}
|
||||
titleNode := s.Find(".result__a")
|
||||
title := strings.TrimSpace(titleNode.Text())
|
||||
var link string
|
||||
if href, exists := titleNode.Attr("href"); exists {
|
||||
link = cleanDDGURL(href)
|
||||
}
|
||||
snippet := strings.TrimSpace(s.Find(".result__snippet").Text())
|
||||
if title != "" && link != "" {
|
||||
results = append(results, &types.WebSearchResult{
|
||||
Title: title,
|
||||
URL: link,
|
||||
Snippet: snippet,
|
||||
Source: "duckduckgo",
|
||||
})
|
||||
}
|
||||
})
|
||||
|
||||
logger.Infof(ctx, "DuckDuckGo HTML search returned %d results for query: %s", len(results), query)
|
||||
return results, nil
|
||||
}
|
||||
|
||||
func (p *DuckDuckGoProvider) searchAPI(ctx context.Context, query string, maxResults int) ([]*types.WebSearchResult, error) {
|
||||
baseURL := "https://api.duckduckgo.com/"
|
||||
params := url.Values{}
|
||||
params.Set("q", query)
|
||||
params.Set("format", "json")
|
||||
params.Set("no_html", "1")
|
||||
params.Set("skip_disambig", "1")
|
||||
|
||||
reqURL := baseURL + "?" + params.Encode()
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", reqURL, nil)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create request: %w", err)
|
||||
}
|
||||
req.Header.Set("User-Agent", "WeKnora/1.0")
|
||||
|
||||
resp, err := p.client.Do(req)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to perform request: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
return nil, fmt.Errorf("duckduckgo API returned status %d: %s", resp.StatusCode, string(body))
|
||||
}
|
||||
|
||||
var apiResponse struct {
|
||||
AbstractText string `json:"AbstractText"`
|
||||
AbstractURL string `json:"AbstractURL"`
|
||||
Heading string `json:"Heading"`
|
||||
RelatedTopics []struct {
|
||||
FirstURL string `json:"FirstURL"`
|
||||
Text string `json:"Text"`
|
||||
} `json:"RelatedTopics"`
|
||||
Results []struct {
|
||||
FirstURL string `json:"FirstURL"`
|
||||
Text string `json:"Text"`
|
||||
} `json:"Results"`
|
||||
}
|
||||
if err := json.NewDecoder(resp.Body).Decode(&apiResponse); err != nil {
|
||||
return nil, fmt.Errorf("failed to decode API response: %w", err)
|
||||
}
|
||||
|
||||
results := make([]*types.WebSearchResult, 0, maxResults)
|
||||
if apiResponse.AbstractText != "" && apiResponse.AbstractURL != "" {
|
||||
results = append(results, &types.WebSearchResult{
|
||||
Title: apiResponse.Heading,
|
||||
URL: apiResponse.AbstractURL,
|
||||
Snippet: apiResponse.AbstractText,
|
||||
Source: "duckduckgo",
|
||||
})
|
||||
}
|
||||
for _, topic := range apiResponse.RelatedTopics {
|
||||
if len(results) >= maxResults {
|
||||
break
|
||||
}
|
||||
if topic.Text != "" && topic.FirstURL != "" {
|
||||
results = append(results, &types.WebSearchResult{
|
||||
Title: extractTitle(topic.Text),
|
||||
URL: topic.FirstURL,
|
||||
Snippet: topic.Text,
|
||||
Source: "duckduckgo",
|
||||
})
|
||||
}
|
||||
}
|
||||
for _, r := range apiResponse.Results {
|
||||
if len(results) >= maxResults {
|
||||
break
|
||||
}
|
||||
if r.Text != "" && r.FirstURL != "" {
|
||||
results = append(results, &types.WebSearchResult{
|
||||
Title: extractTitle(r.Text),
|
||||
URL: r.FirstURL,
|
||||
Snippet: r.Text,
|
||||
Source: "duckduckgo",
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
logger.Infof(ctx, "DuckDuckGo API search returned %d results for query: %s", len(results), query)
|
||||
return results, nil
|
||||
}
|
||||
|
||||
func cleanDDGURL(urlStr string) string {
|
||||
if strings.HasPrefix(urlStr, "//duckduckgo.com/l/?uddg=") {
|
||||
trimmed := strings.TrimPrefix(urlStr, "//duckduckgo.com/l/?uddg=")
|
||||
if idx := strings.Index(trimmed, "&rut="); idx != -1 {
|
||||
decodedStr, err := url.PathUnescape(trimmed[:idx])
|
||||
if err == nil {
|
||||
return decodedStr
|
||||
}
|
||||
return ""
|
||||
}
|
||||
}
|
||||
if strings.HasPrefix(urlStr, "https://duckduckgo.com/l/?uddg=") {
|
||||
if parsedURL, err := url.Parse(urlStr); err == nil {
|
||||
if uddg := parsedURL.Query().Get("uddg"); uddg != "" {
|
||||
return uddg
|
||||
}
|
||||
}
|
||||
}
|
||||
return urlStr
|
||||
}
|
||||
|
||||
func extractTitle(text string) string {
|
||||
lines := strings.Split(text, "\n")
|
||||
if len(lines) > 0 {
|
||||
title := strings.TrimSpace(lines[0])
|
||||
if len(title) > 100 {
|
||||
title = title[:100] + "..."
|
||||
}
|
||||
return title
|
||||
}
|
||||
return strings.TrimSpace(text)
|
||||
}
|
||||
@@ -0,0 +1,154 @@
|
||||
package web_search
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/config"
|
||||
)
|
||||
|
||||
// testRoundTripper rewrites outgoing requests that target DuckDuckGo hosts
|
||||
// to the provided test server, preserving path and query.
|
||||
type testRoundTripper struct {
|
||||
base *url.URL
|
||||
next http.RoundTripper
|
||||
}
|
||||
|
||||
func (t *testRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) {
|
||||
// Only rewrite requests to duckduckgo hosts used by the provider
|
||||
if req.URL.Host == "html.duckduckgo.com" || req.URL.Host == "api.duckduckgo.com" {
|
||||
cloned := *req
|
||||
u := *req.URL
|
||||
u.Scheme = t.base.Scheme
|
||||
u.Host = t.base.Host
|
||||
// Keep original path; our test server handlers should register for the same paths.
|
||||
cloned.URL = &u
|
||||
req = &cloned
|
||||
}
|
||||
return t.next.RoundTrip(req)
|
||||
}
|
||||
|
||||
func newTestClient(ts *httptest.Server) *http.Client {
|
||||
baseURL, _ := url.Parse(ts.URL)
|
||||
return &http.Client{
|
||||
Timeout: 5 * time.Second,
|
||||
Transport: &testRoundTripper{
|
||||
base: baseURL,
|
||||
next: http.DefaultTransport,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func TestDuckDuckGoProvider_Name(t *testing.T) {
|
||||
p, _ := NewDuckDuckGoProvider(config.WebSearchProviderConfig{})
|
||||
if p.Name() != "duckduckgo" {
|
||||
t.Fatalf("expected provider name duckduckgo, got %s", p.Name())
|
||||
}
|
||||
}
|
||||
|
||||
func TestDuckDuckGoProvider_Search_HTMLSuccess(t *testing.T) {
|
||||
// Minimal HTML page with two results, matching selectors used in searchHTML
|
||||
html := `
|
||||
<html>
|
||||
<body>
|
||||
<div class="web-result">
|
||||
<a class="result__a" href="https://duckduckgo.com/l/?uddg=https%3A%2F%2Fexample.com%2Fpage1&rut=">Example One</a>
|
||||
<div class="result__snippet">Snippet one</div>
|
||||
</div>
|
||||
<div class="web-result">
|
||||
<a class="result__a" href="//duckduckgo.com/l/?uddg=https%3A%2F%2Fexample.org%2Fpage2&rut=">Example Two</a>
|
||||
<div class="result__snippet">Snippet two</div>
|
||||
</div>
|
||||
</body>
|
||||
</html>`
|
||||
|
||||
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
// Provider requests GET https://html.duckduckgo.com/html/?q=...&kl=...
|
||||
if r.URL.Path == "/html/" {
|
||||
w.WriteHeader(http.StatusOK)
|
||||
_, _ = w.Write([]byte(html))
|
||||
return
|
||||
}
|
||||
t.Fatalf("unexpected request path: %s", r.URL.Path)
|
||||
}))
|
||||
defer ts.Close()
|
||||
|
||||
// Build provider and inject our test client
|
||||
prov, _ := NewDuckDuckGoProvider(config.WebSearchProviderConfig{})
|
||||
dp := prov.(*DuckDuckGoProvider)
|
||||
dp.client = newTestClient(ts)
|
||||
|
||||
ctx := context.Background()
|
||||
results, err := dp.Search(ctx, "weknora", 5, false)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if len(results) != 2 {
|
||||
t.Fatalf("expected 2 results, got %d", len(results))
|
||||
}
|
||||
if results[0].Title != "Example One" || !strings.HasPrefix(results[0].URL, "https://example.com/") || results[0].Snippet != "Snippet one" {
|
||||
t.Fatalf("unexpected first result: %+v", results[0])
|
||||
}
|
||||
if results[1].Title != "Example Two" || !strings.HasPrefix(results[1].URL, "https://example.org/") || results[1].Snippet != "Snippet two" {
|
||||
t.Fatalf("unexpected second result: %+v", results[1])
|
||||
}
|
||||
}
|
||||
|
||||
func TestDuckDuckGoProvider_Search_APIFallback(t *testing.T) {
|
||||
// Simulate HTML returning non-OK to force API fallback, then a minimal API JSON
|
||||
apiResp := struct {
|
||||
AbstractText string `json:"AbstractText"`
|
||||
AbstractURL string `json:"AbstractURL"`
|
||||
Heading string `json:"Heading"`
|
||||
Results []struct {
|
||||
FirstURL string `json:"FirstURL"`
|
||||
Text string `json:"Text"`
|
||||
} `json:"Results"`
|
||||
}{
|
||||
AbstractText: "Abstract snippet",
|
||||
AbstractURL: "https://example.com/abstract",
|
||||
Heading: "Abstract Heading",
|
||||
Results: []struct {
|
||||
FirstURL string `json:"FirstURL"`
|
||||
Text string `json:"Text"`
|
||||
}{
|
||||
{FirstURL: "https://example.net/x", Text: "Title X - Detail X"},
|
||||
},
|
||||
}
|
||||
|
||||
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
switch r.URL.Path {
|
||||
case "/html/":
|
||||
// Force fallback by returning 500
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
default:
|
||||
// API endpoint path "/"
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
enc := json.NewEncoder(w)
|
||||
_ = enc.Encode(apiResp)
|
||||
}
|
||||
}))
|
||||
defer ts.Close()
|
||||
|
||||
prov, _ := NewDuckDuckGoProvider(config.WebSearchProviderConfig{})
|
||||
dp := prov.(*DuckDuckGoProvider)
|
||||
dp.client = newTestClient(ts)
|
||||
|
||||
ctx := context.Background()
|
||||
results, err := dp.Search(ctx, "weknora", 3, false)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if len(results) == 0 {
|
||||
t.Fatalf("expected some results from API fallback")
|
||||
}
|
||||
if results[0].URL != "https://example.com/abstract" || results[0].Title != "Abstract Heading" {
|
||||
t.Fatalf("unexpected first API result: %+v", results[0])
|
||||
}
|
||||
}
|
||||
@@ -1,38 +0,0 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/config"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
)
|
||||
|
||||
// ExaProvider implements web search using Exa API
|
||||
type ExaProvider struct {
|
||||
apiKey string
|
||||
apiURL string
|
||||
}
|
||||
|
||||
// NewExaProvider creates a new Exa provider
|
||||
func NewExaProvider(cfg config.WebSearchProviderConfig) (WebSearchProvider, error) {
|
||||
return &ExaProvider{
|
||||
apiURL: cfg.APIURL,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Name returns the provider name
|
||||
func (p *ExaProvider) Name() string {
|
||||
return "exa"
|
||||
}
|
||||
|
||||
// Search performs a web search using Exa API
|
||||
func (p *ExaProvider) Search(ctx context.Context, query string, maxResults int, includeDate bool) ([]*types.WebSearchResult, error) {
|
||||
// TODO: Implement Exa search API
|
||||
return nil, fmt.Errorf("exa search provider is not yet implemented")
|
||||
}
|
||||
|
||||
// SetAPIKey sets the API key for the provider
|
||||
func (p *ExaProvider) SetAPIKey(apiKey string) {
|
||||
p.apiKey = apiKey
|
||||
}
|
||||
@@ -1,40 +0,0 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/config"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
)
|
||||
|
||||
// GoogleProvider implements web search using Google Custom Search API
|
||||
type GoogleProvider struct {
|
||||
apiKey string
|
||||
searchEngineID string
|
||||
apiURL string
|
||||
}
|
||||
|
||||
// NewGoogleProvider creates a new Google provider
|
||||
func NewGoogleProvider(cfg config.WebSearchProviderConfig) (WebSearchProvider, error) {
|
||||
return &GoogleProvider{
|
||||
apiURL: cfg.APIURL,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Name returns the provider name
|
||||
func (p *GoogleProvider) Name() string {
|
||||
return "google"
|
||||
}
|
||||
|
||||
// Search performs a web search using Google Custom Search API
|
||||
func (p *GoogleProvider) Search(ctx context.Context, query string, maxResults int, includeDate bool) ([]*types.WebSearchResult, error) {
|
||||
// TODO: Implement Google Custom Search API
|
||||
return nil, fmt.Errorf("google search provider is not yet implemented")
|
||||
}
|
||||
|
||||
// SetAPIKey sets the API key and search engine ID
|
||||
func (p *GoogleProvider) SetAPIKey(apiKey, searchEngineID string) {
|
||||
p.apiKey = apiKey
|
||||
p.searchEngineID = searchEngineID
|
||||
}
|
||||
@@ -1,112 +0,0 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/config"
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
)
|
||||
|
||||
// KuaisouProvider implements web search using Kuaisou API
|
||||
type KuaisouProvider struct {
|
||||
apiKey string
|
||||
apiURL string
|
||||
client *http.Client
|
||||
}
|
||||
|
||||
// NewKuaisouProvider creates a new Kuaisou provider
|
||||
func NewKuaisouProvider(cfg config.WebSearchProviderConfig) (WebSearchProvider, error) {
|
||||
if cfg.APIURL == "" {
|
||||
return nil, fmt.Errorf("kuaisou API URL is required")
|
||||
}
|
||||
|
||||
return &KuaisouProvider{
|
||||
apiKey: "", // Will be set from tenant config
|
||||
apiURL: cfg.APIURL,
|
||||
client: &http.Client{
|
||||
Timeout: 10 * time.Second,
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Name returns the provider name
|
||||
func (p *KuaisouProvider) Name() string {
|
||||
return "kuaisou"
|
||||
}
|
||||
|
||||
// Search performs a web search using Kuaisou API
|
||||
func (p *KuaisouProvider) Search(ctx context.Context, query string, maxResults int, includeDate bool) ([]*types.WebSearchResult, error) {
|
||||
if p.apiKey == "" {
|
||||
return nil, fmt.Errorf("kuaisou API key is required")
|
||||
}
|
||||
|
||||
// Prepare request
|
||||
reqBody := map[string]interface{}{
|
||||
"query": query,
|
||||
"max_results": maxResults,
|
||||
}
|
||||
|
||||
jsonData, err := json.Marshal(reqBody)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to marshal request: %w", err)
|
||||
}
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, "POST", p.apiURL, bytes.NewBuffer(jsonData))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create request: %w", err)
|
||||
}
|
||||
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
req.Header.Set("Authorization", "Bearer "+p.apiKey)
|
||||
|
||||
// Send request
|
||||
resp, err := p.client.Do(req)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to send request: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
return nil, fmt.Errorf("kuaisou API returned status %d: %s", resp.StatusCode, string(body))
|
||||
}
|
||||
|
||||
// Parse response
|
||||
var response struct {
|
||||
Results []struct {
|
||||
Title string `json:"title"`
|
||||
URL string `json:"url"`
|
||||
Snippet string `json:"snippet"`
|
||||
} `json:"results"`
|
||||
}
|
||||
|
||||
if err := json.NewDecoder(resp.Body).Decode(&response); err != nil {
|
||||
return nil, fmt.Errorf("failed to decode response: %w", err)
|
||||
}
|
||||
|
||||
// Convert to WebSearchResult
|
||||
results := make([]*types.WebSearchResult, 0, len(response.Results))
|
||||
for _, item := range response.Results {
|
||||
results = append(results, &types.WebSearchResult{
|
||||
Title: item.Title,
|
||||
URL: item.URL,
|
||||
Snippet: item.Snippet,
|
||||
Source: "kuaisou",
|
||||
})
|
||||
}
|
||||
|
||||
logger.Infof(ctx, "Kuaisou search returned %d results for query: %s", len(results), query)
|
||||
return results, nil
|
||||
}
|
||||
|
||||
// SetAPIKey sets the API key for the provider
|
||||
func (p *KuaisouProvider) SetAPIKey(apiKey string) {
|
||||
p.apiKey = apiKey
|
||||
}
|
||||
@@ -1,32 +0,0 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/config"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
)
|
||||
|
||||
// SearxngProvider implements web search using Searxng API
|
||||
type SearxngProvider struct {
|
||||
apiURL string
|
||||
}
|
||||
|
||||
// NewSearxngProvider creates a new Searxng provider
|
||||
func NewSearxngProvider(cfg config.WebSearchProviderConfig) (WebSearchProvider, error) {
|
||||
return &SearxngProvider{
|
||||
apiURL: cfg.APIURL,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Name returns the provider name
|
||||
func (p *SearxngProvider) Name() string {
|
||||
return "searxng"
|
||||
}
|
||||
|
||||
// Search performs a web search using Searxng API
|
||||
func (p *SearxngProvider) Search(ctx context.Context, query string, maxResults int, includeDate bool) ([]*types.WebSearchResult, error) {
|
||||
// TODO: Implement Searxng search API
|
||||
return nil, fmt.Errorf("searxng search provider is not yet implemented")
|
||||
}
|
||||
@@ -1,38 +0,0 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/config"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
)
|
||||
|
||||
// TavilyProvider implements web search using Tavily API
|
||||
type TavilyProvider struct {
|
||||
apiKey string
|
||||
apiURL string
|
||||
}
|
||||
|
||||
// NewTavilyProvider creates a new Tavily provider
|
||||
func NewTavilyProvider(cfg config.WebSearchProviderConfig) (WebSearchProvider, error) {
|
||||
return &TavilyProvider{
|
||||
apiURL: cfg.APIURL,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Name returns the provider name
|
||||
func (p *TavilyProvider) Name() string {
|
||||
return "tavily"
|
||||
}
|
||||
|
||||
// Search performs a web search using Tavily API
|
||||
func (p *TavilyProvider) Search(ctx context.Context, query string, maxResults int, includeDate bool) ([]*types.WebSearchResult, error) {
|
||||
// TODO: Implement Tavily search API
|
||||
return nil, fmt.Errorf("tavily search provider is not yet implemented")
|
||||
}
|
||||
|
||||
// SetAPIKey sets the API key for the provider
|
||||
func (p *TavilyProvider) SetAPIKey(apiKey string) {
|
||||
p.apiKey = apiKey
|
||||
}
|
||||
@@ -1,38 +0,0 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/config"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
)
|
||||
|
||||
// ZhipuProvider implements web search using Zhipu API
|
||||
type ZhipuProvider struct {
|
||||
apiKey string
|
||||
apiURL string
|
||||
}
|
||||
|
||||
// NewZhipuProvider creates a new Zhipu provider
|
||||
func NewZhipuProvider(cfg config.WebSearchProviderConfig) (WebSearchProvider, error) {
|
||||
return &ZhipuProvider{
|
||||
apiURL: cfg.APIURL,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Name returns the provider name
|
||||
func (p *ZhipuProvider) Name() string {
|
||||
return "zhipu"
|
||||
}
|
||||
|
||||
// Search performs a web search using Zhipu API
|
||||
func (p *ZhipuProvider) Search(ctx context.Context, query string, maxResults int, includeDate bool) ([]*types.WebSearchResult, error) {
|
||||
// TODO: Implement Zhipu search API
|
||||
return nil, fmt.Errorf("zhipu search provider is not yet implemented")
|
||||
}
|
||||
|
||||
// SetAPIKey sets the API key for the provider
|
||||
func (p *ZhipuProvider) SetAPIKey(apiKey string) {
|
||||
p.apiKey = apiKey
|
||||
}
|
||||
@@ -1,33 +0,0 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/config"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
)
|
||||
|
||||
// BaiduProvider implements web search using Baidu API
|
||||
type BaiduProvider struct {
|
||||
apiURL string
|
||||
}
|
||||
|
||||
// NewBaiduProvider creates a new Baidu provider
|
||||
func NewBaiduProvider(cfg config.WebSearchProviderConfig) (WebSearchProvider, error) {
|
||||
return &BaiduProvider{
|
||||
apiURL: cfg.APIURL,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Name returns the provider name
|
||||
func (p *BaiduProvider) Name() string {
|
||||
return "baidu"
|
||||
}
|
||||
|
||||
// Search performs a web search using Baidu API
|
||||
func (p *BaiduProvider) Search(ctx context.Context, query string, maxResults int, includeDate bool) ([]*types.WebSearchResult, error) {
|
||||
// TODO: Implement Baidu search API
|
||||
return nil, fmt.Errorf("baidu search provider is not yet implemented")
|
||||
}
|
||||
|
||||
@@ -1,33 +0,0 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/config"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
)
|
||||
|
||||
// BingProvider implements web search using Bing Search API
|
||||
type BingProvider struct {
|
||||
apiURL string
|
||||
}
|
||||
|
||||
// NewBingProvider creates a new Bing provider
|
||||
func NewBingProvider(cfg config.WebSearchProviderConfig) (WebSearchProvider, error) {
|
||||
return &BingProvider{
|
||||
apiURL: cfg.APIURL,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Name returns the provider name
|
||||
func (p *BingProvider) Name() string {
|
||||
return "bing"
|
||||
}
|
||||
|
||||
// Search performs a web search using Bing Search API
|
||||
func (p *BingProvider) Search(ctx context.Context, query string, maxResults int, includeDate bool) ([]*types.WebSearchResult, error) {
|
||||
// TODO: Implement Bing Search API
|
||||
return nil, fmt.Errorf("bing search provider is not yet implemented")
|
||||
}
|
||||
|
||||
@@ -1,39 +0,0 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/config"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
)
|
||||
|
||||
// BochaProvider implements web search using Bocha API
|
||||
type BochaProvider struct {
|
||||
apiKey string
|
||||
apiURL string
|
||||
}
|
||||
|
||||
// NewBochaProvider creates a new Bocha provider
|
||||
func NewBochaProvider(cfg config.WebSearchProviderConfig) (WebSearchProvider, error) {
|
||||
return &BochaProvider{
|
||||
apiURL: cfg.APIURL,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Name returns the provider name
|
||||
func (p *BochaProvider) Name() string {
|
||||
return "bocha"
|
||||
}
|
||||
|
||||
// Search performs a web search using Bocha API
|
||||
func (p *BochaProvider) Search(ctx context.Context, query string, maxResults int, includeDate bool) ([]*types.WebSearchResult, error) {
|
||||
// TODO: Implement Bocha search API
|
||||
return nil, fmt.Errorf("bocha search provider is not yet implemented")
|
||||
}
|
||||
|
||||
// SetAPIKey sets the API key for the provider
|
||||
func (p *BochaProvider) SetAPIKey(apiKey string) {
|
||||
p.apiKey = apiKey
|
||||
}
|
||||
|
||||
@@ -1,39 +0,0 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/config"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
)
|
||||
|
||||
// ExaProvider implements web search using Exa API
|
||||
type ExaProvider struct {
|
||||
apiKey string
|
||||
apiURL string
|
||||
}
|
||||
|
||||
// NewExaProvider creates a new Exa provider
|
||||
func NewExaProvider(cfg config.WebSearchProviderConfig) (WebSearchProvider, error) {
|
||||
return &ExaProvider{
|
||||
apiURL: cfg.APIURL,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Name returns the provider name
|
||||
func (p *ExaProvider) Name() string {
|
||||
return "exa"
|
||||
}
|
||||
|
||||
// Search performs a web search using Exa API
|
||||
func (p *ExaProvider) Search(ctx context.Context, query string, maxResults int, includeDate bool) ([]*types.WebSearchResult, error) {
|
||||
// TODO: Implement Exa search API
|
||||
return nil, fmt.Errorf("exa search provider is not yet implemented")
|
||||
}
|
||||
|
||||
// SetAPIKey sets the API key for the provider
|
||||
func (p *ExaProvider) SetAPIKey(apiKey string) {
|
||||
p.apiKey = apiKey
|
||||
}
|
||||
|
||||
@@ -1,41 +0,0 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/config"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
)
|
||||
|
||||
// GoogleProvider implements web search using Google Custom Search API
|
||||
type GoogleProvider struct {
|
||||
apiKey string
|
||||
searchEngineID string
|
||||
apiURL string
|
||||
}
|
||||
|
||||
// NewGoogleProvider creates a new Google provider
|
||||
func NewGoogleProvider(cfg config.WebSearchProviderConfig) (WebSearchProvider, error) {
|
||||
return &GoogleProvider{
|
||||
apiURL: cfg.APIURL,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Name returns the provider name
|
||||
func (p *GoogleProvider) Name() string {
|
||||
return "google"
|
||||
}
|
||||
|
||||
// Search performs a web search using Google Custom Search API
|
||||
func (p *GoogleProvider) Search(ctx context.Context, query string, maxResults int, includeDate bool) ([]*types.WebSearchResult, error) {
|
||||
// TODO: Implement Google Custom Search API
|
||||
return nil, fmt.Errorf("google search provider is not yet implemented")
|
||||
}
|
||||
|
||||
// SetAPIKey sets the API key and search engine ID
|
||||
func (p *GoogleProvider) SetAPIKey(apiKey, searchEngineID string) {
|
||||
p.apiKey = apiKey
|
||||
p.searchEngineID = searchEngineID
|
||||
}
|
||||
|
||||
@@ -1,113 +0,0 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/config"
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
)
|
||||
|
||||
// KuaisouProvider implements web search using Kuaisou API
|
||||
type KuaisouProvider struct {
|
||||
apiKey string
|
||||
apiURL string
|
||||
client *http.Client
|
||||
}
|
||||
|
||||
// NewKuaisouProvider creates a new Kuaisou provider
|
||||
func NewKuaisouProvider(cfg config.WebSearchProviderConfig) (WebSearchProvider, error) {
|
||||
if cfg.APIURL == "" {
|
||||
return nil, fmt.Errorf("kuaisou API URL is required")
|
||||
}
|
||||
|
||||
return &KuaisouProvider{
|
||||
apiKey: "", // Will be set from tenant config
|
||||
apiURL: cfg.APIURL,
|
||||
client: &http.Client{
|
||||
Timeout: 10 * time.Second,
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Name returns the provider name
|
||||
func (p *KuaisouProvider) Name() string {
|
||||
return "kuaisou"
|
||||
}
|
||||
|
||||
// Search performs a web search using Kuaisou API
|
||||
func (p *KuaisouProvider) Search(ctx context.Context, query string, maxResults int, includeDate bool) ([]*types.WebSearchResult, error) {
|
||||
if p.apiKey == "" {
|
||||
return nil, fmt.Errorf("kuaisou API key is required")
|
||||
}
|
||||
|
||||
// Prepare request
|
||||
reqBody := map[string]interface{}{
|
||||
"query": query,
|
||||
"max_results": maxResults,
|
||||
}
|
||||
|
||||
jsonData, err := json.Marshal(reqBody)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to marshal request: %w", err)
|
||||
}
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, "POST", p.apiURL, bytes.NewBuffer(jsonData))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create request: %w", err)
|
||||
}
|
||||
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
req.Header.Set("Authorization", "Bearer "+p.apiKey)
|
||||
|
||||
// Send request
|
||||
resp, err := p.client.Do(req)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to send request: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
return nil, fmt.Errorf("kuaisou API returned status %d: %s", resp.StatusCode, string(body))
|
||||
}
|
||||
|
||||
// Parse response
|
||||
var response struct {
|
||||
Results []struct {
|
||||
Title string `json:"title"`
|
||||
URL string `json:"url"`
|
||||
Snippet string `json:"snippet"`
|
||||
} `json:"results"`
|
||||
}
|
||||
|
||||
if err := json.NewDecoder(resp.Body).Decode(&response); err != nil {
|
||||
return nil, fmt.Errorf("failed to decode response: %w", err)
|
||||
}
|
||||
|
||||
// Convert to WebSearchResult
|
||||
results := make([]*types.WebSearchResult, 0, len(response.Results))
|
||||
for _, item := range response.Results {
|
||||
results = append(results, &types.WebSearchResult{
|
||||
Title: item.Title,
|
||||
URL: item.URL,
|
||||
Snippet: item.Snippet,
|
||||
Source: "kuaisou",
|
||||
})
|
||||
}
|
||||
|
||||
logger.Infof(ctx, "Kuaisou search returned %d results for query: %s", len(results), query)
|
||||
return results, nil
|
||||
}
|
||||
|
||||
// SetAPIKey sets the API key for the provider
|
||||
func (p *KuaisouProvider) SetAPIKey(apiKey string) {
|
||||
p.apiKey = apiKey
|
||||
}
|
||||
|
||||
@@ -1,33 +0,0 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/config"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
)
|
||||
|
||||
// SearxngProvider implements web search using Searxng API
|
||||
type SearxngProvider struct {
|
||||
apiURL string
|
||||
}
|
||||
|
||||
// NewSearxngProvider creates a new Searxng provider
|
||||
func NewSearxngProvider(cfg config.WebSearchProviderConfig) (WebSearchProvider, error) {
|
||||
return &SearxngProvider{
|
||||
apiURL: cfg.APIURL,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Name returns the provider name
|
||||
func (p *SearxngProvider) Name() string {
|
||||
return "searxng"
|
||||
}
|
||||
|
||||
// Search performs a web search using Searxng API
|
||||
func (p *SearxngProvider) Search(ctx context.Context, query string, maxResults int, includeDate bool) ([]*types.WebSearchResult, error) {
|
||||
// TODO: Implement Searxng search API
|
||||
return nil, fmt.Errorf("searxng search provider is not yet implemented")
|
||||
}
|
||||
|
||||
@@ -1,39 +0,0 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/config"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
)
|
||||
|
||||
// TavilyProvider implements web search using Tavily API
|
||||
type TavilyProvider struct {
|
||||
apiKey string
|
||||
apiURL string
|
||||
}
|
||||
|
||||
// NewTavilyProvider creates a new Tavily provider
|
||||
func NewTavilyProvider(cfg config.WebSearchProviderConfig) (WebSearchProvider, error) {
|
||||
return &TavilyProvider{
|
||||
apiURL: cfg.APIURL,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Name returns the provider name
|
||||
func (p *TavilyProvider) Name() string {
|
||||
return "tavily"
|
||||
}
|
||||
|
||||
// Search performs a web search using Tavily API
|
||||
func (p *TavilyProvider) Search(ctx context.Context, query string, maxResults int, includeDate bool) ([]*types.WebSearchResult, error) {
|
||||
// TODO: Implement Tavily search API
|
||||
return nil, fmt.Errorf("tavily search provider is not yet implemented")
|
||||
}
|
||||
|
||||
// SetAPIKey sets the API key for the provider
|
||||
func (p *TavilyProvider) SetAPIKey(apiKey string) {
|
||||
p.apiKey = apiKey
|
||||
}
|
||||
|
||||
@@ -1,39 +0,0 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/config"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
)
|
||||
|
||||
// ZhipuProvider implements web search using Zhipu API
|
||||
type ZhipuProvider struct {
|
||||
apiKey string
|
||||
apiURL string
|
||||
}
|
||||
|
||||
// NewZhipuProvider creates a new Zhipu provider
|
||||
func NewZhipuProvider(cfg config.WebSearchProviderConfig) (WebSearchProvider, error) {
|
||||
return &ZhipuProvider{
|
||||
apiURL: cfg.APIURL,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Name returns the provider name
|
||||
func (p *ZhipuProvider) Name() string {
|
||||
return "zhipu"
|
||||
}
|
||||
|
||||
// Search performs a web search using Zhipu API
|
||||
func (p *ZhipuProvider) Search(ctx context.Context, query string, maxResults int, includeDate bool) ([]*types.WebSearchResult, error) {
|
||||
// TODO: Implement Zhipu search API
|
||||
return nil, fmt.Errorf("zhipu search provider is not yet implemented")
|
||||
}
|
||||
|
||||
// SetAPIKey sets the API key for the provider
|
||||
func (p *ZhipuProvider) SetAPIKey(apiKey string) {
|
||||
p.apiKey = apiKey
|
||||
}
|
||||
|
||||
@@ -128,6 +128,11 @@ func BuildContainer(container *dig.Container) *dig.Container {
|
||||
|
||||
// Chat pipeline components for processing chat requests
|
||||
must(container.Provide(chatpipline.NewEventManager))
|
||||
// Ensure Async task components are registered before invoking plugins that depend on KnowledgeService
|
||||
// KnowledgeService depends on *asynq.Client, and plugins (like PluginSearch) are invoked below.
|
||||
must(container.Provide(router.NewAsyncqClient))
|
||||
must(container.Provide(router.NewAsynqServer))
|
||||
must(container.Invoke(router.RunAsynqServer))
|
||||
must(container.Invoke(chatpipline.NewPluginTracing))
|
||||
must(container.Invoke(chatpipline.NewPluginSearch))
|
||||
must(container.Invoke(chatpipline.NewPluginRerank))
|
||||
@@ -159,9 +164,6 @@ func BuildContainer(container *dig.Container) *dig.Container {
|
||||
|
||||
// Router configuration
|
||||
must(container.Provide(router.NewRouter))
|
||||
must(container.Provide(router.NewAsyncqClient))
|
||||
must(container.Provide(router.NewAsynqServer))
|
||||
must(container.Invoke(router.RunAsynqServer))
|
||||
|
||||
return container
|
||||
}
|
||||
|
||||
@@ -203,11 +203,16 @@ func (h *Handler) AgentQA(c *gin.Context) {
|
||||
}
|
||||
|
||||
// Check if agent mode has changed
|
||||
currentAgentEnabled := session.AgentConfig.Enabled
|
||||
currentAgentEnabled := session.AgentConfig.AgentModeEnabled
|
||||
if request.AgentEnabled != currentAgentEnabled {
|
||||
logger.Infof(ctx, "Agent mode changed from %v to %v", currentAgentEnabled, request.AgentEnabled)
|
||||
configChanged = true
|
||||
}
|
||||
currentWebSearchEnabled := session.AgentConfig.AgentModeEnabled
|
||||
if request.WebSearchEnabled != currentWebSearchEnabled {
|
||||
logger.Infof(ctx, "Web search mode changed from %v to %v", currentWebSearchEnabled, request.WebSearchEnabled)
|
||||
configChanged = true
|
||||
}
|
||||
|
||||
// If configuration changed, clear context and update session
|
||||
if configChanged {
|
||||
@@ -219,8 +224,12 @@ func (h *Handler) AgentQA(c *gin.Context) {
|
||||
// Continue anyway - this is not a fatal error
|
||||
}
|
||||
}
|
||||
if knowledgeBasesChanged {
|
||||
// todo clear temp kb
|
||||
}
|
||||
session.AgentConfig.KnowledgeBases = request.KnowledgeBaseIDs
|
||||
session.AgentConfig.Enabled = request.AgentEnabled
|
||||
session.AgentConfig.AgentModeEnabled = request.AgentEnabled
|
||||
session.AgentConfig.WebSearchEnabled = request.WebSearchEnabled
|
||||
// Persist the session changes
|
||||
if err := h.sessionService.UpdateSession(ctx, session); err != nil {
|
||||
logger.Errorf(ctx, "Failed to update session %s: %v", sessionID, err)
|
||||
|
||||
@@ -29,10 +29,14 @@ func getAsynqRedisClientOpt() *asynq.RedisClientOpt {
|
||||
return opt
|
||||
}
|
||||
|
||||
func NewAsyncqClient() *asynq.Client {
|
||||
func NewAsyncqClient() (*asynq.Client, error) {
|
||||
opt := getAsynqRedisClientOpt()
|
||||
client := asynq.NewClient(opt)
|
||||
return client
|
||||
err := client.Ping()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return client, nil
|
||||
}
|
||||
|
||||
func NewAsynqServer() *asynq.Server {
|
||||
|
||||
@@ -24,8 +24,9 @@ type AgentConfig struct {
|
||||
// SessionAgentConfig represents session-level agent configuration
|
||||
// Sessions only store Enabled and KnowledgeBases; other configs are read from Tenant at runtime
|
||||
type SessionAgentConfig struct {
|
||||
Enabled bool `json:"enabled"` // Whether agent mode is enabled for this session
|
||||
KnowledgeBases []string `json:"knowledge_bases"` // Accessible knowledge base IDs for this session
|
||||
AgentModeEnabled bool `json:"agent_mode_enabled"` // Whether agent mode is enabled for this session
|
||||
WebSearchEnabled bool `json:"web_search_enabled"` // Whether web search is enabled for this session
|
||||
KnowledgeBases []string `json:"knowledge_bases"` // Accessible knowledge base IDs for this session
|
||||
}
|
||||
|
||||
// Value implements driver.Valuer interface for AgentConfig
|
||||
|
||||
@@ -22,6 +22,8 @@ type KnowledgeService interface {
|
||||
CreateKnowledgeFromURL(ctx context.Context, kbID string, url string, enableMultimodel *bool) (*types.Knowledge, error)
|
||||
// CreateKnowledgeFromPassage creates knowledge from text passages.
|
||||
CreateKnowledgeFromPassage(ctx context.Context, kbID string, passage []string) (*types.Knowledge, error)
|
||||
// CreateKnowledgeFromPassageSync creates knowledge from text passages and waits until chunks are indexed.
|
||||
CreateKnowledgeFromPassageSync(ctx context.Context, kbID string, passage []string) (*types.Knowledge, error)
|
||||
// GetKnowledgeByID retrieves knowledge by ID.
|
||||
GetKnowledgeByID(ctx context.Context, id string) (*types.Knowledge, error)
|
||||
// GetKnowledgeBatch retrieves a batch of knowledge by IDs.
|
||||
|
||||
@@ -6,7 +6,17 @@ import (
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
)
|
||||
|
||||
// WebSearchService defines the interface for web search service
|
||||
// WebSearchProvider defines the interface for web search providers
|
||||
type WebSearchProvider interface {
|
||||
Search(ctx context.Context, query string, maxResults int, includeDate bool) ([]*types.WebSearchResult, error)
|
||||
Name() string
|
||||
}
|
||||
|
||||
type WebSearchService interface {
|
||||
Search(ctx context.Context, config *types.WebSearchConfig, query string) ([]*types.WebSearchResult, error)
|
||||
CompressWithRAG(ctx context.Context, sessionID string, tempKBID string, questions []string,
|
||||
webSearchResults []*types.WebSearchResult, cfg *types.WebSearchConfig,
|
||||
kbSvc KnowledgeBaseService, knowSvc KnowledgeService,
|
||||
seenURLs map[string]bool, knowledgeIDs []string,
|
||||
) (compressed []*types.WebSearchResult, kbID string, newSeen map[string]bool, newIDs []string, err error)
|
||||
}
|
||||
|
||||
@@ -18,6 +18,8 @@ type KnowledgeBase struct {
|
||||
ID string `yaml:"id" json:"id" gorm:"type:varchar(36);primaryKey"`
|
||||
// Name of the knowledge base
|
||||
Name string `yaml:"name" json:"name"`
|
||||
// Whether this knowledge base is temporary (ephemeral) and should be hidden from UI
|
||||
IsTemporary bool `yaml:"is_temporary" json:"is_temporary" gorm:"default:false"`
|
||||
// Description of the knowledge base
|
||||
Description string `yaml:"description" json:"description"`
|
||||
// Tenant ID
|
||||
|
||||
@@ -44,7 +44,7 @@ type WebSearchResult struct {
|
||||
URL string `json:"url"` // 结果URL
|
||||
Snippet string `json:"snippet"` // 摘要片段
|
||||
Content string `json:"content"` // 完整内容（可选，需要额外抓取）
|
||||
Source string `json:"source"` // 来源（如：kuaisou, baidu等）
|
||||
Source string `json:"source"` // 来源（如：duckduckgo等）
|
||||
PublishedAt *time.Time `json:"published_at,omitempty"` // 发布时间（如果有）
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,5 @@
|
||||
-- Add is_temporary flag to knowledge_bases to support ephemeral KBs
|
||||
ALTER TABLE knowledge_bases
|
||||
ADD COLUMN IF NOT EXISTS is_temporary TINYINT(1) NOT NULL DEFAULT 0 COMMENT 'Temporary/hidden KB';
|
||||
|
||||
|
||||
@@ -0,0 +1,5 @@
|
||||
-- Add is_temporary flag to knowledge_bases to support ephemeral KBs
|
||||
ALTER TABLE knowledge_bases
|
||||
ADD COLUMN IF NOT EXISTS is_temporary BOOLEAN NOT NULL DEFAULT FALSE;
|
||||
|
||||
|
||||
Reference in New Issue
Block a user