diff --git a/config/config.yaml b/config/config.yaml index 29ccb79ce..04afd9479 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -618,57 +618,16 @@ agent: web_search: # 可用搜索引擎列表 providers: - - id: "kuaisou" - name: "快搜" - free: false - requires_api_key: true - description: "高性能实时网页搜索" - api_url: "https://platform.kuaisou.com/api/v1/search" - - id: "google" - name: "Google" + - id: "duckduckgo" + name: "DuckDuckGo" free: true requires_api_key: false - description: "Google搜索（免费）" - api_url: "https://www.googleapis.com/customsearch/v1" - - id: "baidu" - name: "百度" - free: true - requires_api_key: false - description: "百度搜索（免费）" - - id: "bing" - name: "Bing" - free: true - requires_api_key: false - description: "Bing搜索（免费）" - - id: "bocha" - name: "博查" - free: false - requires_api_key: true - description: "专为AI应用设计的搜索API" - - id: "zhipu" - name: "智谱" - free: false - requires_api_key: true - description: "智谱搜索API" - - id: "tavily" - name: "Tavily" - free: false - requires_api_key: true - description: "Tavily AI搜索API" - - id: "searxng" - name: "Searxng" - free: true - requires_api_key: false - description: "Searxng开源元搜索引擎" - - id: "exa" - name: "Exa" - free: false - requires_api_key: true - description: "Exa AI搜索API" + description: "DuckDuckGo API" + # 默认配置 default: - provider: "kuaisou" + provider: "duckduckgo" max_results: 5 include_date: true compression_method: "none" diff --git a/frontend/src/views/chat/components/AgentStreamDisplay.vue b/frontend/src/views/chat/components/AgentStreamDisplay.vue index 9e305bda5..3d78a4b38 100644 --- a/frontend/src/views/chat/components/AgentStreamDisplay.vue +++ b/frontend/src/views/chat/components/AgentStreamDisplay.vue @@ -44,7 +44,7 @@ -
+
{{ provider.name }}
-
- {{ provider.description }} -
diff --git a/go.mod b/go.mod index ce23823c2..4c1c65308 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.24.0 toolchain go1.24.2 require ( + github.com/PuerkitoBio/goquery v1.10.3 github.com/elastic/go-elasticsearch/v7 v7.17.10 github.com/elastic/go-elasticsearch/v8 v8.18.0 github.com/gin-contrib/cors v1.7.5 @@ -45,6 +46,7 @@ require ( require ( github.com/andybalholm/brotli v1.1.0 // indirect + github.com/andybalholm/cascadia v1.3.3 // indirect github.com/bahlo/generic-list-go v0.2.0 // indirect github.com/buger/jsonparser v1.1.1 // indirect github.com/bytedance/sonic v1.13.2 // indirect diff --git a/go.sum b/go.sum index b810460ff..a2a66eafa 100644 --- a/go.sum +++ b/go.sum @@ -4,9 +4,13 @@ github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 h1:L/gRVlceqvL25 github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= +github.com/PuerkitoBio/goquery v1.10.3 h1:pFYcNSqHxBD06Fpj/KsbStFRsgRATgnf3LeXiUkhzPo= +github.com/PuerkitoBio/goquery v1.10.3/go.mod h1:tMUX0zDMHXYlAQk6p35XxQMqMweEKB7iK7iLNd4RH4Y= github.com/QcloudApi/qcloud_sign_golang v0.0.0-20141224014652-e4130a326409/go.mod h1:1pk82RBxDY/JZnPQrtqHlUFfCctgdorsd9M06fMynOM= github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M= github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY= +github.com/andybalholm/cascadia v1.3.3 h1:AG2YHrzJIm4BZ19iwJ/DAua6Btl3IwJX+VI4kktS1LM= +github.com/andybalholm/cascadia v1.3.3/go.mod h1:xNd9bqTn98Ln4DwST8/nG+H0yuB8Hmgu1YHNnWw0GeA= github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPnH1Wvgk= github.com/bahlo/generic-list-go v0.2.0/go.mod h1:2KvAjgMlE5NNynlg/5iLrrCCZ2+5xWbdbCW3pNTGyYg= github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= @@ -103,6 +107,7 @@ github.com/golang-migrate/migrate/v4 v4.19.0/go.mod h1:9dyEcu+hO+G9hPSw8AIg50yg6 github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= @@ -285,6 +290,7 @@ github.com/yanyiwu/gojieba v1.4.5 h1:VyZogGtdFSnJbACHvDRvDreXPPVPCg8axKFUdblU/JI github.com/yanyiwu/gojieba v1.4.5/go.mod h1:JUq4DddFVGdHXJHxxepxRmhrKlDpaBxR8O28v6fKYLY= github.com/yosida95/uritemplate/v3 v3.0.2 h1:Ed3Oyj9yrmi9087+NczuL5BwkIc4wvTb5zIM+UJPGz4= github.com/yosida95/uritemplate/v3 v3.0.2/go.mod h1:ILOh0sOhIJR3+L/8afwt/kE++YT040gmv5BQTMR2HP4= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0 h1:TT4fX+nBOA/+LUkobKGW1ydGcn+G3vRw9+g5HwCphpk= @@ -315,22 +321,85 @@ go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= golang.org/x/arch v0.15.0 h1:QtOrQd0bTUnhNVNndMpLHNWrDmYzZ2KDqSrEymqInZw= golang.org/x/arch v0.15.0/go.mod h1:JmwW7aLIoRUKgaTzhkiEFxvcEiQGyOg9BMonBJUS7EE= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= +golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= +golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= +golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= golang.org/x/crypto v0.42.0 h1:chiH31gIWm57EkTXpwnqf8qeuMUi0yekh6mT2AvFlqI= golang.org/x/crypto v0.42.0/go.mod h1:4+rDnOTJhQCx2q7/j6rAN5XDw8kPjeaXEUR2eL94ix8= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= +golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= +golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= +golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= golang.org/x/net v0.43.0 h1:lat02VYK2j4aLzMzecihNvTlJNQUq316m2Mr9rnM6YE= golang.org/x/net v0.43.0/go.mod h1:vhO1fvI4dGsIjh73sWfUVjj3N7CA9WkKJNQm2svM6Jg= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= +golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug= golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.36.0 h1:KVRy2GtZBrk1cBYA7MKu5bEZFxQk4NIDV6RLVcC8o0k= golang.org/x/sys v0.36.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= +golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= +golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= +golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= +golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= golang.org/x/term v0.35.0 h1:bZBVKBudEyhRcajGcNc3jIfWPqV4y/Kt2XcoigOWtDQ= golang.org/x/term v0.35.0/go.mod h1:TPGtkTLesOwf2DE8CgVYiZinHAOuy5AYUYT1lENIZnA= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= golang.org/x/text v0.29.0 h1:1neNs90w9YzJ9BocxfsQNHKuAT4pkghyXc4nhZ6sJvk= golang.org/x/text v0.29.0/go.mod h1:7MhJOA9CD2qZyOKYazxdYMF85OwPdEr9jTtBpO7ydH4= golang.org/x/time v0.13.0 h1:eUlYslOIt32DgYD6utsuUeHs4d7AsEYLuIAdg7FlYgI= golang.org/x/time v0.13.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/genproto/googleapis/api v0.0.0-20250603155806-513f23925822 h1:oWVWY3NzT7KJppx2UKhKmzPq4SRe0LdCijVRwvGeikY= google.golang.org/genproto/googleapis/api v0.0.0-20250603155806-513f23925822/go.mod h1:h3c4v36UTKzUiuaOKQ6gr3S+0hovBtUrXzTG/i3+XEc= diff --git a/internal/application/repository/knowledgebase.go b/internal/application/repository/knowledgebase.go index efd7ca535..749c208ae 100644 --- a/internal/application/repository/knowledgebase.go +++ b/internal/application/repository/knowledgebase.go @@ -52,7 +52,7 @@ func (r *knowledgeBaseRepository) ListKnowledgeBasesByTenantID( ctx context.Context, tenantID uint, ) ([]*types.KnowledgeBase, error) { var kbs []*types.KnowledgeBase - if err := r.db.WithContext(ctx).Where("tenant_id = ?", tenantID). + if err := r.db.WithContext(ctx).Where("tenant_id = ? AND is_temporary = ?", tenantID, false). Order("created_at DESC").Find(&kbs).Error; err != nil { return nil, err } diff --git a/internal/application/service/chat_pipline/search.go b/internal/application/service/chat_pipline/search.go index 69fa0c2d2..26457eb39 100644 --- a/internal/application/service/chat_pipline/search.go +++ b/internal/application/service/chat_pipline/search.go @@ -2,6 +2,7 @@ package chatpipline import ( "context" + "encoding/json" "fmt" "strings" "sync" @@ -11,30 +12,37 @@ import ( "github.com/Tencent/WeKnora/internal/logger" "github.com/Tencent/WeKnora/internal/types" "github.com/Tencent/WeKnora/internal/types/interfaces" + "github.com/redis/go-redis/v9" ) // PluginSearch implements search functionality for chat pipeline type PluginSearch struct { knowledgeBaseService interfaces.KnowledgeBaseService + knowledgeService interfaces.KnowledgeService modelService interfaces.ModelService config *config.Config webSearchService interfaces.WebSearchService tenantService interfaces.TenantService + redisClient *redis.Client } func NewPluginSearch(eventManager *EventManager, knowledgeBaseService interfaces.KnowledgeBaseService, + knowledgeService interfaces.KnowledgeService, modelService interfaces.ModelService, config *config.Config, webSearchService interfaces.WebSearchService, tenantService interfaces.TenantService, + redisClient *redis.Client, ) *PluginSearch { res := &PluginSearch{ knowledgeBaseService: knowledgeBaseService, + knowledgeService: knowledgeService, modelService: modelService, config: config, webSearchService: webSearchService, tenantService: tenantService, + redisClient: redisClient, } eventManager.Register(res) return res @@ -62,44 +70,37 @@ func (p *PluginSearch) OnEvent(ctx context.Context, return ErrSearch.WithError(nil) } + // Run KB search and web search concurrently logger.Infof(ctx, "Searching across %d knowledge base(s): %v", len(knowledgeBaseIDs), knowledgeBaseIDs) - - // Prepare search parameters - searchParams := types.SearchParams{ - QueryText: strings.TrimSpace(chatManage.RewriteQuery), - VectorThreshold: chatManage.VectorThreshold, - KeywordThreshold: chatManage.KeywordThreshold, - MatchCount: chatManage.EmbeddingTopK, - } - logger.Infof(ctx, "Search parameters: %v", searchParams) - - // Parallel search across multiple knowledge bases var wg sync.WaitGroup var mu sync.Mutex - var allResults []*types.SearchResult - - for _, kbID := range knowledgeBaseIDs { - wg.Add(1) - go func(knowledgeBaseID string) { - defer wg.Done() - - results, err := p.knowledgeBaseService.HybridSearch(ctx, knowledgeBaseID, searchParams) - if err != nil { - logger.Errorf(ctx, "Failed to search KB %s: %v", knowledgeBaseID, err) - return - } - - logger.Infof(ctx, "KB %s search results count: %d", knowledgeBaseID, len(results)) + allResults := make([]*types.SearchResult, 0) + wg.Add(2) + // Goroutine 1: Knowledge base search (rewrite + processed) + go func() { + defer wg.Done() + kbResults := p.searchKnowledgeBases(ctx, knowledgeBaseIDs, chatManage) + if len(kbResults) > 0 { mu.Lock() - allResults = append(allResults, results...) + allResults = append(allResults, kbResults...) mu.Unlock() - }(kbID) - } + } + }() + + // Goroutine 2: Web search (if enabled) + go func() { + defer wg.Done() + webResults := p.searchWebIfEnabled(ctx, chatManage) + if len(webResults) > 0 { + mu.Lock() + allResults = append(allResults, webResults...) + mu.Unlock() + } + }() wg.Wait() - logger.Infof(ctx, "Total search results from all KBs: %d", len(allResults)) chatManage.SearchResult = allResults // Add relevant results from chat history @@ -109,67 +110,6 @@ func (p *PluginSearch) OnEvent(ctx context.Context, chatManage.SearchResult = append(chatManage.SearchResult, historyResult...) } - // Try search with processed query if different from rewrite query - if chatManage.RewriteQuery != chatManage.ProcessedQuery { - searchParams.QueryText = strings.TrimSpace(chatManage.ProcessedQuery) - logger.Infof(ctx, "Searching with processed query: %s", searchParams.QueryText) - - var wg2 sync.WaitGroup - var mu2 sync.Mutex - var processedResults []*types.SearchResult - - for _, kbID := range knowledgeBaseIDs { - wg2.Add(1) - go func(knowledgeBaseID string) { - defer wg2.Done() - - results, err := p.knowledgeBaseService.HybridSearch(ctx, knowledgeBaseID, searchParams) - if err != nil { - logger.Errorf(ctx, "Failed to search KB %s with processed query: %v", knowledgeBaseID, err) - return - } - - logger.Infof(ctx, "KB %s processed query results count: %d", knowledgeBaseID, len(results)) - - mu2.Lock() - processedResults = append(processedResults, results...) - mu2.Unlock() - }(kbID) - } - - wg2.Wait() - - logger.Infof(ctx, "Total processed query results from all KBs: %d", len(processedResults)) - chatManage.SearchResult = append(chatManage.SearchResult, processedResults...) - } - - // Perform web search if enabled and merge results with KB search results - if chatManage.WebSearchEnabled && p.webSearchService != nil && p.tenantService != nil && chatManage.TenantID > 0 { - // Get tenant to retrieve web search config - tenant, err := p.tenantService.GetTenantByID(ctx, chatManage.TenantID) - if err != nil { - logger.Warnf(ctx, "Failed to get tenant for web search: %v", err) - } else if tenant != nil && tenant.WebSearchConfig != nil && tenant.WebSearchConfig.Provider != "" { - // Perform web search in parallel with KB search (already completed) - logger.Infof(ctx, "Performing web search with provider: %s", tenant.WebSearchConfig.Provider) - webResults, err := p.webSearchService.Search(ctx, tenant.WebSearchConfig, chatManage.RewriteQuery) - if err != nil { - logger.Warnf(ctx, "Web search failed: %v", err) - } else { - // Convert web search results to SearchResult - webSearchResults := convertWebSearchResults(webResults) - logger.Infof(ctx, "Web search returned %d results", len(webSearchResults)) - // Merge web search results with KB search results - if len(webSearchResults) > 0 { - chatManage.SearchResult = append(chatManage.SearchResult, webSearchResults...) - logger.Infof(ctx, "Merged web search results, total results: %d", len(chatManage.SearchResult)) - } - } - } else { - logger.Warnf(ctx, "Web search enabled but no valid configuration found for tenant %d", chatManage.TenantID) - } - } - // Remove duplicate results chatManage.SearchResult = removeDuplicateResults(chatManage.SearchResult) @@ -216,6 +156,136 @@ func removeDuplicateResults(results []*types.SearchResult) []*types.SearchResult return uniqueResults } +// searchKnowledgeBases performs KB searches for rewrite and processed queries across KB IDs +func (p *PluginSearch) searchKnowledgeBases(ctx context.Context, knowledgeBaseIDs []string, chatManage *types.ChatManage) []*types.SearchResult { + // Build base params for rewrite query + baseParams := types.SearchParams{ + QueryText: strings.TrimSpace(chatManage.RewriteQuery), + VectorThreshold: chatManage.VectorThreshold, + KeywordThreshold: chatManage.KeywordThreshold, + MatchCount: chatManage.EmbeddingTopK, + } + + var wg sync.WaitGroup + var mu sync.Mutex + var results []*types.SearchResult + + // Search with rewrite query + for _, kbID := range knowledgeBaseIDs { + wg.Add(1) + go func(knowledgeBaseID string) { + defer wg.Done() + res, err := p.knowledgeBaseService.HybridSearch(ctx, knowledgeBaseID, baseParams) + if err != nil { + logger.Errorf(ctx, "Failed to search KB %s: %v", knowledgeBaseID, err) + return + } + logger.Infof(ctx, "KB %s search results count: %d", knowledgeBaseID, len(res)) + mu.Lock() + results = append(results, res...) + mu.Unlock() + }(kbID) + } + + wg.Wait() + + // If processed query differs, search again + if chatManage.RewriteQuery != chatManage.ProcessedQuery { + paramsProcessed := baseParams + paramsProcessed.QueryText = strings.TrimSpace(chatManage.ProcessedQuery) + logger.Infof(ctx, "Searching with processed query: %s", paramsProcessed.QueryText) + + wg = sync.WaitGroup{} + for _, kbID := range knowledgeBaseIDs { + wg.Add(1) + go func(knowledgeBaseID string) { + defer wg.Done() + res, err := p.knowledgeBaseService.HybridSearch(ctx, knowledgeBaseID, paramsProcessed) + if err != nil { + logger.Errorf(ctx, "Failed to search KB %s with processed query: %v", knowledgeBaseID, err) + return + } + logger.Infof(ctx, "KB %s processed query results count: %d", knowledgeBaseID, len(res)) + mu.Lock() + results = append(results, res...) + mu.Unlock() + }(kbID) + } + wg.Wait() + } + + logger.Infof(ctx, "Total KB results (rewrite + processed): %d", len(results)) + return results +} + +// searchWebIfEnabled executes web search when enabled and returns converted results +func (p *PluginSearch) searchWebIfEnabled(ctx context.Context, chatManage *types.ChatManage) []*types.SearchResult { + if !(chatManage.WebSearchEnabled && p.webSearchService != nil && p.tenantService != nil && chatManage.TenantID > 0) { + return nil + } + tenant := ctx.Value(types.TenantInfoContextKey).(*types.Tenant) + if tenant == nil || tenant.WebSearchConfig == nil || tenant.WebSearchConfig.Provider == "" { + logger.Warnf(ctx, "Web search enabled but no valid configuration found for tenant %d", chatManage.TenantID) + return nil + } + + logger.Infof(ctx, "Performing web search with provider: %s", tenant.WebSearchConfig.Provider) + webResults, err := p.webSearchService.Search(ctx, tenant.WebSearchConfig, chatManage.RewriteQuery) + if err != nil { + logger.Warnf(ctx, "Web search failed: %v", err) + return nil + } + // Build questions (rewrite + processed if different) + questions := []string{strings.TrimSpace(chatManage.RewriteQuery)} + if chatManage.ProcessedQuery != "" && chatManage.ProcessedQuery != chatManage.RewriteQuery { + questions = append(questions, strings.TrimSpace(chatManage.ProcessedQuery)) + } + // Load session-scoped temp KB state from Redis + var tempKBID string + seen := map[string]bool{} + ids := []string{} + stateKey := fmt.Sprintf("tempkb:%s", chatManage.SessionID) + if raw, getErr := p.redisClient.Get(ctx, stateKey).Bytes(); getErr == nil && len(raw) > 0 { + var state struct { + KBID string `json:"kbID"` + KnowledgeIDs []string `json:"knowledgeIDs"` + SeenURLs map[string]bool `json:"seenURLs"` + } + if err := json.Unmarshal(raw, &state); err == nil { + tempKBID = state.KBID + ids = state.KnowledgeIDs + if state.SeenURLs != nil { + seen = state.SeenURLs + } + } + } + compressed, kbID, newSeen, newIDs, err := p.webSearchService.CompressWithRAG( + ctx, chatManage.SessionID, tempKBID, questions, webResults, tenant.WebSearchConfig, + p.knowledgeBaseService, p.knowledgeService, seen, ids, + ) + if err != nil { + logger.Warnf(ctx, "RAG compression failed, falling back to raw: %v", err) + } else { + webResults = compressed + // Persist temp KB state back into Redis + state := struct { + KBID string `json:"kbID"` + KnowledgeIDs []string `json:"knowledgeIDs"` + SeenURLs map[string]bool `json:"seenURLs"` + }{ + KBID: kbID, + KnowledgeIDs: newIDs, + SeenURLs: newSeen, + } + if b, mErr := json.Marshal(state); mErr == nil { + _ = p.redisClient.Set(ctx, stateKey, b, 0).Err() + } + } + res := convertWebSearchResults(webResults) + logger.Infof(ctx, "Web search returned %d results", len(res)) + return res +} + // convertWebSearchResults converts WebSearchResult to SearchResult // This is a duplicate of the function in service/web_search.go to avoid circular imports func convertWebSearchResults(webResults []*types.WebSearchResult) []*types.SearchResult { diff --git a/internal/application/service/knowledge.go b/internal/application/service/knowledge.go index 886866f06..7f756c06d 100644 --- a/internal/application/service/knowledge.go +++ b/internal/application/service/knowledge.go @@ -351,7 +351,26 @@ func (s *knowledgeService) CreateKnowledgeFromURL(ctx context.Context, func (s *knowledgeService) CreateKnowledgeFromPassage(ctx context.Context, kbID string, passage []string, ) (*types.Knowledge, error) { - logger.Info(ctx, "Start creating knowledge from passage") + return s.createKnowledgeFromPassageInternal(ctx, kbID, passage, false) +} + +// CreateKnowledgeFromPassageSync creates a knowledge entry from text passages and waits for indexing to complete. +func (s *knowledgeService) CreateKnowledgeFromPassageSync(ctx context.Context, + kbID string, passage []string, +) (*types.Knowledge, error) { + return s.createKnowledgeFromPassageInternal(ctx, kbID, passage, true) +} + +// createKnowledgeFromPassageInternal consolidates the common logic for creating knowledge from passages. +// When syncMode is true, chunk processing is performed synchronously; otherwise, it's processed asynchronously. +func (s *knowledgeService) createKnowledgeFromPassageInternal(ctx context.Context, + kbID string, passage []string, syncMode bool, +) (*types.Knowledge, error) { + if syncMode { + logger.Info(ctx, "Start creating knowledge from passage (sync)") + } else { + logger.Info(ctx, "Start creating knowledge from passage") + } logger.Infof(ctx, "Knowledge base ID: %s, passage count: %d", kbID, len(passage)) // 验证段落内容安全性 @@ -374,7 +393,11 @@ func (s *knowledgeService) CreateKnowledgeFromPassage(ctx context.Context, } // Create knowledge record - logger.Info(ctx, "Creating knowledge record") + if syncMode { + logger.Info(ctx, "Creating knowledge record (sync)") + } else { + logger.Info(ctx, "Creating knowledge record") + } knowledge := &types.Knowledge{ ID: uuid.New().String(), TenantID: ctx.Value(types.TenantIDContextKey).(uint), @@ -394,11 +417,16 @@ func (s *knowledgeService) CreateKnowledgeFromPassage(ctx context.Context, return nil, err } - // Process passages asynchronously - logger.Info(ctx, "Starting asynchronous passage processing") - go s.processDocumentFromPassage(ctx, kb, knowledge, safePassages) - - logger.Infof(ctx, "Knowledge from passage created successfully, ID: %s", knowledge.ID) + // Process passages + if syncMode { + logger.Info(ctx, "Processing passage synchronously") + s.processDocumentFromPassage(ctx, kb, knowledge, safePassages) + logger.Infof(ctx, "Knowledge from passage created successfully (sync), ID: %s", knowledge.ID) + } else { + logger.Info(ctx, "Starting asynchronous passage processing") + go s.processDocumentFromPassage(ctx, kb, knowledge, safePassages) + logger.Infof(ctx, "Knowledge from passage created successfully, ID: %s", knowledge.ID) + } return knowledge, nil } diff --git a/internal/application/service/session.go b/internal/application/service/session.go index 91d09a8cb..3f8c8f23d 100644 --- a/internal/application/service/session.go +++ b/internal/application/service/session.go @@ -2,6 +2,7 @@ package service import ( "context" + "encoding/json" "errors" "fmt" "strings" @@ -16,6 +17,7 @@ import ( "github.com/Tencent/WeKnora/internal/types" "github.com/Tencent/WeKnora/internal/types/interfaces" "github.com/google/uuid" + "github.com/redis/go-redis/v9" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/codes" ) @@ -36,6 +38,8 @@ type sessionService struct { eventManager *chatpipline.EventManager // Event manager for chat pipeline agentService interfaces.AgentService // Service for agent operations sessionStorage llmcontext.ContextStorage // Session storage + knowledgeService interfaces.KnowledgeService // Service for knowledge operations + redisClient *redis.Client // Redis client for temp KB state } // NewSessionService creates a new session service instance with all required dependencies @@ -43,22 +47,26 @@ func NewSessionService(cfg *config.Config, sessionRepo interfaces.SessionRepository, messageRepo interfaces.MessageRepository, knowledgeBaseService interfaces.KnowledgeBaseService, + knowledgeService interfaces.KnowledgeService, modelService interfaces.ModelService, tenantService interfaces.TenantService, eventManager *chatpipline.EventManager, agentService interfaces.AgentService, sessionStorage llmcontext.ContextStorage, + redisClient *redis.Client, ) interfaces.SessionService { return &sessionService{ cfg: cfg, sessionRepo: sessionRepo, messageRepo: messageRepo, knowledgeBaseService: knowledgeBaseService, + knowledgeService: knowledgeService, modelService: modelService, tenantService: tenantService, eventManager: eventManager, agentService: agentService, sessionStorage: sessionStorage, + redisClient: redisClient, } } @@ -202,6 +210,30 @@ func (s *sessionService) DeleteSession(ctx context.Context, id string) error { tenantID := ctx.Value(types.TenantIDContextKey).(uint) logger.Infof(ctx, "Deleting session, ID: %s, tenant ID: %d", id, tenantID) + // Cleanup temporary KB stored in Redis for this session + if s.redisClient != nil { + stateKey := fmt.Sprintf("tempkb:%s", id) + if raw, getErr := s.redisClient.Get(ctx, stateKey).Bytes(); getErr == nil && len(raw) > 0 { + var state struct { + KBID string `json:"kbID"` + KnowledgeIDs []string `json:"knowledgeIDs"` + SeenURLs map[string]bool `json:"seenURLs"` + } + if err := json.Unmarshal(raw, &state); err == nil && strings.TrimSpace(state.KBID) != "" { + logger.Infof(ctx, "Cleaning temporary KB for session %s: %s", id, state.KBID) + for _, kid := range state.KnowledgeIDs { + if delErr := s.knowledgeService.DeleteKnowledge(ctx, kid); delErr != nil { + logger.Warnf(ctx, "Failed to delete temp knowledge %s: %v", kid, delErr) + } + } + if delErr := s.knowledgeBaseService.DeleteKnowledgeBase(ctx, state.KBID); delErr != nil { + logger.Warnf(ctx, "Failed to delete temp knowledge base %s: %v", state.KBID, delErr) + } + _ = s.redisClient.Del(ctx, stateKey).Err() + } + } + } + // Delete session from repository err := s.sessionRepo.Delete(ctx, tenantID, id) if err != nil { diff --git a/internal/application/service/web_search.go b/internal/application/service/web_search.go index 2c1b1495b..d53e79fe3 100644 --- a/internal/application/service/web_search.go +++ b/internal/application/service/web_search.go @@ -7,23 +7,238 @@ import ( "strings" "time" + "github.com/Tencent/WeKnora/internal/application/service/web_search" "github.com/Tencent/WeKnora/internal/config" "github.com/Tencent/WeKnora/internal/logger" "github.com/Tencent/WeKnora/internal/types" + "github.com/Tencent/WeKnora/internal/types/interfaces" ) -// WebSearchProvider defines the interface for web search providers -type WebSearchProvider interface { - Search(ctx context.Context, query string, maxResults int, includeDate bool) ([]*types.WebSearchResult, error) - Name() string -} - // WebSearchService provides web search functionality type WebSearchService struct { - providers map[string]WebSearchProvider + providers map[string]interfaces.WebSearchProvider config *config.WebSearchConfig } +// CompressWithRAG performs RAG-based compression using a temporary, hidden knowledge base. +// The temporary knowledge base is deleted after use. The UI will not list it due to repo filtering. +func (s *WebSearchService) CompressWithRAG( + ctx context.Context, sessionID string, tempKBID string, questions []string, + webSearchResults []*types.WebSearchResult, cfg *types.WebSearchConfig, + kbSvc interfaces.KnowledgeBaseService, knowSvc interfaces.KnowledgeService, + seenURLs map[string]bool, knowledgeIDs []string, +) (compressed []*types.WebSearchResult, kbID string, newSeen map[string]bool, newIDs []string, err error) { + if len(webSearchResults) == 0 || len(questions) == 0 { + return + } + if cfg == nil { + return nil, tempKBID, seenURLs, knowledgeIDs, fmt.Errorf("web search config is required for RAG compression") + } + if cfg.EmbeddingModelID == "" { + return nil, tempKBID, seenURLs, knowledgeIDs, fmt.Errorf("embedding_model_id is required for RAG compression") + } + var createdKB *types.KnowledgeBase + // reuse or create temp KB + if strings.TrimSpace(tempKBID) != "" { + createdKB, err = kbSvc.GetKnowledgeBaseByID(ctx, tempKBID) + if err != nil { + logger.Warnf(ctx, "Temp KB %s not available, recreating: %v", tempKBID, err) + createdKB = nil + } + } + if createdKB == nil { + kb := &types.KnowledgeBase{ + Name: fmt.Sprintf("tmp-websearch-%d", time.Now().UnixNano()), + Description: "Ephemeral search compression KB", + IsTemporary: true, + EmbeddingModelID: cfg.EmbeddingModelID, + RerankModelID: cfg.RerankModelID, + } + createdKB, err = kbSvc.CreateKnowledgeBase(ctx, kb) + if err != nil { + return nil, tempKBID, seenURLs, knowledgeIDs, fmt.Errorf("failed to create temporary knowledge base: %w", err) + } + tempKBID = createdKB.ID + } + + // Ingest all web results as passages synchronously + // dedupe by URL across queries within the same temp KB for this request/session + if seenURLs == nil { + seenURLs = map[string]bool{} + } + for _, r := range webSearchResults { + sourceURL := r.URL + title := strings.TrimSpace(r.Title) + snippet := strings.TrimSpace(r.Snippet) + body := strings.TrimSpace(r.Content) + // skip if already ingested for this KB + if sourceURL != "" && seenURLs[sourceURL] { + continue + } + contentLines := make([]string, 0, 4) + contentLines = append(contentLines, fmt.Sprintf("[sourceUrl]: %s", sourceURL)) + if title != "" { + contentLines = append(contentLines, title) + } + if snippet != "" { + contentLines = append(contentLines, snippet) + } + if body != "" { + contentLines = append(contentLines, body) + } + knowledge, err := knowSvc.CreateKnowledgeFromPassageSync(ctx, createdKB.ID, contentLines) + if err != nil { + logger.Warnf(ctx, "failed to ingest passage into temp KB: %v", err) + continue + } + if sourceURL != "" { + seenURLs[sourceURL] = true + } + knowledgeIDs = append(knowledgeIDs, knowledge.ID) + } + + // Retrieve references for questions + matchCount := cfg.DocumentFragments + if matchCount <= 0 { + matchCount = 3 + } + var allRefs []*types.SearchResult + for _, q := range questions { + params := types.SearchParams{ + QueryText: q, + VectorThreshold: 0.5, + KeywordThreshold: 0.5, + MatchCount: matchCount, + } + results, err := kbSvc.HybridSearch(ctx, tempKBID, params) + if err != nil { + logger.Warnf(ctx, "hybrid search failed for temp KB: %v", err) + continue + } + allRefs = append(allRefs, results...) + } + + // Round-robin select references across the original results by source URL + selected := s.selectReferencesRoundRobin(webSearchResults, allRefs, matchCount*len(webSearchResults)) + // Consolidate by URL back into the web results + compressedResults := s.consolidateReferencesByURL(webSearchResults, selected) + return compressedResults, tempKBID, seenURLs, knowledgeIDs, nil +} + +// selectReferencesRoundRobin selects up to limit references, distributing fairly across source URLs. +func (s *WebSearchService) selectReferencesRoundRobin( + raw []*types.WebSearchResult, + refs []*types.SearchResult, + limit int, +) []*types.SearchResult { + if limit <= 0 || len(refs) == 0 { + return nil + } + // group refs by url marker in content + urlToRefs := map[string][]*types.SearchResult{} + for _, r := range refs { + url := extractSourceURLFromContent(r.Content) + if url == "" { + continue + } + urlToRefs[url] = append(urlToRefs[url], r) + } + // preserve order based on raw results + order := make([]string, 0, len(raw)) + seen := map[string]bool{} + for _, r := range raw { + if r.URL != "" && !seen[r.URL] { + order = append(order, r.URL) + seen[r.URL] = true + } + } + var out []*types.SearchResult + for len(out) < limit { + progress := false + for _, url := range order { + if len(out) >= limit { + break + } + list := urlToRefs[url] + if len(list) == 0 { + continue + } + out = append(out, list[0]) + urlToRefs[url] = list[1:] + progress = true + } + if !progress { + break + } + } + return out +} + +// consolidateReferencesByURL merges selected references back into the original results grouped by URL. +func (s *WebSearchService) consolidateReferencesByURL( + raw []*types.WebSearchResult, + selected []*types.SearchResult, +) []*types.WebSearchResult { + if len(selected) == 0 { + return raw + } + agg := map[string][]string{} + for _, ref := range selected { + url := extractSourceURLFromContent(ref.Content) + if url == "" { + continue + } + // strip the first marker line to avoid duplication + agg[url] = append(agg[url], stripMarker(ref.Content)) + } + // build outputs, preserving raw ordering and metadata + out := make([]*types.WebSearchResult, 0, len(raw)) + for _, r := range raw { + parts := agg[r.URL] + if len(parts) == 0 { + out = append(out, r) + continue + } + merged := strings.Join(parts, "\n---\n") + out = append(out, &types.WebSearchResult{ + Title: r.Title, + URL: r.URL, + Snippet: r.Snippet, + Content: merged, + Source: r.Source, + PublishedAt: r.PublishedAt, + }) + } + return out +} + +func extractSourceURLFromContent(content string) string { + if content == "" { + return "" + } + lines := strings.Split(content, "\n") + if len(lines) == 0 { + return "" + } + first := strings.TrimSpace(lines[0]) + const prefix = "[sourceUrl]: " + if strings.HasPrefix(first, prefix) { + return strings.TrimSpace(strings.TrimPrefix(first, prefix)) + } + return "" +} + +func stripMarker(content string) string { + lines := strings.Split(content, "\n") + if len(lines) == 0 { + return content + } + if strings.HasPrefix(strings.TrimSpace(lines[0]), "[sourceUrl]: ") { + return strings.Join(lines[1:], "\n") + } + return content +} + // Search performs web search using the specified provider // This method implements the interface expected by PluginSearch func (s *WebSearchService) Search(ctx context.Context, config *types.WebSearchConfig, query string) ([]*types.WebSearchResult, error) { @@ -36,11 +251,6 @@ func (s *WebSearchService) Search(ctx context.Context, config *types.WebSearchCo return nil, fmt.Errorf("web search provider %s is not available", config.Provider) } - // Set API key for providers that need it - if config.APIKey != "" { - s.setProviderAPIKey(config.Provider, provider, config) - } - // Set timeout timeout := time.Duration(s.config.Timeout) * time.Second if timeout == 0 { @@ -75,44 +285,24 @@ func NewWebSearchService(cfg *config.Config) (*WebSearchService, error) { } service := &WebSearchService{ - providers: make(map[string]WebSearchProvider), + providers: make(map[string]interfaces.WebSearchProvider), config: cfg.WebSearch, } // Initialize providers based on config for _, providerConfig := range cfg.WebSearch.Providers { - var provider WebSearchProvider + var provider interfaces.WebSearchProvider var err error switch providerConfig.ID { - case "kuaisou": - provider, err = NewKuaisouProvider(providerConfig) - case "baidu": - provider, err = NewBaiduProvider(providerConfig) - case "google": - provider, err = NewGoogleProvider(providerConfig) - case "bing": - provider, err = NewBingProvider(providerConfig) - case "bocha": - provider, err = NewBochaProvider(providerConfig) - case "zhipu": - provider, err = NewZhipuProvider(providerConfig) - case "tavily": - provider, err = NewTavilyProvider(providerConfig) - case "searxng": - provider, err = NewSearxngProvider(providerConfig) - case "exa": - provider, err = NewExaProvider(providerConfig) + case "duckduckgo": + provider, err = web_search.NewDuckDuckGoProvider(providerConfig) default: - logger.Warnf(context.Background(), "Unknown web search provider: %s", providerConfig.ID) - continue + return nil, fmt.Errorf("unknown web search provider: %s", providerConfig.ID) } - if err != nil { - logger.Warnf(context.Background(), "Failed to initialize provider %s: %v", providerConfig.ID, err) - continue + return nil, fmt.Errorf("failed to initialize provider %s: %v", providerConfig.ID, err) } - service.providers[providerConfig.ID] = provider logger.Infof(context.Background(), "Initialized web search provider: %s", providerConfig.ID) } @@ -120,27 +310,6 @@ func NewWebSearchService(cfg *config.Config) (*WebSearchService, error) { return service, nil } -// setProviderAPIKey sets the API key for a provider based on its type -func (s *WebSearchService) setProviderAPIKey(providerID string, provider WebSearchProvider, config *types.WebSearchConfig) { - switch p := provider.(type) { - case *KuaisouProvider: - p.SetAPIKey(config.APIKey) - case *BochaProvider: - p.SetAPIKey(config.APIKey) - case *ZhipuProvider: - p.SetAPIKey(config.APIKey) - case *TavilyProvider: - p.SetAPIKey(config.APIKey) - case *ExaProvider: - p.SetAPIKey(config.APIKey) - case *GoogleProvider: - // Google needs both API key and search engine ID - // For now, we'll use API key as search engine ID if not provided separately - // This can be extended later to support separate fields - p.SetAPIKey(config.APIKey, config.APIKey) // TODO: Add search engine ID to config - } -} - // filterBlacklist filters results based on blacklist rules func (s *WebSearchService) filterBlacklist(results []*types.WebSearchResult, blacklist []string) []*types.WebSearchResult { if len(blacklist) == 0 { diff --git a/internal/application/service/web_search/baidu.go b/internal/application/service/web_search/baidu.go deleted file mode 100644 index aa527eca0..000000000 --- a/internal/application/service/web_search/baidu.go +++ /dev/null @@ -1,32 +0,0 @@ -package service - -import ( - "context" - "fmt" - - "github.com/Tencent/WeKnora/internal/config" - "github.com/Tencent/WeKnora/internal/types" -) - -// BaiduProvider implements web search using Baidu API -type BaiduProvider struct { - apiURL string -} - -// NewBaiduProvider creates a new Baidu provider -func NewBaiduProvider(cfg config.WebSearchProviderConfig) (WebSearchProvider, error) { - return &BaiduProvider{ - apiURL: cfg.APIURL, - }, nil -} - -// Name returns the provider name -func (p *BaiduProvider) Name() string { - return "baidu" -} - -// Search performs a web search using Baidu API -func (p *BaiduProvider) Search(ctx context.Context, query string, maxResults int, includeDate bool) ([]*types.WebSearchResult, error) { - // TODO: Implement Baidu search API - return nil, fmt.Errorf("baidu search provider is not yet implemented") -} diff --git a/internal/application/service/web_search/bing.go b/internal/application/service/web_search/bing.go deleted file mode 100644 index bb617841a..000000000 --- a/internal/application/service/web_search/bing.go +++ /dev/null @@ -1,32 +0,0 @@ -package service - -import ( - "context" - "fmt" - - "github.com/Tencent/WeKnora/internal/config" - "github.com/Tencent/WeKnora/internal/types" -) - -// BingProvider implements web search using Bing Search API -type BingProvider struct { - apiURL string -} - -// NewBingProvider creates a new Bing provider -func NewBingProvider(cfg config.WebSearchProviderConfig) (WebSearchProvider, error) { - return &BingProvider{ - apiURL: cfg.APIURL, - }, nil -} - -// Name returns the provider name -func (p *BingProvider) Name() string { - return "bing" -} - -// Search performs a web search using Bing Search API -func (p *BingProvider) Search(ctx context.Context, query string, maxResults int, includeDate bool) ([]*types.WebSearchResult, error) { - // TODO: Implement Bing Search API - return nil, fmt.Errorf("bing search provider is not yet implemented") -} diff --git a/internal/application/service/web_search/bocha.go b/internal/application/service/web_search/bocha.go deleted file mode 100644 index 7f55b3752..000000000 --- a/internal/application/service/web_search/bocha.go +++ /dev/null @@ -1,38 +0,0 @@ -package service - -import ( - "context" - "fmt" - - "github.com/Tencent/WeKnora/internal/config" - "github.com/Tencent/WeKnora/internal/types" -) - -// BochaProvider implements web search using Bocha API -type BochaProvider struct { - apiKey string - apiURL string -} - -// NewBochaProvider creates a new Bocha provider -func NewBochaProvider(cfg config.WebSearchProviderConfig) (WebSearchProvider, error) { - return &BochaProvider{ - apiURL: cfg.APIURL, - }, nil -} - -// Name returns the provider name -func (p *BochaProvider) Name() string { - return "bocha" -} - -// Search performs a web search using Bocha API -func (p *BochaProvider) Search(ctx context.Context, query string, maxResults int, includeDate bool) ([]*types.WebSearchResult, error) { - // TODO: Implement Bocha search API - return nil, fmt.Errorf("bocha search provider is not yet implemented") -} - -// SetAPIKey sets the API key for the provider -func (p *BochaProvider) SetAPIKey(apiKey string) { - p.apiKey = apiKey -} diff --git a/internal/application/service/web_search/duckduckgo.go b/internal/application/service/web_search/duckduckgo.go new file mode 100644 index 000000000..6c97b6443 --- /dev/null +++ b/internal/application/service/web_search/duckduckgo.go @@ -0,0 +1,230 @@ +package web_search + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "strings" + "time" + + "github.com/PuerkitoBio/goquery" + "github.com/Tencent/WeKnora/internal/config" + "github.com/Tencent/WeKnora/internal/logger" + "github.com/Tencent/WeKnora/internal/types" + "github.com/Tencent/WeKnora/internal/types/interfaces" +) + +// DuckDuckGoProvider implements web search using DuckDuckGo (HTML first, API fallback) +type DuckDuckGoProvider struct { + client *http.Client +} + +// NewDuckDuckGoProvider creates a new DuckDuckGo provider +func NewDuckDuckGoProvider(_ config.WebSearchProviderConfig) (interfaces.WebSearchProvider, error) { + return &DuckDuckGoProvider{ + client: &http.Client{ + Timeout: 30 * time.Second, + }, + }, nil +} + +// Name returns the provider name +func (p *DuckDuckGoProvider) Name() string { + return "duckduckgo" +} + +// Search performs a web search using DuckDuckGo HTML endpoint with API fallback +func (p *DuckDuckGoProvider) Search(ctx context.Context, query string, maxResults int, includeDate bool) ([]*types.WebSearchResult, error) { + if maxResults <= 0 { + maxResults = 5 + } + // Try HTML scraping first (more reliable for general results) + htmlResults, err := p.searchHTML(ctx, query, maxResults) + if err == nil && len(htmlResults) > 0 { + return htmlResults, nil + } + // Fallback to Instant Answer API + apiResults, apiErr := p.searchAPI(ctx, query, maxResults) + if apiErr == nil && len(apiResults) > 0 { + return apiResults, nil + } + if err != nil { + return nil, fmt.Errorf("duckduckgo HTML search failed: %w", err) + } + return nil, fmt.Errorf("duckduckgo API search failed: %w", apiErr) +} + +func (p *DuckDuckGoProvider) searchHTML(ctx context.Context, query string, maxResults int) ([]*types.WebSearchResult, error) { + baseURL := "https://html.duckduckgo.com/html/" + params := url.Values{} + params.Set("q", query) + // Prefer Chinese results if applicable; otherwise DDG will auto-detect + params.Set("kl", "cn-zh") + + reqURL := baseURL + "?" + params.Encode() + req, err := http.NewRequestWithContext(ctx, "GET", reqURL, nil) + if err != nil { + return nil, fmt.Errorf("failed to create request: %w", err) + } + // Use a realistic UA to avoid blocks + req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36") + + resp, err := p.client.Do(req) + if err != nil { + return nil, fmt.Errorf("failed to perform request: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusAccepted { + return nil, fmt.Errorf("duckduckgo HTML returned status %d", resp.StatusCode) + } + + doc, err := goquery.NewDocumentFromReader(resp.Body) + if err != nil { + return nil, fmt.Errorf("failed to parse HTML: %w", err) + } + + results := make([]*types.WebSearchResult, 0, maxResults) + // Structure based on DDG HTML page + doc.Find(".web-result").Each(func(i int, s *goquery.Selection) { + if len(results) >= maxResults { + return + } + titleNode := s.Find(".result__a") + title := strings.TrimSpace(titleNode.Text()) + var link string + if href, exists := titleNode.Attr("href"); exists { + link = cleanDDGURL(href) + } + snippet := strings.TrimSpace(s.Find(".result__snippet").Text()) + if title != "" && link != "" { + results = append(results, &types.WebSearchResult{ + Title: title, + URL: link, + Snippet: snippet, + Source: "duckduckgo", + }) + } + }) + + logger.Infof(ctx, "DuckDuckGo HTML search returned %d results for query: %s", len(results), query) + return results, nil +} + +func (p *DuckDuckGoProvider) searchAPI(ctx context.Context, query string, maxResults int) ([]*types.WebSearchResult, error) { + baseURL := "https://api.duckduckgo.com/" + params := url.Values{} + params.Set("q", query) + params.Set("format", "json") + params.Set("no_html", "1") + params.Set("skip_disambig", "1") + + reqURL := baseURL + "?" + params.Encode() + req, err := http.NewRequestWithContext(ctx, "GET", reqURL, nil) + if err != nil { + return nil, fmt.Errorf("failed to create request: %w", err) + } + req.Header.Set("User-Agent", "WeKnora/1.0") + + resp, err := p.client.Do(req) + if err != nil { + return nil, fmt.Errorf("failed to perform request: %w", err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(resp.Body) + return nil, fmt.Errorf("duckduckgo API returned status %d: %s", resp.StatusCode, string(body)) + } + + var apiResponse struct { + AbstractText string `json:"AbstractText"` + AbstractURL string `json:"AbstractURL"` + Heading string `json:"Heading"` + RelatedTopics []struct { + FirstURL string `json:"FirstURL"` + Text string `json:"Text"` + } `json:"RelatedTopics"` + Results []struct { + FirstURL string `json:"FirstURL"` + Text string `json:"Text"` + } `json:"Results"` + } + if err := json.NewDecoder(resp.Body).Decode(&apiResponse); err != nil { + return nil, fmt.Errorf("failed to decode API response: %w", err) + } + + results := make([]*types.WebSearchResult, 0, maxResults) + if apiResponse.AbstractText != "" && apiResponse.AbstractURL != "" { + results = append(results, &types.WebSearchResult{ + Title: apiResponse.Heading, + URL: apiResponse.AbstractURL, + Snippet: apiResponse.AbstractText, + Source: "duckduckgo", + }) + } + for _, topic := range apiResponse.RelatedTopics { + if len(results) >= maxResults { + break + } + if topic.Text != "" && topic.FirstURL != "" { + results = append(results, &types.WebSearchResult{ + Title: extractTitle(topic.Text), + URL: topic.FirstURL, + Snippet: topic.Text, + Source: "duckduckgo", + }) + } + } + for _, r := range apiResponse.Results { + if len(results) >= maxResults { + break + } + if r.Text != "" && r.FirstURL != "" { + results = append(results, &types.WebSearchResult{ + Title: extractTitle(r.Text), + URL: r.FirstURL, + Snippet: r.Text, + Source: "duckduckgo", + }) + } + } + + logger.Infof(ctx, "DuckDuckGo API search returned %d results for query: %s", len(results), query) + return results, nil +} + +func cleanDDGURL(urlStr string) string { + if strings.HasPrefix(urlStr, "//duckduckgo.com/l/?uddg=") { + trimmed := strings.TrimPrefix(urlStr, "//duckduckgo.com/l/?uddg=") + if idx := strings.Index(trimmed, "&rut="); idx != -1 { + decodedStr, err := url.PathUnescape(trimmed[:idx]) + if err == nil { + return decodedStr + } + return "" + } + } + if strings.HasPrefix(urlStr, "https://duckduckgo.com/l/?uddg=") { + if parsedURL, err := url.Parse(urlStr); err == nil { + if uddg := parsedURL.Query().Get("uddg"); uddg != "" { + return uddg + } + } + } + return urlStr +} + +func extractTitle(text string) string { + lines := strings.Split(text, "\n") + if len(lines) > 0 { + title := strings.TrimSpace(lines[0]) + if len(title) > 100 { + title = title[:100] + "..." + } + return title + } + return strings.TrimSpace(text) +} diff --git a/internal/application/service/web_search/duckduckgo_test.go b/internal/application/service/web_search/duckduckgo_test.go new file mode 100644 index 000000000..9a79342b9 --- /dev/null +++ b/internal/application/service/web_search/duckduckgo_test.go @@ -0,0 +1,154 @@ +package web_search + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "net/url" + "strings" + "testing" + "time" + + "github.com/Tencent/WeKnora/internal/config" +) + +// testRoundTripper rewrites outgoing requests that target DuckDuckGo hosts +// to the provided test server, preserving path and query. +type testRoundTripper struct { + base *url.URL + next http.RoundTripper +} + +func (t *testRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) { + // Only rewrite requests to duckduckgo hosts used by the provider + if req.URL.Host == "html.duckduckgo.com" || req.URL.Host == "api.duckduckgo.com" { + cloned := *req + u := *req.URL + u.Scheme = t.base.Scheme + u.Host = t.base.Host + // Keep original path; our test server handlers should register for the same paths. + cloned.URL = &u + req = &cloned + } + return t.next.RoundTrip(req) +} + +func newTestClient(ts *httptest.Server) *http.Client { + baseURL, _ := url.Parse(ts.URL) + return &http.Client{ + Timeout: 5 * time.Second, + Transport: &testRoundTripper{ + base: baseURL, + next: http.DefaultTransport, + }, + } +} + +func TestDuckDuckGoProvider_Name(t *testing.T) { + p, _ := NewDuckDuckGoProvider(config.WebSearchProviderConfig{}) + if p.Name() != "duckduckgo" { + t.Fatalf("expected provider name duckduckgo, got %s", p.Name()) + } +} + +func TestDuckDuckGoProvider_Search_HTMLSuccess(t *testing.T) { + // Minimal HTML page with two results, matching selectors used in searchHTML + html := ` + + +
+ Example One +
Snippet one
+
+
+ Example Two +
Snippet two
+
+ +` + + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // Provider requests GET https://html.duckduckgo.com/html/?q=...&kl=... + if r.URL.Path == "/html/" { + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte(html)) + return + } + t.Fatalf("unexpected request path: %s", r.URL.Path) + })) + defer ts.Close() + + // Build provider and inject our test client + prov, _ := NewDuckDuckGoProvider(config.WebSearchProviderConfig{}) + dp := prov.(*DuckDuckGoProvider) + dp.client = newTestClient(ts) + + ctx := context.Background() + results, err := dp.Search(ctx, "weknora", 5, false) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(results) != 2 { + t.Fatalf("expected 2 results, got %d", len(results)) + } + if results[0].Title != "Example One" || !strings.HasPrefix(results[0].URL, "https://example.com/") || results[0].Snippet != "Snippet one" { + t.Fatalf("unexpected first result: %+v", results[0]) + } + if results[1].Title != "Example Two" || !strings.HasPrefix(results[1].URL, "https://example.org/") || results[1].Snippet != "Snippet two" { + t.Fatalf("unexpected second result: %+v", results[1]) + } +} + +func TestDuckDuckGoProvider_Search_APIFallback(t *testing.T) { + // Simulate HTML returning non-OK to force API fallback, then a minimal API JSON + apiResp := struct { + AbstractText string `json:"AbstractText"` + AbstractURL string `json:"AbstractURL"` + Heading string `json:"Heading"` + Results []struct { + FirstURL string `json:"FirstURL"` + Text string `json:"Text"` + } `json:"Results"` + }{ + AbstractText: "Abstract snippet", + AbstractURL: "https://example.com/abstract", + Heading: "Abstract Heading", + Results: []struct { + FirstURL string `json:"FirstURL"` + Text string `json:"Text"` + }{ + {FirstURL: "https://example.net/x", Text: "Title X - Detail X"}, + }, + } + + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/html/": + // Force fallback by returning 500 + w.WriteHeader(http.StatusInternalServerError) + default: + // API endpoint path "/" + w.Header().Set("Content-Type", "application/json") + enc := json.NewEncoder(w) + _ = enc.Encode(apiResp) + } + })) + defer ts.Close() + + prov, _ := NewDuckDuckGoProvider(config.WebSearchProviderConfig{}) + dp := prov.(*DuckDuckGoProvider) + dp.client = newTestClient(ts) + + ctx := context.Background() + results, err := dp.Search(ctx, "weknora", 3, false) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(results) == 0 { + t.Fatalf("expected some results from API fallback") + } + if results[0].URL != "https://example.com/abstract" || results[0].Title != "Abstract Heading" { + t.Fatalf("unexpected first API result: %+v", results[0]) + } +} diff --git a/internal/application/service/web_search/exa.go b/internal/application/service/web_search/exa.go deleted file mode 100644 index f15df0379..000000000 --- a/internal/application/service/web_search/exa.go +++ /dev/null @@ -1,38 +0,0 @@ -package service - -import ( - "context" - "fmt" - - "github.com/Tencent/WeKnora/internal/config" - "github.com/Tencent/WeKnora/internal/types" -) - -// ExaProvider implements web search using Exa API -type ExaProvider struct { - apiKey string - apiURL string -} - -// NewExaProvider creates a new Exa provider -func NewExaProvider(cfg config.WebSearchProviderConfig) (WebSearchProvider, error) { - return &ExaProvider{ - apiURL: cfg.APIURL, - }, nil -} - -// Name returns the provider name -func (p *ExaProvider) Name() string { - return "exa" -} - -// Search performs a web search using Exa API -func (p *ExaProvider) Search(ctx context.Context, query string, maxResults int, includeDate bool) ([]*types.WebSearchResult, error) { - // TODO: Implement Exa search API - return nil, fmt.Errorf("exa search provider is not yet implemented") -} - -// SetAPIKey sets the API key for the provider -func (p *ExaProvider) SetAPIKey(apiKey string) { - p.apiKey = apiKey -} diff --git a/internal/application/service/web_search/google.go b/internal/application/service/web_search/google.go deleted file mode 100644 index 3689cb92d..000000000 --- a/internal/application/service/web_search/google.go +++ /dev/null @@ -1,40 +0,0 @@ -package service - -import ( - "context" - "fmt" - - "github.com/Tencent/WeKnora/internal/config" - "github.com/Tencent/WeKnora/internal/types" -) - -// GoogleProvider implements web search using Google Custom Search API -type GoogleProvider struct { - apiKey string - searchEngineID string - apiURL string -} - -// NewGoogleProvider creates a new Google provider -func NewGoogleProvider(cfg config.WebSearchProviderConfig) (WebSearchProvider, error) { - return &GoogleProvider{ - apiURL: cfg.APIURL, - }, nil -} - -// Name returns the provider name -func (p *GoogleProvider) Name() string { - return "google" -} - -// Search performs a web search using Google Custom Search API -func (p *GoogleProvider) Search(ctx context.Context, query string, maxResults int, includeDate bool) ([]*types.WebSearchResult, error) { - // TODO: Implement Google Custom Search API - return nil, fmt.Errorf("google search provider is not yet implemented") -} - -// SetAPIKey sets the API key and search engine ID -func (p *GoogleProvider) SetAPIKey(apiKey, searchEngineID string) { - p.apiKey = apiKey - p.searchEngineID = searchEngineID -} diff --git a/internal/application/service/web_search/kuaisou.go b/internal/application/service/web_search/kuaisou.go deleted file mode 100644 index 4a0fed9ad..000000000 --- a/internal/application/service/web_search/kuaisou.go +++ /dev/null @@ -1,112 +0,0 @@ -package service - -import ( - "bytes" - "context" - "encoding/json" - "fmt" - "io" - "net/http" - "time" - - "github.com/Tencent/WeKnora/internal/config" - "github.com/Tencent/WeKnora/internal/logger" - "github.com/Tencent/WeKnora/internal/types" -) - -// KuaisouProvider implements web search using Kuaisou API -type KuaisouProvider struct { - apiKey string - apiURL string - client *http.Client -} - -// NewKuaisouProvider creates a new Kuaisou provider -func NewKuaisouProvider(cfg config.WebSearchProviderConfig) (WebSearchProvider, error) { - if cfg.APIURL == "" { - return nil, fmt.Errorf("kuaisou API URL is required") - } - - return &KuaisouProvider{ - apiKey: "", // Will be set from tenant config - apiURL: cfg.APIURL, - client: &http.Client{ - Timeout: 10 * time.Second, - }, - }, nil -} - -// Name returns the provider name -func (p *KuaisouProvider) Name() string { - return "kuaisou" -} - -// Search performs a web search using Kuaisou API -func (p *KuaisouProvider) Search(ctx context.Context, query string, maxResults int, includeDate bool) ([]*types.WebSearchResult, error) { - if p.apiKey == "" { - return nil, fmt.Errorf("kuaisou API key is required") - } - - // Prepare request - reqBody := map[string]interface{}{ - "query": query, - "max_results": maxResults, - } - - jsonData, err := json.Marshal(reqBody) - if err != nil { - return nil, fmt.Errorf("failed to marshal request: %w", err) - } - - req, err := http.NewRequestWithContext(ctx, "POST", p.apiURL, bytes.NewBuffer(jsonData)) - if err != nil { - return nil, fmt.Errorf("failed to create request: %w", err) - } - - req.Header.Set("Content-Type", "application/json") - req.Header.Set("Authorization", "Bearer "+p.apiKey) - - // Send request - resp, err := p.client.Do(req) - if err != nil { - return nil, fmt.Errorf("failed to send request: %w", err) - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - body, _ := io.ReadAll(resp.Body) - return nil, fmt.Errorf("kuaisou API returned status %d: %s", resp.StatusCode, string(body)) - } - - // Parse response - var response struct { - Results []struct { - Title string `json:"title"` - URL string `json:"url"` - Snippet string `json:"snippet"` - } `json:"results"` - } - - if err := json.NewDecoder(resp.Body).Decode(&response); err != nil { - return nil, fmt.Errorf("failed to decode response: %w", err) - } - - // Convert to WebSearchResult - results := make([]*types.WebSearchResult, 0, len(response.Results)) - for _, item := range response.Results { - results = append(results, &types.WebSearchResult{ - Title: item.Title, - URL: item.URL, - Snippet: item.Snippet, - Source: "kuaisou", - }) - } - - logger.Infof(ctx, "Kuaisou search returned %d results for query: %s", len(results), query) - return results, nil -} - -// SetAPIKey sets the API key for the provider -func (p *KuaisouProvider) SetAPIKey(apiKey string) { - p.apiKey = apiKey -} diff --git a/internal/application/service/web_search/searxng.go b/internal/application/service/web_search/searxng.go deleted file mode 100644 index d2f5d2837..000000000 --- a/internal/application/service/web_search/searxng.go +++ /dev/null @@ -1,32 +0,0 @@ -package service - -import ( - "context" - "fmt" - - "github.com/Tencent/WeKnora/internal/config" - "github.com/Tencent/WeKnora/internal/types" -) - -// SearxngProvider implements web search using Searxng API -type SearxngProvider struct { - apiURL string -} - -// NewSearxngProvider creates a new Searxng provider -func NewSearxngProvider(cfg config.WebSearchProviderConfig) (WebSearchProvider, error) { - return &SearxngProvider{ - apiURL: cfg.APIURL, - }, nil -} - -// Name returns the provider name -func (p *SearxngProvider) Name() string { - return "searxng" -} - -// Search performs a web search using Searxng API -func (p *SearxngProvider) Search(ctx context.Context, query string, maxResults int, includeDate bool) ([]*types.WebSearchResult, error) { - // TODO: Implement Searxng search API - return nil, fmt.Errorf("searxng search provider is not yet implemented") -} diff --git a/internal/application/service/web_search/tavily.go b/internal/application/service/web_search/tavily.go deleted file mode 100644 index 700c74a26..000000000 --- a/internal/application/service/web_search/tavily.go +++ /dev/null @@ -1,38 +0,0 @@ -package service - -import ( - "context" - "fmt" - - "github.com/Tencent/WeKnora/internal/config" - "github.com/Tencent/WeKnora/internal/types" -) - -// TavilyProvider implements web search using Tavily API -type TavilyProvider struct { - apiKey string - apiURL string -} - -// NewTavilyProvider creates a new Tavily provider -func NewTavilyProvider(cfg config.WebSearchProviderConfig) (WebSearchProvider, error) { - return &TavilyProvider{ - apiURL: cfg.APIURL, - }, nil -} - -// Name returns the provider name -func (p *TavilyProvider) Name() string { - return "tavily" -} - -// Search performs a web search using Tavily API -func (p *TavilyProvider) Search(ctx context.Context, query string, maxResults int, includeDate bool) ([]*types.WebSearchResult, error) { - // TODO: Implement Tavily search API - return nil, fmt.Errorf("tavily search provider is not yet implemented") -} - -// SetAPIKey sets the API key for the provider -func (p *TavilyProvider) SetAPIKey(apiKey string) { - p.apiKey = apiKey -} diff --git a/internal/application/service/web_search/zhipu.go b/internal/application/service/web_search/zhipu.go deleted file mode 100644 index 91b488a3d..000000000 --- a/internal/application/service/web_search/zhipu.go +++ /dev/null @@ -1,38 +0,0 @@ -package service - -import ( - "context" - "fmt" - - "github.com/Tencent/WeKnora/internal/config" - "github.com/Tencent/WeKnora/internal/types" -) - -// ZhipuProvider implements web search using Zhipu API -type ZhipuProvider struct { - apiKey string - apiURL string -} - -// NewZhipuProvider creates a new Zhipu provider -func NewZhipuProvider(cfg config.WebSearchProviderConfig) (WebSearchProvider, error) { - return &ZhipuProvider{ - apiURL: cfg.APIURL, - }, nil -} - -// Name returns the provider name -func (p *ZhipuProvider) Name() string { - return "zhipu" -} - -// Search performs a web search using Zhipu API -func (p *ZhipuProvider) Search(ctx context.Context, query string, maxResults int, includeDate bool) ([]*types.WebSearchResult, error) { - // TODO: Implement Zhipu search API - return nil, fmt.Errorf("zhipu search provider is not yet implemented") -} - -// SetAPIKey sets the API key for the provider -func (p *ZhipuProvider) SetAPIKey(apiKey string) { - p.apiKey = apiKey -} diff --git a/internal/application/service/web_search_baidu.go b/internal/application/service/web_search_baidu.go deleted file mode 100644 index 433454573..000000000 --- a/internal/application/service/web_search_baidu.go +++ /dev/null @@ -1,33 +0,0 @@ -package service - -import ( - "context" - "fmt" - - "github.com/Tencent/WeKnora/internal/config" - "github.com/Tencent/WeKnora/internal/types" -) - -// BaiduProvider implements web search using Baidu API -type BaiduProvider struct { - apiURL string -} - -// NewBaiduProvider creates a new Baidu provider -func NewBaiduProvider(cfg config.WebSearchProviderConfig) (WebSearchProvider, error) { - return &BaiduProvider{ - apiURL: cfg.APIURL, - }, nil -} - -// Name returns the provider name -func (p *BaiduProvider) Name() string { - return "baidu" -} - -// Search performs a web search using Baidu API -func (p *BaiduProvider) Search(ctx context.Context, query string, maxResults int, includeDate bool) ([]*types.WebSearchResult, error) { - // TODO: Implement Baidu search API - return nil, fmt.Errorf("baidu search provider is not yet implemented") -} - diff --git a/internal/application/service/web_search_bing.go b/internal/application/service/web_search_bing.go deleted file mode 100644 index 510d443e9..000000000 --- a/internal/application/service/web_search_bing.go +++ /dev/null @@ -1,33 +0,0 @@ -package service - -import ( - "context" - "fmt" - - "github.com/Tencent/WeKnora/internal/config" - "github.com/Tencent/WeKnora/internal/types" -) - -// BingProvider implements web search using Bing Search API -type BingProvider struct { - apiURL string -} - -// NewBingProvider creates a new Bing provider -func NewBingProvider(cfg config.WebSearchProviderConfig) (WebSearchProvider, error) { - return &BingProvider{ - apiURL: cfg.APIURL, - }, nil -} - -// Name returns the provider name -func (p *BingProvider) Name() string { - return "bing" -} - -// Search performs a web search using Bing Search API -func (p *BingProvider) Search(ctx context.Context, query string, maxResults int, includeDate bool) ([]*types.WebSearchResult, error) { - // TODO: Implement Bing Search API - return nil, fmt.Errorf("bing search provider is not yet implemented") -} - diff --git a/internal/application/service/web_search_bocha.go b/internal/application/service/web_search_bocha.go deleted file mode 100644 index 6b5b08ace..000000000 --- a/internal/application/service/web_search_bocha.go +++ /dev/null @@ -1,39 +0,0 @@ -package service - -import ( - "context" - "fmt" - - "github.com/Tencent/WeKnora/internal/config" - "github.com/Tencent/WeKnora/internal/types" -) - -// BochaProvider implements web search using Bocha API -type BochaProvider struct { - apiKey string - apiURL string -} - -// NewBochaProvider creates a new Bocha provider -func NewBochaProvider(cfg config.WebSearchProviderConfig) (WebSearchProvider, error) { - return &BochaProvider{ - apiURL: cfg.APIURL, - }, nil -} - -// Name returns the provider name -func (p *BochaProvider) Name() string { - return "bocha" -} - -// Search performs a web search using Bocha API -func (p *BochaProvider) Search(ctx context.Context, query string, maxResults int, includeDate bool) ([]*types.WebSearchResult, error) { - // TODO: Implement Bocha search API - return nil, fmt.Errorf("bocha search provider is not yet implemented") -} - -// SetAPIKey sets the API key for the provider -func (p *BochaProvider) SetAPIKey(apiKey string) { - p.apiKey = apiKey -} - diff --git a/internal/application/service/web_search_exa.go b/internal/application/service/web_search_exa.go deleted file mode 100644 index cc9c7d5a6..000000000 --- a/internal/application/service/web_search_exa.go +++ /dev/null @@ -1,39 +0,0 @@ -package service - -import ( - "context" - "fmt" - - "github.com/Tencent/WeKnora/internal/config" - "github.com/Tencent/WeKnora/internal/types" -) - -// ExaProvider implements web search using Exa API -type ExaProvider struct { - apiKey string - apiURL string -} - -// NewExaProvider creates a new Exa provider -func NewExaProvider(cfg config.WebSearchProviderConfig) (WebSearchProvider, error) { - return &ExaProvider{ - apiURL: cfg.APIURL, - }, nil -} - -// Name returns the provider name -func (p *ExaProvider) Name() string { - return "exa" -} - -// Search performs a web search using Exa API -func (p *ExaProvider) Search(ctx context.Context, query string, maxResults int, includeDate bool) ([]*types.WebSearchResult, error) { - // TODO: Implement Exa search API - return nil, fmt.Errorf("exa search provider is not yet implemented") -} - -// SetAPIKey sets the API key for the provider -func (p *ExaProvider) SetAPIKey(apiKey string) { - p.apiKey = apiKey -} - diff --git a/internal/application/service/web_search_google.go b/internal/application/service/web_search_google.go deleted file mode 100644 index 844b48ac5..000000000 --- a/internal/application/service/web_search_google.go +++ /dev/null @@ -1,41 +0,0 @@ -package service - -import ( - "context" - "fmt" - - "github.com/Tencent/WeKnora/internal/config" - "github.com/Tencent/WeKnora/internal/types" -) - -// GoogleProvider implements web search using Google Custom Search API -type GoogleProvider struct { - apiKey string - searchEngineID string - apiURL string -} - -// NewGoogleProvider creates a new Google provider -func NewGoogleProvider(cfg config.WebSearchProviderConfig) (WebSearchProvider, error) { - return &GoogleProvider{ - apiURL: cfg.APIURL, - }, nil -} - -// Name returns the provider name -func (p *GoogleProvider) Name() string { - return "google" -} - -// Search performs a web search using Google Custom Search API -func (p *GoogleProvider) Search(ctx context.Context, query string, maxResults int, includeDate bool) ([]*types.WebSearchResult, error) { - // TODO: Implement Google Custom Search API - return nil, fmt.Errorf("google search provider is not yet implemented") -} - -// SetAPIKey sets the API key and search engine ID -func (p *GoogleProvider) SetAPIKey(apiKey, searchEngineID string) { - p.apiKey = apiKey - p.searchEngineID = searchEngineID -} - diff --git a/internal/application/service/web_search_kuaisou.go b/internal/application/service/web_search_kuaisou.go deleted file mode 100644 index 91ffeb58c..000000000 --- a/internal/application/service/web_search_kuaisou.go +++ /dev/null @@ -1,113 +0,0 @@ -package service - -import ( - "bytes" - "context" - "encoding/json" - "fmt" - "io" - "net/http" - "time" - - "github.com/Tencent/WeKnora/internal/config" - "github.com/Tencent/WeKnora/internal/logger" - "github.com/Tencent/WeKnora/internal/types" -) - -// KuaisouProvider implements web search using Kuaisou API -type KuaisouProvider struct { - apiKey string - apiURL string - client *http.Client -} - -// NewKuaisouProvider creates a new Kuaisou provider -func NewKuaisouProvider(cfg config.WebSearchProviderConfig) (WebSearchProvider, error) { - if cfg.APIURL == "" { - return nil, fmt.Errorf("kuaisou API URL is required") - } - - return &KuaisouProvider{ - apiKey: "", // Will be set from tenant config - apiURL: cfg.APIURL, - client: &http.Client{ - Timeout: 10 * time.Second, - }, - }, nil -} - -// Name returns the provider name -func (p *KuaisouProvider) Name() string { - return "kuaisou" -} - -// Search performs a web search using Kuaisou API -func (p *KuaisouProvider) Search(ctx context.Context, query string, maxResults int, includeDate bool) ([]*types.WebSearchResult, error) { - if p.apiKey == "" { - return nil, fmt.Errorf("kuaisou API key is required") - } - - // Prepare request - reqBody := map[string]interface{}{ - "query": query, - "max_results": maxResults, - } - - jsonData, err := json.Marshal(reqBody) - if err != nil { - return nil, fmt.Errorf("failed to marshal request: %w", err) - } - - req, err := http.NewRequestWithContext(ctx, "POST", p.apiURL, bytes.NewBuffer(jsonData)) - if err != nil { - return nil, fmt.Errorf("failed to create request: %w", err) - } - - req.Header.Set("Content-Type", "application/json") - req.Header.Set("Authorization", "Bearer "+p.apiKey) - - // Send request - resp, err := p.client.Do(req) - if err != nil { - return nil, fmt.Errorf("failed to send request: %w", err) - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - body, _ := io.ReadAll(resp.Body) - return nil, fmt.Errorf("kuaisou API returned status %d: %s", resp.StatusCode, string(body)) - } - - // Parse response - var response struct { - Results []struct { - Title string `json:"title"` - URL string `json:"url"` - Snippet string `json:"snippet"` - } `json:"results"` - } - - if err := json.NewDecoder(resp.Body).Decode(&response); err != nil { - return nil, fmt.Errorf("failed to decode response: %w", err) - } - - // Convert to WebSearchResult - results := make([]*types.WebSearchResult, 0, len(response.Results)) - for _, item := range response.Results { - results = append(results, &types.WebSearchResult{ - Title: item.Title, - URL: item.URL, - Snippet: item.Snippet, - Source: "kuaisou", - }) - } - - logger.Infof(ctx, "Kuaisou search returned %d results for query: %s", len(results), query) - return results, nil -} - -// SetAPIKey sets the API key for the provider -func (p *KuaisouProvider) SetAPIKey(apiKey string) { - p.apiKey = apiKey -} - diff --git a/internal/application/service/web_search_searxng.go b/internal/application/service/web_search_searxng.go deleted file mode 100644 index bdd820e1b..000000000 --- a/internal/application/service/web_search_searxng.go +++ /dev/null @@ -1,33 +0,0 @@ -package service - -import ( - "context" - "fmt" - - "github.com/Tencent/WeKnora/internal/config" - "github.com/Tencent/WeKnora/internal/types" -) - -// SearxngProvider implements web search using Searxng API -type SearxngProvider struct { - apiURL string -} - -// NewSearxngProvider creates a new Searxng provider -func NewSearxngProvider(cfg config.WebSearchProviderConfig) (WebSearchProvider, error) { - return &SearxngProvider{ - apiURL: cfg.APIURL, - }, nil -} - -// Name returns the provider name -func (p *SearxngProvider) Name() string { - return "searxng" -} - -// Search performs a web search using Searxng API -func (p *SearxngProvider) Search(ctx context.Context, query string, maxResults int, includeDate bool) ([]*types.WebSearchResult, error) { - // TODO: Implement Searxng search API - return nil, fmt.Errorf("searxng search provider is not yet implemented") -} - diff --git a/internal/application/service/web_search_tavily.go b/internal/application/service/web_search_tavily.go deleted file mode 100644 index 365e5fed7..000000000 --- a/internal/application/service/web_search_tavily.go +++ /dev/null @@ -1,39 +0,0 @@ -package service - -import ( - "context" - "fmt" - - "github.com/Tencent/WeKnora/internal/config" - "github.com/Tencent/WeKnora/internal/types" -) - -// TavilyProvider implements web search using Tavily API -type TavilyProvider struct { - apiKey string - apiURL string -} - -// NewTavilyProvider creates a new Tavily provider -func NewTavilyProvider(cfg config.WebSearchProviderConfig) (WebSearchProvider, error) { - return &TavilyProvider{ - apiURL: cfg.APIURL, - }, nil -} - -// Name returns the provider name -func (p *TavilyProvider) Name() string { - return "tavily" -} - -// Search performs a web search using Tavily API -func (p *TavilyProvider) Search(ctx context.Context, query string, maxResults int, includeDate bool) ([]*types.WebSearchResult, error) { - // TODO: Implement Tavily search API - return nil, fmt.Errorf("tavily search provider is not yet implemented") -} - -// SetAPIKey sets the API key for the provider -func (p *TavilyProvider) SetAPIKey(apiKey string) { - p.apiKey = apiKey -} - diff --git a/internal/application/service/web_search_zhipu.go b/internal/application/service/web_search_zhipu.go deleted file mode 100644 index 208fe7d68..000000000 --- a/internal/application/service/web_search_zhipu.go +++ /dev/null @@ -1,39 +0,0 @@ -package service - -import ( - "context" - "fmt" - - "github.com/Tencent/WeKnora/internal/config" - "github.com/Tencent/WeKnora/internal/types" -) - -// ZhipuProvider implements web search using Zhipu API -type ZhipuProvider struct { - apiKey string - apiURL string -} - -// NewZhipuProvider creates a new Zhipu provider -func NewZhipuProvider(cfg config.WebSearchProviderConfig) (WebSearchProvider, error) { - return &ZhipuProvider{ - apiURL: cfg.APIURL, - }, nil -} - -// Name returns the provider name -func (p *ZhipuProvider) Name() string { - return "zhipu" -} - -// Search performs a web search using Zhipu API -func (p *ZhipuProvider) Search(ctx context.Context, query string, maxResults int, includeDate bool) ([]*types.WebSearchResult, error) { - // TODO: Implement Zhipu search API - return nil, fmt.Errorf("zhipu search provider is not yet implemented") -} - -// SetAPIKey sets the API key for the provider -func (p *ZhipuProvider) SetAPIKey(apiKey string) { - p.apiKey = apiKey -} - diff --git a/internal/container/container.go b/internal/container/container.go index 8a28bad10..efb6101d4 100644 --- a/internal/container/container.go +++ b/internal/container/container.go @@ -128,6 +128,11 @@ func BuildContainer(container *dig.Container) *dig.Container { // Chat pipeline components for processing chat requests must(container.Provide(chatpipline.NewEventManager)) + // Ensure Async task components are registered before invoking plugins that depend on KnowledgeService + // KnowledgeService depends on *asynq.Client, and plugins (like PluginSearch) are invoked below. + must(container.Provide(router.NewAsyncqClient)) + must(container.Provide(router.NewAsynqServer)) + must(container.Invoke(router.RunAsynqServer)) must(container.Invoke(chatpipline.NewPluginTracing)) must(container.Invoke(chatpipline.NewPluginSearch)) must(container.Invoke(chatpipline.NewPluginRerank)) @@ -159,9 +164,6 @@ func BuildContainer(container *dig.Container) *dig.Container { // Router configuration must(container.Provide(router.NewRouter)) - must(container.Provide(router.NewAsyncqClient)) - must(container.Provide(router.NewAsynqServer)) - must(container.Invoke(router.RunAsynqServer)) return container } diff --git a/internal/handler/session/qa.go b/internal/handler/session/qa.go index 1a7e3e24a..ca25167ff 100644 --- a/internal/handler/session/qa.go +++ b/internal/handler/session/qa.go @@ -203,11 +203,16 @@ func (h *Handler) AgentQA(c *gin.Context) { } // Check if agent mode has changed - currentAgentEnabled := session.AgentConfig.Enabled + currentAgentEnabled := session.AgentConfig.AgentModeEnabled if request.AgentEnabled != currentAgentEnabled { logger.Infof(ctx, "Agent mode changed from %v to %v", currentAgentEnabled, request.AgentEnabled) configChanged = true } + currentWebSearchEnabled := session.AgentConfig.AgentModeEnabled + if request.WebSearchEnabled != currentWebSearchEnabled { + logger.Infof(ctx, "Web search mode changed from %v to %v", currentWebSearchEnabled, request.WebSearchEnabled) + configChanged = true + } // If configuration changed, clear context and update session if configChanged { @@ -219,8 +224,12 @@ func (h *Handler) AgentQA(c *gin.Context) { // Continue anyway - this is not a fatal error } } + if knowledgeBasesChanged { + // todo clear temp kb + } session.AgentConfig.KnowledgeBases = request.KnowledgeBaseIDs - session.AgentConfig.Enabled = request.AgentEnabled + session.AgentConfig.AgentModeEnabled = request.AgentEnabled + session.AgentConfig.WebSearchEnabled = request.WebSearchEnabled // Persist the session changes if err := h.sessionService.UpdateSession(ctx, session); err != nil { logger.Errorf(ctx, "Failed to update session %s: %v", sessionID, err) diff --git a/internal/router/task.go b/internal/router/task.go index 547c4f1d9..55dee374f 100644 --- a/internal/router/task.go +++ b/internal/router/task.go @@ -29,10 +29,14 @@ func getAsynqRedisClientOpt() *asynq.RedisClientOpt { return opt } -func NewAsyncqClient() *asynq.Client { +func NewAsyncqClient() (*asynq.Client, error) { opt := getAsynqRedisClientOpt() client := asynq.NewClient(opt) - return client + err := client.Ping() + if err != nil { + return nil, err + } + return client, nil } func NewAsynqServer() *asynq.Server { diff --git a/internal/types/agent.go b/internal/types/agent.go index 74e4651e8..993833f61 100644 --- a/internal/types/agent.go +++ b/internal/types/agent.go @@ -24,8 +24,9 @@ type AgentConfig struct { // SessionAgentConfig represents session-level agent configuration // Sessions only store Enabled and KnowledgeBases; other configs are read from Tenant at runtime type SessionAgentConfig struct { - Enabled bool `json:"enabled"` // Whether agent mode is enabled for this session - KnowledgeBases []string `json:"knowledge_bases"` // Accessible knowledge base IDs for this session + AgentModeEnabled bool `json:"agent_mode_enabled"` // Whether agent mode is enabled for this session + WebSearchEnabled bool `json:"web_search_enabled"` // Whether web search is enabled for this session + KnowledgeBases []string `json:"knowledge_bases"` // Accessible knowledge base IDs for this session } // Value implements driver.Valuer interface for AgentConfig diff --git a/internal/types/interfaces/knowledge.go b/internal/types/interfaces/knowledge.go index db798e69c..c1eeac5d8 100644 --- a/internal/types/interfaces/knowledge.go +++ b/internal/types/interfaces/knowledge.go @@ -22,6 +22,8 @@ type KnowledgeService interface { CreateKnowledgeFromURL(ctx context.Context, kbID string, url string, enableMultimodel *bool) (*types.Knowledge, error) // CreateKnowledgeFromPassage creates knowledge from text passages. CreateKnowledgeFromPassage(ctx context.Context, kbID string, passage []string) (*types.Knowledge, error) + // CreateKnowledgeFromPassageSync creates knowledge from text passages and waits until chunks are indexed. + CreateKnowledgeFromPassageSync(ctx context.Context, kbID string, passage []string) (*types.Knowledge, error) // GetKnowledgeByID retrieves knowledge by ID. GetKnowledgeByID(ctx context.Context, id string) (*types.Knowledge, error) // GetKnowledgeBatch retrieves a batch of knowledge by IDs. diff --git a/internal/types/interfaces/web_search.go b/internal/types/interfaces/web_search.go index 6e5ecd706..a7814a7a0 100644 --- a/internal/types/interfaces/web_search.go +++ b/internal/types/interfaces/web_search.go @@ -6,7 +6,17 @@ import ( "github.com/Tencent/WeKnora/internal/types" ) -// WebSearchService defines the interface for web search service +// WebSearchProvider defines the interface for web search providers +type WebSearchProvider interface { + Search(ctx context.Context, query string, maxResults int, includeDate bool) ([]*types.WebSearchResult, error) + Name() string +} + type WebSearchService interface { Search(ctx context.Context, config *types.WebSearchConfig, query string) ([]*types.WebSearchResult, error) + CompressWithRAG(ctx context.Context, sessionID string, tempKBID string, questions []string, + webSearchResults []*types.WebSearchResult, cfg *types.WebSearchConfig, + kbSvc KnowledgeBaseService, knowSvc KnowledgeService, + seenURLs map[string]bool, knowledgeIDs []string, + ) (compressed []*types.WebSearchResult, kbID string, newSeen map[string]bool, newIDs []string, err error) } diff --git a/internal/types/knowledgebase.go b/internal/types/knowledgebase.go index c2a42ec2d..894626bfd 100644 --- a/internal/types/knowledgebase.go +++ b/internal/types/knowledgebase.go @@ -18,6 +18,8 @@ type KnowledgeBase struct { ID string `yaml:"id" json:"id" gorm:"type:varchar(36);primaryKey"` // Name of the knowledge base Name string `yaml:"name" json:"name"` + // Whether this knowledge base is temporary (ephemeral) and should be hidden from UI + IsTemporary bool `yaml:"is_temporary" json:"is_temporary" gorm:"default:false"` // Description of the knowledge base Description string `yaml:"description" json:"description"` // Tenant ID diff --git a/internal/types/web_search.go b/internal/types/web_search.go index d002cf74e..47c1986fa 100644 --- a/internal/types/web_search.go +++ b/internal/types/web_search.go @@ -44,7 +44,7 @@ type WebSearchResult struct { URL string `json:"url"` // 结果URL Snippet string `json:"snippet"` // 摘要片段 Content string `json:"content"` // 完整内容（可选，需要额外抓取） - Source string `json:"source"` // 来源（如：kuaisou, baidu等） + Source string `json:"source"` // 来源（如：duckduckgo等） PublishedAt *time.Time `json:"published_at,omitempty"` // 发布时间（如果有） } diff --git a/migrations/mysql/07_add_is_temporary.sql b/migrations/mysql/07_add_is_temporary.sql new file mode 100644 index 000000000..9d5ad7ef6 --- /dev/null +++ b/migrations/mysql/07_add_is_temporary.sql @@ -0,0 +1,5 @@ +-- Add is_temporary flag to knowledge_bases to support ephemeral KBs +ALTER TABLE knowledge_bases + ADD COLUMN IF NOT EXISTS is_temporary TINYINT(1) NOT NULL DEFAULT 0 COMMENT 'Temporary/hidden KB'; + + diff --git a/migrations/paradedb/07_add_is_temporary.sql b/migrations/paradedb/07_add_is_temporary.sql new file mode 100644 index 000000000..0fff62cac --- /dev/null +++ b/migrations/paradedb/07_add_is_temporary.sql @@ -0,0 +1,5 @@ +-- Add is_temporary flag to knowledge_bases to support ephemeral KBs +ALTER TABLE knowledge_bases + ADD COLUMN IF NOT EXISTS is_temporary BOOLEAN NOT NULL DEFAULT FALSE; + +