opensource/WeKnora
3
0
Fork 0
mirror of https://github.com/Tencent/WeKnora.git synced 2026-10-25 09:42:26 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix(security): redact IM list credentials and harden MinerU SSRF

Browse Source 
Stop returning bot credentials from Viewer-accessible IM channel list
endpoints and validate self-hosted MinerU URLs at config save and fetch time.
This commit is contained in:
wizardchen
2026-07-02 17:58:20 +08:00
committed by lyingbug
parent 929697618f
commit 7099a7e181
7 changed files with 190 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/handler/im.go
+2 -3
View File
@@ -131,13 +131,12 @@ func (h *IMHandler) ListIMChannels(c *gin.Context) {
return
}
c.JSON(http.StatusOK, gin.H{"data": channels})
c.JSON(http.StatusOK, gin.H{"data": im.SummarizeIMChannels(channels)})
}
// ListAllIMChannels lists every IM channel in the current tenant, across
// agents, for the cross-agent overview page. Credentials are intentionally
// NOT included in the response — callers that need credentials must use the
// per-agent endpoint (GET /agents/:id/im-channels).
// NOT included in the response.
func (h *IMHandler) ListAllIMChannels(c *gin.Context) {
tenantID, ok := c.Request.Context().Value(types.TenantIDContextKey).(uint64)
if !ok {
internal/handler/tenant.go
+22
View File
@@ -2,6 +2,7 @@ package handler
import (
"context"
"fmt"
"net/http"
"strconv"
"strings"
@@ -1168,6 +1169,10 @@ func (h *TenantHandler) updateTenantParserEngineConfigInternal(c *gin.Context) {
return
}
merged := types.MergeParserEngineConfigForUpdate(&cfg, tenant.ParserEngineConfig)
if err := validateParserEngineOutboundURLs(merged); err != nil {
c.Error(errors.NewValidationError(err.Error()))
return
}
tenant.ParserEngineConfig = merged
updatedTenant, err := h.service.UpdateTenant(ctx, tenant)
if err != nil {
@@ -1465,3 +1470,20 @@ func (h *TenantHandler) updateTenantRetrievalConfigInternal(c *gin.Context) {
"message": "Retrieval configuration updated successfully",
})
}
func validateParserEngineOutboundURLs(cfg *types.ParserEngineConfig) error {
if cfg == nil {
return nil
}
if endpoint := strings.TrimSpace(cfg.MinerUEndpoint); endpoint != "" {
if err := secutils.ValidateURLForSSRF(endpoint); err != nil {
return fmt.Errorf("mineru_endpoint failed SSRF validation: %v", err)
}
}
if vlmURL := strings.TrimSpace(cfg.MinerUVLMServerURL); vlmURL != "" {
if err := secutils.ValidateURLForSSRF(vlmURL); err != nil {
return fmt.Errorf("mineru_vlm_server_url failed SSRF validation: %v", err)
}
}
return nil
}
internal/handler/tenant_parser_ssrf_test.go
+25
View File
@@ -0,0 +1,25 @@
package handler
import (
"testing"
"github.com/Tencent/WeKnora/internal/types"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestValidateParserEngineOutboundURLs_RejectsPrivateMinerUEndpoint(t *testing.T) {
err := validateParserEngineOutboundURLs(&types.ParserEngineConfig{
MinerUEndpoint: "http://127.0.0.1:8080",
})
require.Error(t, err)
assert.Contains(t, err.Error(), "mineru_endpoint")
}
func TestValidateParserEngineOutboundURLs_RejectsPrivateVLMServerURL(t *testing.T) {
err := validateParserEngineOutboundURLs(&types.ParserEngineConfig{
MinerUVLMServerURL: "http://169.254.169.254",
})
require.Error(t, err)
assert.Contains(t, err.Error(), "mineru_vlm_server_url")
}
internal/im/types.go
+54
View File
@@ -35,6 +35,60 @@ func (IMChannel) TableName() string {
return "im_channels"
}
// IMChannelSummary is the HTTP-safe list shape for IM channels. Credentials
// are never included — use Admin+ Create/Update responses to read back
// values immediately after a mutation.
type IMChannelSummary struct {
ID string `json:"id"`
TenantID uint64 `json:"tenant_id"`
AgentID string `json:"agent_id"`
Platform string `json:"platform"`
Name string `json:"name"`
Enabled bool `json:"enabled"`
Mode string `json:"mode"`
OutputMode string `json:"output_mode"`
KnowledgeBaseID string `json:"knowledge_base_id"`
BotIdentity string `json:"bot_identity"`
SessionMode string `json:"session_mode"`
CredentialsConfigured bool `json:"credentials_configured"`
CreatedAt time.Time `json:"created_at"`
UpdatedAt time.Time `json:"updated_at"`
}
// SummarizeIMChannel converts a stored channel into its list response shape.
func SummarizeIMChannel(ch IMChannel) IMChannelSummary {
return IMChannelSummary{
ID: ch.ID,
TenantID: ch.TenantID,
AgentID: ch.AgentID,
Platform: ch.Platform,
Name: ch.Name,
Enabled: ch.Enabled,
Mode: ch.Mode,
OutputMode: ch.OutputMode,
KnowledgeBaseID: ch.KnowledgeBaseID,
BotIdentity: ch.BotIdentity,
SessionMode: ch.SessionMode,
CredentialsConfigured: imCredentialsConfigured(ch.Credentials),
CreatedAt: ch.CreatedAt,
UpdatedAt: ch.UpdatedAt,
}
}
// SummarizeIMChannels converts stored channels into list response shapes.
func SummarizeIMChannels(channels []IMChannel) []IMChannelSummary {
out := make([]IMChannelSummary, 0, len(channels))
for _, ch := range channels {
out = append(out, SummarizeIMChannel(ch))
}
return out
}
func imCredentialsConfigured(cred types.JSON) bool {
s := strings.TrimSpace(string(cred))
return s != "" && s != "{}"
}
func (ch *IMChannel) BeforeCreate(tx *gorm.DB) error {
if ch.ID == "" {
ch.ID = uuid.New().String()
internal/im/types_summary_test.go
+34
View File
@@ -0,0 +1,34 @@
package im
import (
"encoding/json"
"testing"
"github.com/Tencent/WeKnora/internal/types"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestSummarizeIMChannel_OmitsCredentials(t *testing.T) {
ch := IMChannel{
ID: "ch-1",
TenantID: 1,
AgentID: "agent-1",
Platform: "feishu",
Name: "support",
Credentials: types.JSON(`{"app_secret":"top-secret"}`),
}
summary := SummarizeIMChannel(ch)
body, err := json.Marshal(summary)
require.NoError(t, err)
assert.True(t, summary.CredentialsConfigured)
assert.NotContains(t, string(body), "top-secret")
assert.NotContains(t, string(body), `"credentials":`)
}
func TestSummarizeIMChannel_EmptyCredentialsNotConfigured(t *testing.T) {
summary := SummarizeIMChannel(IMChannel{Credentials: types.JSON("{}")})
assert.False(t, summary.CredentialsConfigured)
}
internal/infrastructure/docparser/mineru_converter.go
+33 -2
View File
@@ -18,6 +18,7 @@ import (
"github.com/Tencent/WeKnora/internal/logger"
"github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/internal/utils"
)
const mineruTimeout = 1000 * time.Second // large docs can take a while
@@ -53,6 +54,14 @@ func (c *MinerUReader) Read(ctx context.Context, req *types.ReadRequest) (*types
if c.endpoint == "" {
return &types.ReadResult{Error: "MinerU endpoint is not configured"}, nil
}
if err := validateMinerUOutboundURL(c.endpoint); err != nil {
return &types.ReadResult{Error: err.Error()}, nil
}
if c.vlmServerURL != "" {
if err := validateMinerUOutboundURL(c.vlmServerURL); err != nil {
return &types.ReadResult{Error: err.Error()}, nil
}
}
content := req.FileContent
if len(content) == 0 {
@@ -146,7 +155,10 @@ func (c *MinerUReader) callFileParse(ctx context.Context, content []byte) (strin
}
httpReq.Header.Set("Content-Type", writer.FormDataContentType())
client := &http.Client{Timeout: mineruTimeout}
client := utils.NewSSRFSafeHTTPClient(utils.SSRFSafeHTTPClientConfig{
Timeout: mineruTimeout,
MaxRedirects: 5,
})
resp, err := client.Do(httpReq)
if err != nil {
return "", nil, fmt.Errorf("HTTP request: %w", err)
@@ -258,13 +270,32 @@ func (c *MinerUReader) logMinerUResponseStructure(obj interface{}, prefix string
logResponseStructure("MinerU", obj, prefix)
}
// validateMinerUOutboundURL rejects MinerU endpoints that would reach private
// or otherwise restricted hosts when parsed or probed from the app server.
func validateMinerUOutboundURL(rawURL string) error {
rawURL = strings.TrimSpace(rawURL)
if rawURL == "" {
return nil
}
if err := utils.ValidateURLForSSRF(rawURL); err != nil {
return fmt.Errorf("MinerU URL blocked by SSRF check: %v", err)
}
return nil
}
// PingMinerU checks if the self-hosted MinerU service is reachable.
func PingMinerU(endpoint string) (bool, string) {
endpoint = strings.TrimRight(endpoint, "/")
if endpoint == "" {
return false, "未配置 MinerU 端点"
}
client := &http.Client{Timeout: 5 * time.Second}
if err := validateMinerUOutboundURL(endpoint); err != nil {
return false, err.Error()
}
client := utils.NewSSRFSafeHTTPClient(utils.SSRFSafeHTTPClientConfig{
Timeout: 5 * time.Second,
MaxRedirects: 5,
})
resp, err := client.Get(endpoint + "/docs")
if err != nil {
return false, fmt.Sprintf("MinerU 服务不可达: %v", err)
internal/infrastructure/docparser/mineru_converter_ssrf_test.go
+20
View File
@@ -0,0 +1,20 @@
package docparser
import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestValidateMinerUOutboundURL_RejectsLoopback(t *testing.T) {
err := validateMinerUOutboundURL("http://127.0.0.1:8080")
require.Error(t, err)
assert.Contains(t, err.Error(), "SSRF")
}
func TestPingMinerU_RejectsPrivateEndpoint(t *testing.T) {
ok, msg := PingMinerU("http://127.0.0.1:8080")
assert.False(t, ok)
assert.Contains(t, msg, "SSRF")
}