mirror of
https://github.com/coder/coder.git
synced 2026-09-24 15:04:27 +08:00
fix(chatd): sanitize \u0000 from JSON before JSONB insertion (#22637)
## Problem Users hit this error when agent tool results contain Unicode null characters: ``` persist step: insert tool result: pq: unsupported Unicode escape sequence ``` PostgreSQL's `jsonb` type rejects `\u0000` (Unicode null, U+0000) with that error, even though it's valid JSON per RFC 8259. Tool results from agents can contain this sequence — e.g. binary data, C-style strings, or certain API responses. ## Root cause `MarshalToolResult` and `MarshalContent` in `chatprompt.go` serialize content blocks to JSON and pass them directly to `InsertChatMessage` which casts to `::jsonb`. Go's `json.Marshal` / `json.Valid` accept `\u0000`, but Postgres does not. ## Fix Added `sanitizeJSONForPG()` which strips `\u0000` escape sequences from serialized JSON before insertion. Uses `bytes.Contains` as a fast-path check to avoid allocation when no null bytes are present (the common case). Applied to both `MarshalContent` (assistant messages) and `MarshalToolResult` (tool result messages).
This commit is contained in:
@@ -1,6 +1,7 @@
|
||||
package chatprompt
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"regexp"
|
||||
"strings"
|
||||
@@ -422,6 +423,7 @@ func MarshalContent(blocks []fantasy.Content) (pqtype.NullRawMessage, error) {
|
||||
if err != nil {
|
||||
return pqtype.NullRawMessage{}, xerrors.Errorf("encode content blocks: %w", err)
|
||||
}
|
||||
data = sanitizeJSONForPG(data)
|
||||
return pqtype.NullRawMessage{RawMessage: data, Valid: true}, nil
|
||||
}
|
||||
|
||||
@@ -439,6 +441,7 @@ func MarshalToolResult(toolCallID, toolName string, result json.RawMessage, isEr
|
||||
if err != nil {
|
||||
return pqtype.NullRawMessage{}, xerrors.Errorf("encode tool result: %w", err)
|
||||
}
|
||||
data = sanitizeJSONForPG(data)
|
||||
return pqtype.NullRawMessage{RawMessage: data, Valid: true}, nil
|
||||
}
|
||||
|
||||
@@ -835,6 +838,22 @@ func sanitizeToolCallID(id string) string {
|
||||
return toolCallIDSanitizer.ReplaceAllString(id, "_")
|
||||
}
|
||||
|
||||
// jsonNullEscape is the JSON escape sequence for Unicode null (U+0000).
|
||||
var jsonNullEscape = []byte(`\u0000`)
|
||||
|
||||
// sanitizeJSONForPG strips \u0000 escape sequences from JSON data.
|
||||
// PostgreSQL's jsonb type rejects the Unicode null character (U+0000)
|
||||
// with "unsupported Unicode escape sequence", even though \u0000 is
|
||||
// valid JSON per RFC 8259. Tool results from agents may contain this
|
||||
// sequence (e.g. binary data or C-style strings), so we strip it
|
||||
// before insertion.
|
||||
func sanitizeJSONForPG(data []byte) []byte {
|
||||
if bytes.Contains(data, jsonNullEscape) {
|
||||
data = bytes.ReplaceAll(data, jsonNullEscape, nil)
|
||||
}
|
||||
return data
|
||||
}
|
||||
|
||||
func marshalContentBlock(block fantasy.Content) (json.RawMessage, error) {
|
||||
encoded, err := json.Marshal(block)
|
||||
if err != nil {
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package chatprompt_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"testing"
|
||||
|
||||
@@ -89,3 +90,27 @@ func TestConvertMessages_NormalizesAssistantToolCallInput(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestMarshalToolResult_SanitizesNullBytes(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
result := json.RawMessage(`{"output":"hello\u0000world"}`)
|
||||
got, err := chatprompt.MarshalToolResult("call_1", "my_tool", result, false)
|
||||
require.NoError(t, err)
|
||||
require.True(t, got.Valid)
|
||||
require.False(t, bytes.Contains(got.RawMessage, []byte(`\u0000`)),
|
||||
"output should not contain \\u0000 escape sequences")
|
||||
}
|
||||
|
||||
func TestMarshalContent_SanitizesNullBytes(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
blocks := []fantasy.Content{
|
||||
fantasy.TextContent{Text: "before\u0000after"},
|
||||
}
|
||||
got, err := chatprompt.MarshalContent(blocks)
|
||||
require.NoError(t, err)
|
||||
require.True(t, got.Valid)
|
||||
require.False(t, bytes.Contains(got.RawMessage, []byte(`\u0000`)),
|
||||
"output should not contain \\u0000 escape sequences")
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user