fix(chatd): sanitize \u0000 from JSON before JSONB insertion (#22637)

## Problem

Users hit this error when agent tool results contain Unicode null
characters:

```
persist step: insert tool result: pq: unsupported Unicode escape sequence
```

PostgreSQL's `jsonb` type rejects `\u0000` (Unicode null, U+0000) with
that error, even though it's valid JSON per RFC 8259. Tool results from
agents can contain this sequence — e.g. binary data, C-style strings, or
certain API responses.

## Root cause

`MarshalToolResult` and `MarshalContent` in `chatprompt.go` serialize
content blocks to JSON and pass them directly to `InsertChatMessage`
which casts to `::jsonb`. Go's `json.Marshal` / `json.Valid` accept
`\u0000`, but Postgres does not.

## Fix

Added `sanitizeJSONForPG()` which strips `\u0000` escape sequences from
serialized JSON before insertion. Uses `bytes.Contains` as a fast-path
check to avoid allocation when no null bytes are present (the common
case).

Applied to both `MarshalContent` (assistant messages) and
`MarshalToolResult` (tool result messages).
This commit is contained in:
Kyle Carberry
2026-03-04 21:14:41 -05:00
committed by GitHub
parent d50fc374c5
commit 27f0f2962c
2 changed files with 44 additions and 0 deletions
+19
View File
@@ -1,6 +1,7 @@
package chatprompt
import (
"bytes"
"encoding/json"
"regexp"
"strings"
@@ -422,6 +423,7 @@ func MarshalContent(blocks []fantasy.Content) (pqtype.NullRawMessage, error) {
if err != nil {
return pqtype.NullRawMessage{}, xerrors.Errorf("encode content blocks: %w", err)
}
data = sanitizeJSONForPG(data)
return pqtype.NullRawMessage{RawMessage: data, Valid: true}, nil
}
@@ -439,6 +441,7 @@ func MarshalToolResult(toolCallID, toolName string, result json.RawMessage, isEr
if err != nil {
return pqtype.NullRawMessage{}, xerrors.Errorf("encode tool result: %w", err)
}
data = sanitizeJSONForPG(data)
return pqtype.NullRawMessage{RawMessage: data, Valid: true}, nil
}
@@ -835,6 +838,22 @@ func sanitizeToolCallID(id string) string {
return toolCallIDSanitizer.ReplaceAllString(id, "_")
}
// jsonNullEscape is the JSON escape sequence for Unicode null (U+0000).
var jsonNullEscape = []byte(`\u0000`)
// sanitizeJSONForPG strips \u0000 escape sequences from JSON data.
// PostgreSQL's jsonb type rejects the Unicode null character (U+0000)
// with "unsupported Unicode escape sequence", even though \u0000 is
// valid JSON per RFC 8259. Tool results from agents may contain this
// sequence (e.g. binary data or C-style strings), so we strip it
// before insertion.
func sanitizeJSONForPG(data []byte) []byte {
if bytes.Contains(data, jsonNullEscape) {
data = bytes.ReplaceAll(data, jsonNullEscape, nil)
}
return data
}
func marshalContentBlock(block fantasy.Content) (json.RawMessage, error) {
encoded, err := json.Marshal(block)
if err != nil {
@@ -1,6 +1,7 @@
package chatprompt_test
import (
"bytes"
"encoding/json"
"testing"
@@ -89,3 +90,27 @@ func TestConvertMessages_NormalizesAssistantToolCallInput(t *testing.T) {
})
}
}
func TestMarshalToolResult_SanitizesNullBytes(t *testing.T) {
t.Parallel()
result := json.RawMessage(`{"output":"hello\u0000world"}`)
got, err := chatprompt.MarshalToolResult("call_1", "my_tool", result, false)
require.NoError(t, err)
require.True(t, got.Valid)
require.False(t, bytes.Contains(got.RawMessage, []byte(`\u0000`)),
"output should not contain \\u0000 escape sequences")
}
func TestMarshalContent_SanitizesNullBytes(t *testing.T) {
t.Parallel()
blocks := []fantasy.Content{
fantasy.TextContent{Text: "before\u0000after"},
}
got, err := chatprompt.MarshalContent(blocks)
require.NoError(t, err)
require.True(t, got.Valid)
require.False(t, bytes.Contains(got.RawMessage, []byte(`\u0000`)),
"output should not contain \\u0000 escape sequences")
}