fix: save media message type to db (#23427)

We had a bug where computer use base64-encoded screenshots would not be
interpreted as screenshots anymore once saved to the db, loaded back
into memory, and sent to Anthropic. Instead, they would be interpreted
as regular text. Once a computer use agent made enough screenshots and
stopped, and you tried sending it another message, you'd get an out of
context error:

<img width="808" height="367" alt="Screenshot 2026-03-23 at 12 02 54"
src="https://github.com/user-attachments/assets/f0bf6be2-4863-47ca-a7a9-9e6d9dfceeed"
/>

This PR fixes that.
This commit is contained in:
Hugo Dutka
2026-03-25 17:11:21 +00:00
committed by GitHub
parent d9fc5a5be1
commit 84740f4619
6 changed files with 497 additions and 34 deletions
+6 -1
View File
@@ -159,6 +159,7 @@ type ChatMessagePart struct {
Result json.RawMessage `json:"result,omitempty" variants:"tool-result?"`
ResultDelta string `json:"result_delta,omitempty"`
IsError bool `json:"is_error,omitempty" variants:"tool-result?"`
IsMedia bool `json:"is_media,omitempty" variants:"tool-result?"`
SourceID string `json:"source_id,omitempty" variants:"source?"`
URL string `json:"url" variants:"source"`
Title string `json:"title,omitempty" variants:"source?"`
@@ -241,13 +242,17 @@ func ChatMessageToolCall(toolCallID, toolName string, args json.RawMessage) Chat
}
// ChatMessageToolResult builds a tool-result chat message part.
func ChatMessageToolResult(toolCallID, toolName string, result json.RawMessage, isError bool) ChatMessagePart {
// The isMedia flag marks the result as carrying binary media content
// (e.g. a screenshot) so that round-trip reconstruction preserves
// the media type instead of sending raw base64 as text tokens.
func ChatMessageToolResult(toolCallID, toolName string, result json.RawMessage, isError bool, isMedia bool) ChatMessagePart {
return ChatMessagePart{
Type: ChatMessagePartTypeToolResult,
ToolCallID: toolCallID,
ToolName: toolName,
Result: result,
IsError: isError,
IsMedia: isMedia,
}
}