fix: use StringDecoder for UTF-8 multi-byte stream decoding

Replace chunk.toString() with StringDecoder in stream processing to
prevent garbled characters (arrows, tree-drawing chars appearing as
mojibake) when multi-byte UTF-8 sequences are split across chunk
boundaries. StringDecoder maintains state across chunks to correctly
reassemble incomplete characters.

Fixed in bash tool, prompt shell execution, and ripgrep output.
This commit is contained in:
Evan Jacobson
2026-03-10 11:34:40 -06:00
parent 75e46decc2
commit 2596acaf3e
3 changed files with 16 additions and 4 deletions
+5 -1
View File
@@ -1,5 +1,6 @@
// Ripgrep utility functions
import path from "path"
import { StringDecoder } from "string_decoder" // kilocode_change - fix UTF-8 multi-byte split
import { Global } from "../global"
import fs from "fs/promises"
import z from "zod"
@@ -252,11 +253,14 @@ export namespace Ripgrep {
}
let buffer = ""
// kilocode_change start - use StringDecoder to handle multi-byte UTF-8 characters split across chunks
const decoder = new StringDecoder("utf8")
const stream = proc.stdout as AsyncIterable<Buffer | string>
for await (const chunk of stream) {
input.signal?.throwIfAborted()
buffer += typeof chunk === "string" ? chunk : chunk.toString()
buffer += typeof chunk === "string" ? chunk : decoder.write(chunk as Buffer)
// kilocode_change end
// Handle both Unix (\n) and Windows (\r\n) line endings
const lines = buffer.split(/\r?\n/)
buffer = lines.pop() || ""
+6 -2
View File
@@ -1,6 +1,7 @@
import path from "path"
import os from "os"
import fs from "fs/promises"
import { StringDecoder } from "string_decoder" // kilocode_change - fix UTF-8 multi-byte split
import z from "zod"
import { Filesystem } from "../util/filesystem"
import { Identifier } from "../id/id"
@@ -1698,9 +1699,11 @@ NOTE: At any point in time through this workflow you should feel free to ask the
})
let output = ""
// kilocode_change start - use StringDecoder to handle multi-byte UTF-8 characters split across chunks
const decoder = new StringDecoder("utf8")
proc.stdout?.on("data", (chunk) => {
output += chunk.toString()
output += decoder.write(chunk)
if (part.state.status === "running") {
part.state.metadata = {
output: output,
@@ -1711,7 +1714,7 @@ NOTE: At any point in time through this workflow you should feel free to ask the
})
proc.stderr?.on("data", (chunk) => {
output += chunk.toString()
output += decoder.write(chunk)
if (part.state.status === "running") {
part.state.metadata = {
output: output,
@@ -1720,6 +1723,7 @@ NOTE: At any point in time through this workflow you should feel free to ask the
Session.updatePart(part)
}
})
// kilocode_change end
let aborted = false
let exited = false
+5 -1
View File
@@ -1,5 +1,6 @@
import z from "zod"
import { spawn } from "child_process"
import { StringDecoder } from "string_decoder" // kilocode_change - fix UTF-8 multi-byte split
import { Tool } from "./tool"
import path from "path"
import DESCRIPTION from "./bash.txt"
@@ -191,8 +192,10 @@ export const BashTool = Tool.define("bash", async () => {
},
})
// kilocode_change start - use StringDecoder to handle multi-byte UTF-8 characters split across chunks
const decoder = new StringDecoder("utf8")
const append = (chunk: Buffer) => {
output += chunk.toString()
output += decoder.write(chunk)
ctx.metadata({
metadata: {
// truncate the metadata to avoid GIANT blobs of data (has nothing to do w/ what agent can access)
@@ -201,6 +204,7 @@ export const BashTool = Tool.define("bash", async () => {
},
})
}
// kilocode_change end
proc.stdout?.on("data", append)
proc.stderr?.on("data", append)