mirror of
https://github.com/Kilo-Org/kilocode.git
synced 2026-10-25 09:43:20 +08:00
fix: use StringDecoder for UTF-8 multi-byte stream decoding
Replace chunk.toString() with StringDecoder in stream processing to prevent garbled characters (arrows, tree-drawing chars appearing as mojibake) when multi-byte UTF-8 sequences are split across chunk boundaries. StringDecoder maintains state across chunks to correctly reassemble incomplete characters. Fixed in bash tool, prompt shell execution, and ripgrep output.
This commit is contained in:
@@ -1,5 +1,6 @@
|
||||
// Ripgrep utility functions
|
||||
import path from "path"
|
||||
import { StringDecoder } from "string_decoder" // kilocode_change - fix UTF-8 multi-byte split
|
||||
import { Global } from "../global"
|
||||
import fs from "fs/promises"
|
||||
import z from "zod"
|
||||
@@ -252,11 +253,14 @@ export namespace Ripgrep {
|
||||
}
|
||||
|
||||
let buffer = ""
|
||||
// kilocode_change start - use StringDecoder to handle multi-byte UTF-8 characters split across chunks
|
||||
const decoder = new StringDecoder("utf8")
|
||||
const stream = proc.stdout as AsyncIterable<Buffer | string>
|
||||
for await (const chunk of stream) {
|
||||
input.signal?.throwIfAborted()
|
||||
|
||||
buffer += typeof chunk === "string" ? chunk : chunk.toString()
|
||||
buffer += typeof chunk === "string" ? chunk : decoder.write(chunk as Buffer)
|
||||
// kilocode_change end
|
||||
// Handle both Unix (\n) and Windows (\r\n) line endings
|
||||
const lines = buffer.split(/\r?\n/)
|
||||
buffer = lines.pop() || ""
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import path from "path"
|
||||
import os from "os"
|
||||
import fs from "fs/promises"
|
||||
import { StringDecoder } from "string_decoder" // kilocode_change - fix UTF-8 multi-byte split
|
||||
import z from "zod"
|
||||
import { Filesystem } from "../util/filesystem"
|
||||
import { Identifier } from "../id/id"
|
||||
@@ -1698,9 +1699,11 @@ NOTE: At any point in time through this workflow you should feel free to ask the
|
||||
})
|
||||
|
||||
let output = ""
|
||||
// kilocode_change start - use StringDecoder to handle multi-byte UTF-8 characters split across chunks
|
||||
const decoder = new StringDecoder("utf8")
|
||||
|
||||
proc.stdout?.on("data", (chunk) => {
|
||||
output += chunk.toString()
|
||||
output += decoder.write(chunk)
|
||||
if (part.state.status === "running") {
|
||||
part.state.metadata = {
|
||||
output: output,
|
||||
@@ -1711,7 +1714,7 @@ NOTE: At any point in time through this workflow you should feel free to ask the
|
||||
})
|
||||
|
||||
proc.stderr?.on("data", (chunk) => {
|
||||
output += chunk.toString()
|
||||
output += decoder.write(chunk)
|
||||
if (part.state.status === "running") {
|
||||
part.state.metadata = {
|
||||
output: output,
|
||||
@@ -1720,6 +1723,7 @@ NOTE: At any point in time through this workflow you should feel free to ask the
|
||||
Session.updatePart(part)
|
||||
}
|
||||
})
|
||||
// kilocode_change end
|
||||
|
||||
let aborted = false
|
||||
let exited = false
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import z from "zod"
|
||||
import { spawn } from "child_process"
|
||||
import { StringDecoder } from "string_decoder" // kilocode_change - fix UTF-8 multi-byte split
|
||||
import { Tool } from "./tool"
|
||||
import path from "path"
|
||||
import DESCRIPTION from "./bash.txt"
|
||||
@@ -191,8 +192,10 @@ export const BashTool = Tool.define("bash", async () => {
|
||||
},
|
||||
})
|
||||
|
||||
// kilocode_change start - use StringDecoder to handle multi-byte UTF-8 characters split across chunks
|
||||
const decoder = new StringDecoder("utf8")
|
||||
const append = (chunk: Buffer) => {
|
||||
output += chunk.toString()
|
||||
output += decoder.write(chunk)
|
||||
ctx.metadata({
|
||||
metadata: {
|
||||
// truncate the metadata to avoid GIANT blobs of data (has nothing to do w/ what agent can access)
|
||||
@@ -201,6 +204,7 @@ export const BashTool = Tool.define("bash", async () => {
|
||||
},
|
||||
})
|
||||
}
|
||||
// kilocode_change end
|
||||
|
||||
proc.stdout?.on("data", append)
|
||||
proc.stderr?.on("data", append)
|
||||
|
||||
Reference in New Issue
Block a user