From 2913c44fa533a137d9328b482fba3f955c70d352 Mon Sep 17 00:00:00 2001 From: "kiloconnect[bot]" <240665456+kiloconnect[bot]@users.noreply.github.com> Date: Wed, 8 Apr 2026 11:43:50 +0000 Subject: [PATCH 01/24] feat(kilocode): add encoding-aware IO utilities and tests Introduce kilocode Encoding module and integrate into IO paths implement readEncoded and writeEncoded in filesystem for encoding-aware file IO propagate encoding through patching workflows (patch, apply_patch, edit, write) add tests for encoding utilities --- packages/opencode/src/kilocode/encoding.ts | 151 ++++++++++ packages/opencode/src/patch/index.ts | 25 +- packages/opencode/src/tool/apply_patch.ts | 26 +- packages/opencode/src/tool/edit.ts | 13 +- packages/opencode/src/tool/write.ts | 9 +- packages/opencode/src/util/filesystem.ts | 36 +++ .../opencode/test/kilocode/encoding.test.ts | 260 ++++++++++++++++++ 7 files changed, 503 insertions(+), 17 deletions(-) create mode 100644 packages/opencode/src/kilocode/encoding.ts create mode 100644 packages/opencode/test/kilocode/encoding.test.ts diff --git a/packages/opencode/src/kilocode/encoding.ts b/packages/opencode/src/kilocode/encoding.ts new file mode 100644 index 0000000000..c171dee7b6 --- /dev/null +++ b/packages/opencode/src/kilocode/encoding.ts @@ -0,0 +1,151 @@ +// kilocode_change - new file +import { readFileSync } from "fs" +import { readFile, writeFile, mkdir } from "fs/promises" +import { dirname } from "path" +import { existsSync } from "fs" + +/** + * Text encoding detection and preservation. + * Detects file encoding from raw bytes (BOM + heuristics) and provides + * round-trip read/write that preserves the original encoding. + */ +export namespace Encoding { + export interface Info { + encoding: "utf-8" | "utf-16le" | "utf-16be" | "latin1" + bom: boolean + } + + const UTF8_BOM = Buffer.from([0xef, 0xbb, 0xbf]) + const UTF16_LE_BOM = Buffer.from([0xff, 0xfe]) + const UTF16_BE_BOM = Buffer.from([0xfe, 0xff]) + + export const DEFAULT: Info = { encoding: "utf-8", bom: false } + + export function detect(bytes: Buffer): Info { + // BOM detection (most reliable signal) + if (bytes.length >= 3 && bytes[0] === 0xef && bytes[1] === 0xbb && bytes[2] === 0xbf) { + return { encoding: "utf-8", bom: true } + } + // Check UTF-16 BE before LE because FF FE could also be the start of UTF-32 LE, + // but FE FF is unambiguously UTF-16 BE. + if (bytes.length >= 2 && bytes[0] === 0xfe && bytes[1] === 0xff) { + return { encoding: "utf-16be", bom: true } + } + if (bytes.length >= 2 && bytes[0] === 0xff && bytes[1] === 0xfe) { + return { encoding: "utf-16le", bom: true } + } + + // Heuristic: detect BOM-less UTF-16 by looking for null-byte patterns + if (bytes.length >= 4) { + let le = 0 + let be = 0 + const sample = Math.min(bytes.length & ~1, 512) // even number of bytes + for (let i = 0; i < sample; i += 2) { + if (bytes[i] !== 0 && bytes[i + 1] === 0) le++ + if (bytes[i] === 0 && bytes[i + 1] !== 0) be++ + } + const pairs = sample / 2 + if (le > pairs / 4) return { encoding: "utf-16le", bom: false } + if (be > pairs / 4) return { encoding: "utf-16be", bom: false } + } + + // Check if valid UTF-8; if not, fall back to Latin-1 + if (!isUtf8(bytes)) { + return { encoding: "latin1", bom: false } + } + + return DEFAULT + } + + export function decode(bytes: Buffer, info: Info): string { + const start = info.bom ? (info.encoding === "utf-8" ? 3 : 2) : 0 + const data = start > 0 ? bytes.subarray(start) : bytes + + switch (info.encoding) { + case "utf-8": + return data.toString("utf-8") + case "utf-16le": + return data.toString("utf16le") + case "utf-16be": { + const swapped = Buffer.allocUnsafe(data.length) + for (let i = 0; i < data.length - 1; i += 2) { + swapped[i] = data[i + 1] + swapped[i + 1] = data[i] + } + return swapped.toString("utf16le") + } + case "latin1": + return data.toString("latin1") + } + } + + export function encode(text: string, info: Info): Buffer { + let body: Buffer + switch (info.encoding) { + case "utf-8": + body = Buffer.from(text, "utf-8") + break + case "utf-16le": + body = Buffer.from(text, "utf16le") + break + case "utf-16be": { + const le = Buffer.from(text, "utf16le") + body = Buffer.allocUnsafe(le.length) + for (let i = 0; i < le.length - 1; i += 2) { + body[i] = le[i + 1] + body[i + 1] = le[i] + } + break + } + case "latin1": + body = Buffer.from(text, "latin1") + break + } + + if (!info.bom) return body + + const bom = + info.encoding === "utf-8" + ? UTF8_BOM + : info.encoding === "utf-16le" + ? UTF16_LE_BOM + : info.encoding === "utf-16be" + ? UTF16_BE_BOM + : Buffer.alloc(0) + + return Buffer.concat([bom, body]) + } + + /** Read a file preserving its encoding info. */ + export async function read(path: string): Promise<{ text: string; info: Info }> { + const bytes = await readFile(path) + const info = detect(bytes) + return { text: decode(bytes, info), info } + } + + /** Read a file synchronously, preserving its encoding info. */ + export function readSync(path: string): { text: string; info: Info } { + const bytes = readFileSync(path) + const info = detect(bytes) + return { text: decode(bytes, info), info } + } + + /** Write text back to a file using the given encoding info. */ + export async function write(path: string, text: string, info: Info): Promise { + const bytes = encode(text, info) + const dir = dirname(path) + if (!existsSync(dir)) { + await mkdir(dir, { recursive: true }) + } + await writeFile(path, bytes) + } + + function isUtf8(bytes: Buffer): boolean { + try { + new TextDecoder("utf-8", { fatal: true }).decode(bytes) + return true + } catch { + return false + } + } +} diff --git a/packages/opencode/src/patch/index.ts b/packages/opencode/src/patch/index.ts index b87ad55528..7e2b0496f6 100644 --- a/packages/opencode/src/patch/index.ts +++ b/packages/opencode/src/patch/index.ts @@ -3,6 +3,7 @@ import * as path from "path" import * as fs from "fs/promises" import { readFileSync } from "fs" import { Log } from "../util/log" +import { Encoding } from "../kilocode/encoding" // kilocode_change export namespace Patch { const log = Log.create({ service: "patch" }) @@ -306,16 +307,21 @@ export namespace Patch { interface ApplyPatchFileUpdate { unified_diff: string content: string + encoding: Encoding.Info // kilocode_change } export function deriveNewContentsFromChunks(filePath: string, chunks: UpdateFileChunk[]): ApplyPatchFileUpdate { - // Read original file content + // kilocode_change start - encoding-aware read let originalContent: string + let enc: Encoding.Info try { - originalContent = readFileSync(filePath, "utf-8") + const result = Encoding.readSync(filePath) + originalContent = result.text + enc = result.info } catch (error) { throw new Error(`Failed to read file ${filePath}: ${error}`) } + // kilocode_change end let originalLines = originalContent.split("\n") @@ -340,6 +346,7 @@ export namespace Patch { return { unified_diff: unifiedDiff, content: newContent, + encoding: enc, // kilocode_change } } @@ -524,6 +531,7 @@ export namespace Patch { const modified: string[] = [] const deleted: string[] = [] + // kilocode_change start - encoding-aware writes for (const hunk of hunks) { switch (hunk.type) { case "add": @@ -533,7 +541,7 @@ export namespace Patch { await fs.mkdir(addDir, { recursive: true }) } - await fs.writeFile(hunk.path, hunk.contents, "utf-8") + await Encoding.write(hunk.path, hunk.contents, Encoding.DEFAULT) added.push(hunk.path) log.info(`Added file: ${hunk.path}`) break @@ -554,19 +562,20 @@ export namespace Patch { await fs.mkdir(moveDir, { recursive: true }) } - await fs.writeFile(hunk.move_path, fileUpdate.content, "utf-8") + await Encoding.write(hunk.move_path, fileUpdate.content, fileUpdate.encoding) await fs.unlink(hunk.path) modified.push(hunk.move_path) log.info(`Moved file: ${hunk.path} -> ${hunk.move_path}`) } else { // Regular update - await fs.writeFile(hunk.path, fileUpdate.content, "utf-8") + await Encoding.write(hunk.path, fileUpdate.content, fileUpdate.encoding) modified.push(hunk.path) log.info(`Updated file: ${hunk.path}`) } break } } + // kilocode_change end return { added, modified, deleted } } @@ -625,11 +634,13 @@ export namespace Patch { // For delete, we need to read the current content const deletePath = path.resolve(effectiveCwd, hunk.path) try { - const content = await fs.readFile(deletePath, "utf-8") + // kilocode_change start - encoding-aware read + const result = await Encoding.read(deletePath) changes.set(resolvedPath, { type: "delete", - content, + content: result.text, }) + // kilocode_change end } catch (error) { return { type: MaybeApplyPatchVerified.CorrectnessError, diff --git a/packages/opencode/src/tool/apply_patch.ts b/packages/opencode/src/tool/apply_patch.ts index 8e932a9ba2..7816908c7c 100644 --- a/packages/opencode/src/tool/apply_patch.ts +++ b/packages/opencode/src/tool/apply_patch.ts @@ -14,6 +14,7 @@ import { Filesystem } from "../util/filesystem" import DESCRIPTION from "./apply_patch.txt" import { File } from "../file" import { filterDiagnostics } from "./diagnostics" // kilocode_change +import { Encoding } from "../kilocode/encoding" // kilocode_change const PatchParams = z.object({ patchText: z.string().describe("The full patch text that describes all changes to be made"), @@ -45,6 +46,7 @@ export const ApplyPatchTool = Tool.define("apply_patch", { } // Validate file paths and check permissions + // kilocode_change start - preserve file encoding const fileChanges: Array<{ filePath: string oldContent: string @@ -54,7 +56,9 @@ export const ApplyPatchTool = Tool.define("apply_patch", { diff: string additions: number deletions: number + encoding: Encoding.Info }> = [] + // kilocode_change end let totalDiff = "" @@ -84,6 +88,7 @@ export const ApplyPatchTool = Tool.define("apply_patch", { diff, additions, deletions, + encoding: Encoding.DEFAULT, // kilocode_change - new files use UTF-8 }) totalDiff += diff + "\n" @@ -97,7 +102,11 @@ export const ApplyPatchTool = Tool.define("apply_patch", { throw new Error(`apply_patch verification failed: Failed to read file to update: ${filePath}`) } - const oldContent = await fs.readFile(filePath, "utf-8") + // kilocode_change start - encoding-aware read + const encoded = await Encoding.read(filePath) + const oldContent = encoded.text + const enc = encoded.info + // kilocode_change end let newContent = oldContent // Apply the update chunks to get new content @@ -129,6 +138,7 @@ export const ApplyPatchTool = Tool.define("apply_patch", { diff, additions, deletions, + encoding: enc, // kilocode_change }) totalDiff += diff + "\n" @@ -136,9 +146,12 @@ export const ApplyPatchTool = Tool.define("apply_patch", { } case "delete": { - const contentToDelete = await fs.readFile(filePath, "utf-8").catch((error) => { + // kilocode_change start - encoding-aware read for delete + const encoded = await Encoding.read(filePath).catch((error) => { throw new Error(`apply_patch verification failed: ${error}`) }) + const contentToDelete = encoded.text + // kilocode_change end const deleteDiff = trimDiff(createTwoFilesPatch(filePath, filePath, contentToDelete, "")) const deletions = contentToDelete.split("\n").length @@ -151,6 +164,7 @@ export const ApplyPatchTool = Tool.define("apply_patch", { diff: deleteDiff, additions: 0, deletions, + encoding: encoded.info, // kilocode_change }) totalDiff += deleteDiff + "\n" @@ -190,16 +204,17 @@ export const ApplyPatchTool = Tool.define("apply_patch", { for (const change of fileChanges) { const edited = change.type === "delete" ? undefined : (change.movePath ?? change.filePath) + // kilocode_change start - encoding-aware writes switch (change.type) { case "add": // Create parent directories (recursive: true is safe on existing/root dirs) await fs.mkdir(path.dirname(change.filePath), { recursive: true }) - await fs.writeFile(change.filePath, change.newContent, "utf-8") + await Encoding.write(change.filePath, change.newContent, change.encoding) updates.push({ file: change.filePath, event: "add" }) break case "update": - await fs.writeFile(change.filePath, change.newContent, "utf-8") + await Encoding.write(change.filePath, change.newContent, change.encoding) updates.push({ file: change.filePath, event: "change" }) break @@ -207,7 +222,7 @@ export const ApplyPatchTool = Tool.define("apply_patch", { if (change.movePath) { // Create parent directories (recursive: true is safe on existing/root dirs) await fs.mkdir(path.dirname(change.movePath), { recursive: true }) - await fs.writeFile(change.movePath, change.newContent, "utf-8") + await Encoding.write(change.movePath, change.newContent, change.encoding) await fs.unlink(change.filePath) updates.push({ file: change.filePath, event: "unlink" }) updates.push({ file: change.movePath, event: "add" }) @@ -219,6 +234,7 @@ export const ApplyPatchTool = Tool.define("apply_patch", { updates.push({ file: change.filePath, event: "unlink" }) break } + // kilocode_change end if (edited) { await Bus.publish(File.Event.Edited, { diff --git a/packages/opencode/src/tool/edit.ts b/packages/opencode/src/tool/edit.ts index 5b4468e3c2..39949e6194 100644 --- a/packages/opencode/src/tool/edit.ts +++ b/packages/opencode/src/tool/edit.ts @@ -18,6 +18,7 @@ import { Instance } from "../project/instance" import { Snapshot } from "@/snapshot" import { assertExternalDirectory } from "./external-directory" import { filterDiagnostics } from "./diagnostics" // kilocode_change +import { Encoding } from "../kilocode/encoding" // kilocode_change const MAX_DIAGNOSTICS_PER_FILE = 20 @@ -87,7 +88,11 @@ export const EditTool = Tool.define("edit", { if (!stats) throw new Error(`File ${filePath} not found`) if (stats.isDirectory()) throw new Error(`Path is a directory, not a file: ${filePath}`) await FileTime.assert(ctx.sessionID, filePath) - contentOld = await Filesystem.readText(filePath) + // kilocode_change start - preserve file encoding + const encoded = await Filesystem.readEncoded(filePath) + contentOld = encoded.text + const enc = encoded.encoding + // kilocode_change end const ending = detectLineEnding(contentOld) const old = convertToLineEnding(normalizeLineEndings(params.oldString), ending) @@ -108,7 +113,7 @@ export const EditTool = Tool.define("edit", { }, }) - await Filesystem.write(filePath, contentNew) + await Filesystem.writeEncoded(filePath, contentNew, enc) // kilocode_change await Bus.publish(File.Event.Edited, { file: filePath, }) @@ -116,7 +121,9 @@ export const EditTool = Tool.define("edit", { file: filePath, event: "change", }) - contentNew = await Filesystem.readText(filePath) + // kilocode_change start - re-read with encoding awareness + contentNew = (await Filesystem.readEncoded(filePath)).text + // kilocode_change end diff = trimDiff( createTwoFilesPatch(filePath, filePath, normalizeLineEndings(contentOld), normalizeLineEndings(contentNew)), ) diff --git a/packages/opencode/src/tool/write.ts b/packages/opencode/src/tool/write.ts index 9de745798f..167271fa80 100644 --- a/packages/opencode/src/tool/write.ts +++ b/packages/opencode/src/tool/write.ts @@ -13,6 +13,7 @@ import { Instance } from "../project/instance" import { trimDiff } from "./edit" import { assertExternalDirectory } from "./external-directory" import { filterDiagnostics } from "./diagnostics" // kilocode_change +import { Encoding } from "../kilocode/encoding" // kilocode_change const MAX_DIAGNOSTICS_PER_FILE = 20 const MAX_PROJECT_DIAGNOSTICS_FILES = 5 @@ -28,7 +29,11 @@ export const WriteTool = Tool.define("write", { await assertExternalDirectory(ctx, filepath) const exists = await Filesystem.exists(filepath) - const contentOld = exists ? await Filesystem.readText(filepath) : "" + // kilocode_change start - preserve file encoding + const encoded = exists ? await Filesystem.readEncoded(filepath) : undefined + const contentOld = encoded ? encoded.text : "" + const enc = encoded ? encoded.encoding : Encoding.DEFAULT + // kilocode_change end if (exists) await FileTime.assert(ctx.sessionID, filepath) const diff = trimDiff(createTwoFilesPatch(filepath, filepath, contentOld, params.content)) @@ -42,7 +47,7 @@ export const WriteTool = Tool.define("write", { }, }) - await Filesystem.write(filepath, params.content) + await Filesystem.writeEncoded(filepath, params.content, enc) // kilocode_change await Bus.publish(File.Event.Edited, { file: filepath, }) diff --git a/packages/opencode/src/util/filesystem.ts b/packages/opencode/src/util/filesystem.ts index fb1f5ab9e5..f4f714494c 100644 --- a/packages/opencode/src/util/filesystem.ts +++ b/packages/opencode/src/util/filesystem.ts @@ -6,6 +6,7 @@ import { dirname, join, relative, resolve as pathResolve } from "path" import { Readable } from "stream" import { pipeline } from "stream/promises" import { Glob } from "./glob" +import { Encoding } from "../kilocode/encoding" // kilocode_change export namespace Filesystem { // Fast sync version for metadata checks @@ -42,6 +43,41 @@ export namespace Filesystem { return readFile(p) } + // kilocode_change start - encoding-aware read/write for tool file operations + export async function readEncoded(p: string): Promise<{ text: string; encoding: Encoding.Info }> { + const bytes = await readFile(p) + const info = Encoding.detect(bytes) + return { text: Encoding.decode(bytes, info), encoding: info } + } + + export async function writeEncoded( + p: string, + content: string, + encoding: Encoding.Info, + mode?: number, + ): Promise { + const bytes = Encoding.encode(content, encoding) + try { + if (mode) { + await writeFile(p, bytes, { mode }) + } else { + await writeFile(p, bytes) + } + } catch (e) { + if (isEnoent(e)) { + await mkdir(dirname(p), { recursive: true }) + if (mode) { + await writeFile(p, bytes, { mode }) + } else { + await writeFile(p, bytes) + } + return + } + throw e + } + } + // kilocode_change end + export async function readArrayBuffer(p: string): Promise { const buf = await readFile(p) return buf.buffer.slice(buf.byteOffset, buf.byteOffset + buf.byteLength) as ArrayBuffer diff --git a/packages/opencode/test/kilocode/encoding.test.ts b/packages/opencode/test/kilocode/encoding.test.ts new file mode 100644 index 0000000000..985da8eaa6 --- /dev/null +++ b/packages/opencode/test/kilocode/encoding.test.ts @@ -0,0 +1,260 @@ +import { test, expect, describe } from "bun:test" +import { Encoding } from "../../src/kilocode/encoding" +import { tmpdir } from "../fixture/fixture" +import path from "path" +import fs from "fs/promises" + +describe("Encoding", () => { + describe("detect", () => { + test("detects UTF-8 BOM", () => { + const bytes = Buffer.concat([Buffer.from([0xef, 0xbb, 0xbf]), Buffer.from("hello", "utf-8")]) + const info = Encoding.detect(bytes) + expect(info.encoding).toBe("utf-8") + expect(info.bom).toBe(true) + }) + + test("detects UTF-16 LE BOM", () => { + const bytes = Buffer.concat([Buffer.from([0xff, 0xfe]), Buffer.from("hello", "utf16le")]) + const info = Encoding.detect(bytes) + expect(info.encoding).toBe("utf-16le") + expect(info.bom).toBe(true) + }) + + test("detects UTF-16 BE BOM", () => { + const le = Buffer.from("hello", "utf16le") + const be = Buffer.allocUnsafe(le.length) + for (let i = 0; i < le.length - 1; i += 2) { + be[i] = le[i + 1] + be[i + 1] = le[i] + } + const bytes = Buffer.concat([Buffer.from([0xfe, 0xff]), be]) + const info = Encoding.detect(bytes) + expect(info.encoding).toBe("utf-16be") + expect(info.bom).toBe(true) + }) + + test("detects plain UTF-8 (no BOM)", () => { + const bytes = Buffer.from("hello world", "utf-8") + const info = Encoding.detect(bytes) + expect(info.encoding).toBe("utf-8") + expect(info.bom).toBe(false) + }) + + test("detects Latin-1 for invalid UTF-8 bytes", () => { + // 0xe9 alone is invalid UTF-8 (incomplete sequence) + const bytes = Buffer.from([0x68, 0x65, 0x6c, 0x6c, 0x6f, 0x20, 0xe9]) + const info = Encoding.detect(bytes) + expect(info.encoding).toBe("latin1") + expect(info.bom).toBe(false) + }) + + test("detects BOM-less UTF-16 LE from null-byte pattern", () => { + // "AB" in UTF-16LE is: 0x41 0x00 0x42 0x00 + const bytes = Buffer.from("ABCDEFGHIJKLMNOP", "utf16le") + const info = Encoding.detect(bytes) + expect(info.encoding).toBe("utf-16le") + expect(info.bom).toBe(false) + }) + + test("returns utf-8 for empty buffer", () => { + const info = Encoding.detect(Buffer.alloc(0)) + expect(info.encoding).toBe("utf-8") + expect(info.bom).toBe(false) + }) + }) + + describe("decode", () => { + test("decodes UTF-8 with BOM", () => { + const bytes = Buffer.concat([Buffer.from([0xef, 0xbb, 0xbf]), Buffer.from("hello", "utf-8")]) + const text = Encoding.decode(bytes, { encoding: "utf-8", bom: true }) + expect(text).toBe("hello") + }) + + test("decodes UTF-8 without BOM", () => { + const bytes = Buffer.from("hello", "utf-8") + const text = Encoding.decode(bytes, { encoding: "utf-8", bom: false }) + expect(text).toBe("hello") + }) + + test("decodes UTF-16 LE with BOM", () => { + const bytes = Buffer.concat([Buffer.from([0xff, 0xfe]), Buffer.from("hello", "utf16le")]) + const text = Encoding.decode(bytes, { encoding: "utf-16le", bom: true }) + expect(text).toBe("hello") + }) + + test("decodes UTF-16 BE with BOM", () => { + const le = Buffer.from("hello", "utf16le") + const be = Buffer.allocUnsafe(le.length) + for (let i = 0; i < le.length - 1; i += 2) { + be[i] = le[i + 1] + be[i + 1] = le[i] + } + const bytes = Buffer.concat([Buffer.from([0xfe, 0xff]), be]) + const text = Encoding.decode(bytes, { encoding: "utf-16be", bom: true }) + expect(text).toBe("hello") + }) + + test("decodes Latin-1", () => { + // "café" in Latin-1: c=0x63, a=0x61, f=0x66, é=0xe9 + const bytes = Buffer.from([0x63, 0x61, 0x66, 0xe9]) + const text = Encoding.decode(bytes, { encoding: "latin1", bom: false }) + expect(text).toBe("caf\u00e9") + }) + }) + + describe("encode", () => { + test("encodes UTF-8 without BOM", () => { + const bytes = Encoding.encode("hello", { encoding: "utf-8", bom: false }) + expect(bytes).toEqual(Buffer.from("hello", "utf-8")) + }) + + test("encodes UTF-8 with BOM", () => { + const bytes = Encoding.encode("hello", { encoding: "utf-8", bom: true }) + expect(bytes[0]).toBe(0xef) + expect(bytes[1]).toBe(0xbb) + expect(bytes[2]).toBe(0xbf) + expect(bytes.subarray(3).toString("utf-8")).toBe("hello") + }) + + test("encodes UTF-16 LE with BOM", () => { + const bytes = Encoding.encode("hi", { encoding: "utf-16le", bom: true }) + expect(bytes[0]).toBe(0xff) + expect(bytes[1]).toBe(0xfe) + expect(bytes.subarray(2).toString("utf16le")).toBe("hi") + }) + + test("encodes UTF-16 BE with BOM", () => { + const bytes = Encoding.encode("A", { encoding: "utf-16be", bom: true }) + // BOM: FE FF + expect(bytes[0]).toBe(0xfe) + expect(bytes[1]).toBe(0xff) + // 'A' in UTF-16BE: 0x00 0x41 + expect(bytes[2]).toBe(0x00) + expect(bytes[3]).toBe(0x41) + }) + + test("encodes Latin-1", () => { + const bytes = Encoding.encode("caf\u00e9", { encoding: "latin1", bom: false }) + expect(bytes).toEqual(Buffer.from([0x63, 0x61, 0x66, 0xe9])) + }) + }) + + describe("round-trip", () => { + test("UTF-8 with BOM round-trips", () => { + const original = Buffer.concat([Buffer.from([0xef, 0xbb, 0xbf]), Buffer.from("hello world\n", "utf-8")]) + const info = Encoding.detect(original) + const text = Encoding.decode(original, info) + const result = Encoding.encode(text, info) + expect(result).toEqual(original) + }) + + test("UTF-16 LE with BOM round-trips", () => { + const original = Buffer.concat([Buffer.from([0xff, 0xfe]), Buffer.from("hello world\n", "utf16le")]) + const info = Encoding.detect(original) + const text = Encoding.decode(original, info) + const result = Encoding.encode(text, info) + expect(result).toEqual(original) + }) + + test("plain UTF-8 round-trips", () => { + const original = Buffer.from("hello world\n", "utf-8") + const info = Encoding.detect(original) + const text = Encoding.decode(original, info) + const result = Encoding.encode(text, info) + expect(result).toEqual(original) + }) + + test("Latin-1 round-trips", () => { + const original = Buffer.from([0x63, 0x61, 0x66, 0xe9, 0x0a]) // "café\n" in latin1 + const info = Encoding.detect(original) + expect(info.encoding).toBe("latin1") + const text = Encoding.decode(original, info) + const result = Encoding.encode(text, info) + expect(result).toEqual(original) + }) + }) + + describe("file read/write", () => { + test("preserves UTF-8 BOM through file write/read cycle", async () => { + await using tmp = await tmpdir() + const file = path.join(tmp.path, "bom.txt") + const original = Buffer.concat([Buffer.from([0xef, 0xbb, 0xbf]), Buffer.from("line one\nline two\n", "utf-8")]) + await fs.writeFile(file, original) + + const { text, info } = await Encoding.read(file) + expect(info.encoding).toBe("utf-8") + expect(info.bom).toBe(true) + expect(text).toBe("line one\nline two\n") + + // Modify and write back + const modified = text.replace("one", "1") + await Encoding.write(file, modified, info) + + // Verify BOM is preserved + const raw = await fs.readFile(file) + expect(raw[0]).toBe(0xef) + expect(raw[1]).toBe(0xbb) + expect(raw[2]).toBe(0xbf) + expect(raw.subarray(3).toString("utf-8")).toBe("line 1\nline two\n") + }) + + test("preserves UTF-16 LE BOM through file write/read cycle", async () => { + await using tmp = await tmpdir() + const file = path.join(tmp.path, "utf16le.txt") + const content = "hello world\n" + const original = Buffer.concat([Buffer.from([0xff, 0xfe]), Buffer.from(content, "utf16le")]) + await fs.writeFile(file, original) + + const { text, info } = await Encoding.read(file) + expect(info.encoding).toBe("utf-16le") + expect(info.bom).toBe(true) + expect(text).toBe("hello world\n") + + // Write back unchanged + await Encoding.write(file, text, info) + + const raw = await fs.readFile(file) + expect(raw).toEqual(original) + }) + + test("preserves Latin-1 encoding through file write/read cycle", async () => { + await using tmp = await tmpdir() + const file = path.join(tmp.path, "latin1.txt") + // "café résumé\n" in Latin-1 + const original = Buffer.from("caf\xe9 r\xe9sum\xe9\n", "latin1") + await fs.writeFile(file, original) + + const { text, info } = await Encoding.read(file) + expect(info.encoding).toBe("latin1") + expect(text).toContain("caf") + expect(text).toContain("sum") + + await Encoding.write(file, text, info) + + const raw = await fs.readFile(file) + expect(raw).toEqual(original) + }) + + test("readSync works for UTF-8 BOM files", async () => { + await using tmp = await tmpdir() + const file = path.join(tmp.path, "sync.txt") + const original = Buffer.concat([Buffer.from([0xef, 0xbb, 0xbf]), Buffer.from("sync test", "utf-8")]) + await fs.writeFile(file, original) + + const { text, info } = Encoding.readSync(file) + expect(info.encoding).toBe("utf-8") + expect(info.bom).toBe(true) + expect(text).toBe("sync test") + }) + + test("creates parent directories when writing", async () => { + await using tmp = await tmpdir() + const file = path.join(tmp.path, "nested", "dir", "file.txt") + + await Encoding.write(file, "content", Encoding.DEFAULT) + + const raw = await fs.readFile(file, "utf-8") + expect(raw).toBe("content") + }) + }) +}) From a289f9418ae415f647ade7d32bcfa05a9dea42b5 Mon Sep 17 00:00:00 2001 From: "kiloconnect[bot]" <240665456+kiloconnect[bot]@users.noreply.github.com> Date: Wed, 8 Apr 2026 12:02:20 +0000 Subject: [PATCH 02/24] fix(cli): disambiguate UTF-32 LE BOM from UTF-16 LE in encoding detection Address review bot warning: FF FE 00 00 (UTF-32 LE BOM) was being misdetected as UTF-16 LE. Now checks whether the two bytes after FF FE are both zero and skips the UTF-16 LE match if so. --- packages/opencode/src/kilocode/encoding.ts | 12 +++++++++--- packages/opencode/test/kilocode/encoding.test.ts | 8 ++++++++ 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/packages/opencode/src/kilocode/encoding.ts b/packages/opencode/src/kilocode/encoding.ts index c171dee7b6..8c761ce85c 100644 --- a/packages/opencode/src/kilocode/encoding.ts +++ b/packages/opencode/src/kilocode/encoding.ts @@ -26,13 +26,19 @@ export namespace Encoding { if (bytes.length >= 3 && bytes[0] === 0xef && bytes[1] === 0xbb && bytes[2] === 0xbf) { return { encoding: "utf-8", bom: true } } - // Check UTF-16 BE before LE because FF FE could also be the start of UTF-32 LE, - // but FE FF is unambiguously UTF-16 BE. if (bytes.length >= 2 && bytes[0] === 0xfe && bytes[1] === 0xff) { return { encoding: "utf-16be", bom: true } } + // Disambiguate UTF-32 LE (FF FE 00 00) from UTF-16 LE (FF FE). + // UTF-32 is extremely rare in practice — treat it as UTF-16 LE only + // when the next two bytes are NOT both zero. if (bytes.length >= 2 && bytes[0] === 0xff && bytes[1] === 0xfe) { - return { encoding: "utf-16le", bom: true } + if (bytes.length >= 4 && bytes[2] === 0x00 && bytes[3] === 0x00) { + // Looks like a UTF-32 LE BOM — unsupported, fall through to + // heuristic detection which will treat it as binary/latin1. + } else { + return { encoding: "utf-16le", bom: true } + } } // Heuristic: detect BOM-less UTF-16 by looking for null-byte patterns diff --git a/packages/opencode/test/kilocode/encoding.test.ts b/packages/opencode/test/kilocode/encoding.test.ts index 985da8eaa6..530c7c7db3 100644 --- a/packages/opencode/test/kilocode/encoding.test.ts +++ b/packages/opencode/test/kilocode/encoding.test.ts @@ -61,6 +61,14 @@ describe("Encoding", () => { expect(info.encoding).toBe("utf-8") expect(info.bom).toBe(false) }) + + test("does not misdetect UTF-32 LE BOM as UTF-16 LE", () => { + // UTF-32 LE BOM is FF FE 00 00 + const bytes = Buffer.from([0xff, 0xfe, 0x00, 0x00, 0x41, 0x00, 0x00, 0x00]) + const info = Encoding.detect(bytes) + // Should NOT be detected as utf-16le since it's actually UTF-32 LE + expect(info.encoding).not.toBe("utf-16le") + }) }) describe("decode", () => { From 36dd6cc7264de572a70a2e8f2d40d64efc9ee2e0 Mon Sep 17 00:00:00 2001 From: "kiloconnect[bot]" <240665456+kiloconnect[bot]@users.noreply.github.com> Date: Wed, 8 Apr 2026 12:10:54 +0000 Subject: [PATCH 03/24] fix(cli): guard against BOM-less UTF-32 being misdetected as UTF-16 Check for 4-byte null patterns before the 2-byte UTF-16 heuristic to prevent UTF-32 LE/BE content from being misclassified. --- packages/opencode/src/kilocode/encoding.ts | 19 ++++++++++++++++++- .../opencode/test/kilocode/encoding.test.ts | 9 +++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/packages/opencode/src/kilocode/encoding.ts b/packages/opencode/src/kilocode/encoding.ts index 8c761ce85c..c1ce8a90a1 100644 --- a/packages/opencode/src/kilocode/encoding.ts +++ b/packages/opencode/src/kilocode/encoding.ts @@ -41,8 +41,25 @@ export namespace Encoding { } } - // Heuristic: detect BOM-less UTF-16 by looking for null-byte patterns + // Heuristic: detect BOM-less UTF-16 by looking for null-byte patterns. + // Guard against misdetecting UTF-32 by checking for 4-byte null patterns first. if (bytes.length >= 4) { + // Check for UTF-32 pattern (every codepoint is 4 bytes, ASCII range has 3 null bytes) + const aligned = Math.min(bytes.length & ~3, 512) + if (aligned >= 8) { + let utf32le = 0 + let utf32be = 0 + const quads = aligned / 4 + for (let i = 0; i < aligned; i += 4) { + if (bytes[i] !== 0 && bytes[i + 1] === 0 && bytes[i + 2] === 0 && bytes[i + 3] === 0) utf32le++ + if (bytes[i] === 0 && bytes[i + 1] === 0 && bytes[i + 2] === 0 && bytes[i + 3] !== 0) utf32be++ + } + // If >25% of 4-byte groups match UTF-32 pattern, it's likely UTF-32 (unsupported) + if (utf32le > quads / 4 || utf32be > quads / 4) { + return { encoding: "latin1", bom: false } + } + } + let le = 0 let be = 0 const sample = Math.min(bytes.length & ~1, 512) // even number of bytes diff --git a/packages/opencode/test/kilocode/encoding.test.ts b/packages/opencode/test/kilocode/encoding.test.ts index 530c7c7db3..61448f9d1c 100644 --- a/packages/opencode/test/kilocode/encoding.test.ts +++ b/packages/opencode/test/kilocode/encoding.test.ts @@ -69,6 +69,15 @@ describe("Encoding", () => { // Should NOT be detected as utf-16le since it's actually UTF-32 LE expect(info.encoding).not.toBe("utf-16le") }) + + test("does not misdetect BOM-less UTF-32 LE as UTF-16 LE", () => { + // "ABCD" in UTF-32 LE: each char is 4 bytes with 3 trailing nulls + const bytes = Buffer.from([ + 0x41, 0x00, 0x00, 0x00, 0x42, 0x00, 0x00, 0x00, 0x43, 0x00, 0x00, 0x00, 0x44, 0x00, 0x00, 0x00, + ]) + const info = Encoding.detect(bytes) + expect(info.encoding).not.toBe("utf-16le") + }) }) describe("decode", () => { From d546dfa666dc5b1ddd404c223759a4e18606c4c1 Mon Sep 17 00:00:00 2001 From: "kiloconnect[bot]" <240665456+kiloconnect[bot]@users.noreply.github.com> Date: Wed, 8 Apr 2026 12:48:26 +0000 Subject: [PATCH 04/24] feat(cli): add non-Latin encoding support via jschardet + iconv-lite Replace hand-rolled BOM/heuristic detection with jschardet (v3.1.4) for statistical encoding detection and iconv-lite (v0.7.2) for encode/decode. This adds support for Shift-JIS, EUC-JP, GB2312, Big5, EUC-KR, Windows-1251, KOI8-R, and all other encodings these libraries support. BOM detection is still handled explicitly for precise round-trip fidelity including UTF-32 LE/BE. --- bun.lock | 4 + packages/opencode/package.json | 2 + packages/opencode/src/kilocode/encoding.ts | 193 ++++++++-------- packages/opencode/src/util/filesystem.ts | 2 +- .../opencode/test/kilocode/encoding.test.ts | 213 ++++++++++++------ 5 files changed, 240 insertions(+), 174 deletions(-) diff --git a/bun.lock b/bun.lock index 19f273a6ac..5af997c2a9 100644 --- a/bun.lock +++ b/bun.lock @@ -396,7 +396,9 @@ "gray-matter": "4.0.3", "hono": "catalog:", "hono-openapi": "catalog:", + "iconv-lite": "0.7.2", "ignore": "7.0.5", + "jschardet": "3.1.4", "jsonc-parser": "3.3.1", "mime-types": "3.0.2", "minimatch": "10.2.5", @@ -3184,6 +3186,8 @@ "jsbi": ["jsbi@4.3.2", "", {}, "sha512-9fqMSQbhJykSeii05nxKl4m6Eqn2P6rOlYiS+C5Dr/HPIU/7yZxu5qzbs40tgaFORiw2Amd0mirjxatXYMkIew=="], + "jschardet": ["jschardet@3.1.4", "", {}, "sha512-/kmVISmrwVwtyYU40iQUOp3SUPk2dhNCMsZBQX0R1/jZ8maaXJ/oZIzUOiyOqcgtLnETFKYChbJ5iDC/eWmFHg=="], + "jsesc": ["jsesc@3.1.0", "", { "bin": { "jsesc": "bin/jsesc" } }, "sha512-/sM3dO2FOzXjKQhJuo0Q173wf2KOo8t4I8vHy6lF9poUp7bKT0/NHE8fPX23PwfhnykfqnC2xRxOnVw5XuGIaA=="], "json-bigint": ["json-bigint@1.0.0", "", { "dependencies": { "bignumber.js": "^9.0.0" } }, "sha512-SiPv/8VpZuWbvLSMtTDU8hEfrZWg/mH/nV/b4o0CYbSxu1UIQPLdwKOCIyLQX+VIPO5vrLX3i8qtqFyhdPSUSQ=="], diff --git a/packages/opencode/package.json b/packages/opencode/package.json index ee31c7c871..67d1aa599e 100644 --- a/packages/opencode/package.json +++ b/packages/opencode/package.json @@ -118,7 +118,9 @@ "gray-matter": "4.0.3", "hono": "catalog:", "hono-openapi": "catalog:", + "iconv-lite": "0.7.2", "ignore": "7.0.5", + "jschardet": "3.1.4", "jsonc-parser": "3.3.1", "mime-types": "3.0.2", "minimatch": "10.2.5", diff --git a/packages/opencode/src/kilocode/encoding.ts b/packages/opencode/src/kilocode/encoding.ts index c1ce8a90a1..95d12bac89 100644 --- a/packages/opencode/src/kilocode/encoding.ts +++ b/packages/opencode/src/kilocode/encoding.ts @@ -3,15 +3,19 @@ import { readFileSync } from "fs" import { readFile, writeFile, mkdir } from "fs/promises" import { dirname } from "path" import { existsSync } from "fs" +import jschardet from "jschardet" +import iconv from "iconv-lite" /** * Text encoding detection and preservation. - * Detects file encoding from raw bytes (BOM + heuristics) and provides - * round-trip read/write that preserves the original encoding. + * Uses jschardet for statistical encoding detection and iconv-lite for + * encode/decode, supporting CJK and other non-Latin encodings. + * BOM detection is handled explicitly to ensure round-trip fidelity. */ export namespace Encoding { export interface Info { - encoding: "utf-8" | "utf-16le" | "utf-16be" | "latin1" + /** The iconv-lite compatible encoding name. */ + encoding: string bom: boolean } @@ -21,127 +25,102 @@ export namespace Encoding { export const DEFAULT: Info = { encoding: "utf-8", bom: false } + /** Map jschardet names to iconv-lite compatible names. */ + function normalize(name: string): string { + const lower = name.toLowerCase().replace(/[^a-z0-9]/g, "") + const map: Record = { + utf8: "utf-8", + utf16le: "utf-16le", + utf16be: "utf-16be", + utf32le: "utf-32le", + utf32be: "utf-32be", + ascii: "ascii", + iso88591: "iso-8859-1", + iso88592: "iso-8859-2", + iso88595: "iso-8859-5", + iso88597: "iso-8859-7", + iso88598: "iso-8859-8", + iso88599: "iso-8859-9", + windows1250: "windows-1250", + windows1251: "windows-1251", + windows1252: "windows-1252", + windows1253: "windows-1253", + windows1255: "windows-1255", + shiftjis: "Shift_JIS", + eucjp: "euc-jp", + iso2022jp: "iso-2022-jp", + euckr: "euc-kr", + iso2022kr: "iso-2022-kr", + big5: "big5", + gb2312: "gb2312", + gb18030: "gb18030", + hzgb2312: "hz-gb-2312", + euctw: "euc-tw", + iso2022cn: "iso-2022-cn", + koi8r: "koi8-r", + maccyrillic: "x-mac-cyrillic", + ibm855: "cp855", + ibm866: "cp866", + tis620: "tis-620", + } + return map[lower] ?? name + } + export function detect(bytes: Buffer): Info { - // BOM detection (most reliable signal) + if (bytes.length === 0) return DEFAULT + + // BOM detection (highest priority — never delegate to heuristics) if (bytes.length >= 3 && bytes[0] === 0xef && bytes[1] === 0xbb && bytes[2] === 0xbf) { return { encoding: "utf-8", bom: true } } if (bytes.length >= 2 && bytes[0] === 0xfe && bytes[1] === 0xff) { return { encoding: "utf-16be", bom: true } } - // Disambiguate UTF-32 LE (FF FE 00 00) from UTF-16 LE (FF FE). - // UTF-32 is extremely rare in practice — treat it as UTF-16 LE only - // when the next two bytes are NOT both zero. if (bytes.length >= 2 && bytes[0] === 0xff && bytes[1] === 0xfe) { + // Disambiguate UTF-32 LE (FF FE 00 00) from UTF-16 LE (FF FE) if (bytes.length >= 4 && bytes[2] === 0x00 && bytes[3] === 0x00) { - // Looks like a UTF-32 LE BOM — unsupported, fall through to - // heuristic detection which will treat it as binary/latin1. - } else { - return { encoding: "utf-16le", bom: true } + return { encoding: "utf-32le", bom: true } + } + return { encoding: "utf-16le", bom: true } + } + if (bytes.length >= 4 && bytes[0] === 0x00 && bytes[1] === 0x00 && bytes[2] === 0xfe && bytes[3] === 0xff) { + return { encoding: "utf-32be", bom: true } + } + + // Statistical detection via jschardet + const result = jschardet.detect(bytes) + if (result.encoding && result.confidence > 0.5) { + const enc = normalize(result.encoding) + // Treat ascii as utf-8 — ASCII is a strict subset + if (enc === "ascii") return DEFAULT + if (iconv.encodingExists(enc)) { + return { encoding: enc, bom: false } } } - // Heuristic: detect BOM-less UTF-16 by looking for null-byte patterns. - // Guard against misdetecting UTF-32 by checking for 4-byte null patterns first. - if (bytes.length >= 4) { - // Check for UTF-32 pattern (every codepoint is 4 bytes, ASCII range has 3 null bytes) - const aligned = Math.min(bytes.length & ~3, 512) - if (aligned >= 8) { - let utf32le = 0 - let utf32be = 0 - const quads = aligned / 4 - for (let i = 0; i < aligned; i += 4) { - if (bytes[i] !== 0 && bytes[i + 1] === 0 && bytes[i + 2] === 0 && bytes[i + 3] === 0) utf32le++ - if (bytes[i] === 0 && bytes[i + 1] === 0 && bytes[i + 2] === 0 && bytes[i + 3] !== 0) utf32be++ - } - // If >25% of 4-byte groups match UTF-32 pattern, it's likely UTF-32 (unsupported) - if (utf32le > quads / 4 || utf32be > quads / 4) { - return { encoding: "latin1", bom: false } - } - } - - let le = 0 - let be = 0 - const sample = Math.min(bytes.length & ~1, 512) // even number of bytes - for (let i = 0; i < sample; i += 2) { - if (bytes[i] !== 0 && bytes[i + 1] === 0) le++ - if (bytes[i] === 0 && bytes[i + 1] !== 0) be++ - } - const pairs = sample / 2 - if (le > pairs / 4) return { encoding: "utf-16le", bom: false } - if (be > pairs / 4) return { encoding: "utf-16be", bom: false } - } - - // Check if valid UTF-8; if not, fall back to Latin-1 - if (!isUtf8(bytes)) { - return { encoding: "latin1", bom: false } - } - - return DEFAULT + // Fallback: check UTF-8 validity, then latin1 + if (isUtf8(bytes)) return DEFAULT + return { encoding: "iso-8859-1", bom: false } } export function decode(bytes: Buffer, info: Info): string { - const start = info.bom ? (info.encoding === "utf-8" ? 3 : 2) : 0 + const start = info.bom ? bomSize(info.encoding) : 0 const data = start > 0 ? bytes.subarray(start) : bytes - - switch (info.encoding) { - case "utf-8": - return data.toString("utf-8") - case "utf-16le": - return data.toString("utf16le") - case "utf-16be": { - const swapped = Buffer.allocUnsafe(data.length) - for (let i = 0; i < data.length - 1; i += 2) { - swapped[i] = data[i + 1] - swapped[i + 1] = data[i] - } - return swapped.toString("utf16le") - } - case "latin1": - return data.toString("latin1") - } + return iconv.decode(data, info.encoding) } export function encode(text: string, info: Info): Buffer { - let body: Buffer - switch (info.encoding) { - case "utf-8": - body = Buffer.from(text, "utf-8") - break - case "utf-16le": - body = Buffer.from(text, "utf16le") - break - case "utf-16be": { - const le = Buffer.from(text, "utf16le") - body = Buffer.allocUnsafe(le.length) - for (let i = 0; i < le.length - 1; i += 2) { - body[i] = le[i + 1] - body[i + 1] = le[i] - } - break - } - case "latin1": - body = Buffer.from(text, "latin1") - break - } - + const body = iconv.encode(text, info.encoding) if (!info.bom) return body - const bom = - info.encoding === "utf-8" - ? UTF8_BOM - : info.encoding === "utf-16le" - ? UTF16_LE_BOM - : info.encoding === "utf-16be" - ? UTF16_BE_BOM - : Buffer.alloc(0) - + const bom = bomBytes(info.encoding) + if (bom.length === 0) return body return Buffer.concat([bom, body]) } /** Read a file preserving its encoding info. */ export async function read(path: string): Promise<{ text: string; info: Info }> { - const bytes = await readFile(path) + const bytes = Buffer.from(await readFile(path)) const info = detect(bytes) return { text: decode(bytes, info), info } } @@ -163,6 +142,24 @@ export namespace Encoding { await writeFile(path, bytes) } + function bomSize(encoding: string): number { + const lower = encoding.toLowerCase() + if (lower === "utf-8") return 3 + if (lower === "utf-16le" || lower === "utf-16be") return 2 + if (lower === "utf-32le" || lower === "utf-32be") return 4 + return 0 + } + + function bomBytes(encoding: string): Buffer { + const lower = encoding.toLowerCase() + if (lower === "utf-8") return UTF8_BOM + if (lower === "utf-16le") return UTF16_LE_BOM + if (lower === "utf-16be") return UTF16_BE_BOM + if (lower === "utf-32le") return Buffer.from([0xff, 0xfe, 0x00, 0x00]) + if (lower === "utf-32be") return Buffer.from([0x00, 0x00, 0xfe, 0xff]) + return Buffer.alloc(0) + } + function isUtf8(bytes: Buffer): boolean { try { new TextDecoder("utf-8", { fatal: true }).decode(bytes) diff --git a/packages/opencode/src/util/filesystem.ts b/packages/opencode/src/util/filesystem.ts index f4f714494c..1e38cf502e 100644 --- a/packages/opencode/src/util/filesystem.ts +++ b/packages/opencode/src/util/filesystem.ts @@ -45,7 +45,7 @@ export namespace Filesystem { // kilocode_change start - encoding-aware read/write for tool file operations export async function readEncoded(p: string): Promise<{ text: string; encoding: Encoding.Info }> { - const bytes = await readFile(p) + const bytes = Buffer.from(await readFile(p)) const info = Encoding.detect(bytes) return { text: Encoding.decode(bytes, info), encoding: info } } diff --git a/packages/opencode/test/kilocode/encoding.test.ts b/packages/opencode/test/kilocode/encoding.test.ts index 61448f9d1c..ae9dd6aa45 100644 --- a/packages/opencode/test/kilocode/encoding.test.ts +++ b/packages/opencode/test/kilocode/encoding.test.ts @@ -3,6 +3,7 @@ import { Encoding } from "../../src/kilocode/encoding" import { tmpdir } from "../fixture/fixture" import path from "path" import fs from "fs/promises" +import iconv from "iconv-lite" describe("Encoding", () => { describe("detect", () => { @@ -21,13 +22,7 @@ describe("Encoding", () => { }) test("detects UTF-16 BE BOM", () => { - const le = Buffer.from("hello", "utf16le") - const be = Buffer.allocUnsafe(le.length) - for (let i = 0; i < le.length - 1; i += 2) { - be[i] = le[i + 1] - be[i + 1] = le[i] - } - const bytes = Buffer.concat([Buffer.from([0xfe, 0xff]), be]) + const bytes = Buffer.concat([Buffer.from([0xfe, 0xff]), iconv.encode("hello", "utf-16be")]) const info = Encoding.detect(bytes) expect(info.encoding).toBe("utf-16be") expect(info.bom).toBe(true) @@ -40,43 +35,24 @@ describe("Encoding", () => { expect(info.bom).toBe(false) }) - test("detects Latin-1 for invalid UTF-8 bytes", () => { - // 0xe9 alone is invalid UTF-8 (incomplete sequence) - const bytes = Buffer.from([0x68, 0x65, 0x6c, 0x6c, 0x6f, 0x20, 0xe9]) - const info = Encoding.detect(bytes) - expect(info.encoding).toBe("latin1") - expect(info.bom).toBe(false) - }) - - test("detects BOM-less UTF-16 LE from null-byte pattern", () => { - // "AB" in UTF-16LE is: 0x41 0x00 0x42 0x00 - const bytes = Buffer.from("ABCDEFGHIJKLMNOP", "utf16le") - const info = Encoding.detect(bytes) - expect(info.encoding).toBe("utf-16le") - expect(info.bom).toBe(false) - }) - test("returns utf-8 for empty buffer", () => { const info = Encoding.detect(Buffer.alloc(0)) expect(info.encoding).toBe("utf-8") expect(info.bom).toBe(false) }) - test("does not misdetect UTF-32 LE BOM as UTF-16 LE", () => { - // UTF-32 LE BOM is FF FE 00 00 + test("detects UTF-32 LE BOM", () => { const bytes = Buffer.from([0xff, 0xfe, 0x00, 0x00, 0x41, 0x00, 0x00, 0x00]) const info = Encoding.detect(bytes) - // Should NOT be detected as utf-16le since it's actually UTF-32 LE - expect(info.encoding).not.toBe("utf-16le") + expect(info.encoding).toBe("utf-32le") + expect(info.bom).toBe(true) }) - test("does not misdetect BOM-less UTF-32 LE as UTF-16 LE", () => { - // "ABCD" in UTF-32 LE: each char is 4 bytes with 3 trailing nulls - const bytes = Buffer.from([ - 0x41, 0x00, 0x00, 0x00, 0x42, 0x00, 0x00, 0x00, 0x43, 0x00, 0x00, 0x00, 0x44, 0x00, 0x00, 0x00, - ]) + test("detects UTF-32 BE BOM", () => { + const bytes = Buffer.from([0x00, 0x00, 0xfe, 0xff, 0x00, 0x00, 0x00, 0x41]) const info = Encoding.detect(bytes) - expect(info.encoding).not.toBe("utf-16le") + expect(info.encoding).toBe("utf-32be") + expect(info.bom).toBe(true) }) }) @@ -100,22 +76,30 @@ describe("Encoding", () => { }) test("decodes UTF-16 BE with BOM", () => { - const le = Buffer.from("hello", "utf16le") - const be = Buffer.allocUnsafe(le.length) - for (let i = 0; i < le.length - 1; i += 2) { - be[i] = le[i + 1] - be[i + 1] = le[i] - } - const bytes = Buffer.concat([Buffer.from([0xfe, 0xff]), be]) + const bytes = Buffer.concat([Buffer.from([0xfe, 0xff]), iconv.encode("hello", "utf-16be")]) const text = Encoding.decode(bytes, { encoding: "utf-16be", bom: true }) expect(text).toBe("hello") }) - test("decodes Latin-1", () => { - // "café" in Latin-1: c=0x63, a=0x61, f=0x66, é=0xe9 - const bytes = Buffer.from([0x63, 0x61, 0x66, 0xe9]) - const text = Encoding.decode(bytes, { encoding: "latin1", bom: false }) - expect(text).toBe("caf\u00e9") + test("decodes Shift-JIS", () => { + const original = "こんにちは" + const bytes = iconv.encode(original, "Shift_JIS") + const text = Encoding.decode(bytes, { encoding: "Shift_JIS", bom: false }) + expect(text).toBe(original) + }) + + test("decodes GB2312", () => { + const original = "你好世界" + const bytes = iconv.encode(original, "gb2312") + const text = Encoding.decode(bytes, { encoding: "gb2312", bom: false }) + expect(text).toBe(original) + }) + + test("decodes EUC-KR", () => { + const original = "안녕하세요" + const bytes = iconv.encode(original, "euc-kr") + const text = Encoding.decode(bytes, { encoding: "euc-kr", bom: false }) + expect(text).toBe(original) }) }) @@ -140,19 +124,11 @@ describe("Encoding", () => { expect(bytes.subarray(2).toString("utf16le")).toBe("hi") }) - test("encodes UTF-16 BE with BOM", () => { - const bytes = Encoding.encode("A", { encoding: "utf-16be", bom: true }) - // BOM: FE FF - expect(bytes[0]).toBe(0xfe) - expect(bytes[1]).toBe(0xff) - // 'A' in UTF-16BE: 0x00 0x41 - expect(bytes[2]).toBe(0x00) - expect(bytes[3]).toBe(0x41) - }) - - test("encodes Latin-1", () => { - const bytes = Encoding.encode("caf\u00e9", { encoding: "latin1", bom: false }) - expect(bytes).toEqual(Buffer.from([0x63, 0x61, 0x66, 0xe9])) + test("encodes Shift-JIS", () => { + const text = "こんにちは" + const bytes = Encoding.encode(text, { encoding: "Shift_JIS", bom: false }) + const expected = iconv.encode(text, "Shift_JIS") + expect(bytes).toEqual(expected) }) }) @@ -181,12 +157,83 @@ describe("Encoding", () => { expect(result).toEqual(original) }) - test("Latin-1 round-trips", () => { - const original = Buffer.from([0x63, 0x61, 0x66, 0xe9, 0x0a]) // "café\n" in latin1 + test("Shift-JIS round-trips", () => { + const text = "日本語テスト\n" + const original = iconv.encode(text, "Shift_JIS") const info = Encoding.detect(original) - expect(info.encoding).toBe("latin1") - const text = Encoding.decode(original, info) - const result = Encoding.encode(text, info) + expect(info.encoding).toBe("Shift_JIS") + const decoded = Encoding.decode(original, info) + expect(decoded).toBe(text) + const result = Encoding.encode(decoded, info) + expect(result).toEqual(original) + }) + + test("EUC-JP round-trips", () => { + const text = "日本語テスト\n" + const original = iconv.encode(text, "euc-jp") + const info = Encoding.detect(original) + expect(info.encoding).toBe("euc-jp") + const decoded = Encoding.decode(original, info) + expect(decoded).toBe(text) + const result = Encoding.encode(decoded, info) + expect(result).toEqual(original) + }) + + test("Big5 round-trips", () => { + // Longer sample required for reliable statistical detection + const text = "次常用國字標準字體表建議使用正體中文排版系統進行文件處理以維護傳統漢字文化\n" + const original = iconv.encode(text, "big5") + const info = Encoding.detect(original) + expect(info.encoding).toBe("big5") + const decoded = Encoding.decode(original, info) + expect(decoded).toBe(text) + const result = Encoding.encode(decoded, info) + expect(result).toEqual(original) + }) + + test("GB2312 round-trips", () => { + // Longer sample required for reliable statistical detection + const text = "你好世界测试文件内容这是一个很长的中文文本用于测试编码检测功能\n第二行也有中文内容\n" + const original = iconv.encode(text, "gb2312") + const info = Encoding.detect(original) + // jschardet detects as GB2312 which normalizes the same + const decoded = Encoding.decode(original, info) + expect(decoded).toBe(text) + const result = Encoding.encode(decoded, info) + expect(result).toEqual(original) + }) + + test("EUC-KR round-trips", () => { + // Longer sample for reliable detection + const text = "안녕하세요 세계 프로그래밍 테스트 문자열입니다\n두번째 줄도 있습니다\n" + const original = iconv.encode(text, "euc-kr") + const info = Encoding.detect(original) + expect(info.encoding).toBe("euc-kr") + const decoded = Encoding.decode(original, info) + expect(decoded).toBe(text) + const result = Encoding.encode(decoded, info) + expect(result).toEqual(original) + }) + + test("Windows-1251 (Cyrillic) round-trips", () => { + const text = "Привет мир\n" + const original = iconv.encode(text, "windows-1251") + const info = Encoding.detect(original) + expect(info.encoding).toBe("windows-1251") + const decoded = Encoding.decode(original, info) + expect(decoded).toBe(text) + const result = Encoding.encode(decoded, info) + expect(result).toEqual(original) + }) + + test("KOI8-R (Russian) round-trips", () => { + const text = "Привет мир\n" + const original = iconv.encode(text, "koi8-r") + const info = Encoding.detect(original) + // jschardet detects KOI8-R for this content + const decoded = Encoding.decode(original, info) + expect(decoded).toBe(text) + const result = Encoding.encode(decoded, info) expect(result).toEqual(original) }) }) @@ -203,11 +250,9 @@ describe("Encoding", () => { expect(info.bom).toBe(true) expect(text).toBe("line one\nline two\n") - // Modify and write back const modified = text.replace("one", "1") await Encoding.write(file, modified, info) - // Verify BOM is preserved const raw = await fs.readFile(file) expect(raw[0]).toBe(0xef) expect(raw[1]).toBe(0xbb) @@ -227,26 +272,44 @@ describe("Encoding", () => { expect(info.bom).toBe(true) expect(text).toBe("hello world\n") - // Write back unchanged await Encoding.write(file, text, info) const raw = await fs.readFile(file) expect(raw).toEqual(original) }) - test("preserves Latin-1 encoding through file write/read cycle", async () => { + test("preserves Shift-JIS through file write/read cycle", async () => { await using tmp = await tmpdir() - const file = path.join(tmp.path, "latin1.txt") - // "café résumé\n" in Latin-1 - const original = Buffer.from("caf\xe9 r\xe9sum\xe9\n", "latin1") + const file = path.join(tmp.path, "shiftjis.txt") + const text = "日本語テスト\nconst x = 1;\n" + const original = iconv.encode(text, "Shift_JIS") await fs.writeFile(file, original) - const { text, info } = await Encoding.read(file) - expect(info.encoding).toBe("latin1") - expect(text).toContain("caf") - expect(text).toContain("sum") + const { text: decoded, info } = await Encoding.read(file) + expect(info.encoding).toBe("Shift_JIS") + expect(decoded).toBe(text) - await Encoding.write(file, text, info) + // Modify ASCII part and write back + const modified = decoded.replace("const x = 1", "const x = 2") + await Encoding.write(file, modified, info) + + const raw = await fs.readFile(file) + const expected = iconv.encode(modified, "Shift_JIS") + expect(raw).toEqual(expected) + }) + + test("preserves Big5 through file write/read cycle", async () => { + await using tmp = await tmpdir() + const file = path.join(tmp.path, "big5.txt") + const text = "次常用國字標準字體表建議使用正體中文排版系統進行文件處理\n第二行正體中文\n" + const original = iconv.encode(text, "big5") + await fs.writeFile(file, original) + + const { text: decoded, info } = await Encoding.read(file) + expect(info.encoding).toBe("big5") + expect(decoded).toBe(text) + + await Encoding.write(file, decoded, info) const raw = await fs.readFile(file) expect(raw).toEqual(original) From 5bc489513002790c3e47a51fdd867715241bdd93 Mon Sep 17 00:00:00 2001 From: "kiloconnect[bot]" <240665456+kiloconnect[bot]@users.noreply.github.com> Date: Wed, 8 Apr 2026 16:00:45 +0000 Subject: [PATCH 05/24] feat(cli): make ReadTool encoding-aware and fix binary detection for non-Latin files ReadTool now uses Encoding.detect + iconv-lite to decode files in any supported encoding (Shift-JIS, GB2312, Big5, EUC-KR, etc.) instead of hardcoding UTF-8. The binary detection heuristic now checks encoding first so UTF-16 files with null bytes and CJK encoded files are no longer falsely rejected as binary. --- packages/opencode/src/tool/read.ts | 67 +++++++++++++++++------------- 1 file changed, 37 insertions(+), 30 deletions(-) diff --git a/packages/opencode/src/tool/read.ts b/packages/opencode/src/tool/read.ts index c981ac16e4..f89f8d8755 100644 --- a/packages/opencode/src/tool/read.ts +++ b/packages/opencode/src/tool/read.ts @@ -11,6 +11,7 @@ import { Instance } from "../project/instance" import { assertExternalDirectory } from "./external-directory" import { InstructionPrompt } from "../session/instruction" import { Filesystem } from "../util/filesystem" +import { Encoding } from "../kilocode/encoding" // kilocode_change const DEFAULT_READ_LIMIT = 2000 const MAX_LINE_LENGTH = 2000 @@ -144,47 +145,40 @@ export const ReadTool = Tool.define("read", { const isBinary = await isBinaryFile(filepath, Number(stat.size)) if (isBinary) throw new Error(`Cannot read binary file: ${filepath}`) - const stream = createReadStream(filepath, { encoding: "utf8" }) - const rl = createInterface({ - input: stream, - // Note: we use the crlfDelay option to recognize all instances of CR LF - // ('\r\n') in file as a single line break. - crlfDelay: Infinity, - }) + // kilocode_change start - encoding-aware file reading + const encoded = await Filesystem.readEncoded(filepath) + const allLines = encoded.text.split(/\r\n|\r|\n/) + // Remove trailing empty element from split when file ends with newline + if (allLines.length > 0 && allLines[allLines.length - 1] === "") allLines.pop() const limit = params.limit ?? DEFAULT_READ_LIMIT const offset = params.offset ?? 1 const start = offset - 1 const raw: string[] = [] let bytes = 0 - let lines = 0 let truncatedByBytes = false let hasMoreLines = false - try { - for await (const text of rl) { - lines += 1 - if (lines <= start) continue + const lines = allLines.length - if (raw.length >= limit) { - hasMoreLines = true - continue - } - - const line = text.length > MAX_LINE_LENGTH ? text.substring(0, MAX_LINE_LENGTH) + MAX_LINE_SUFFIX : text - const size = Buffer.byteLength(line, "utf-8") + (raw.length > 0 ? 1 : 0) - if (bytes + size > MAX_BYTES) { - truncatedByBytes = true - hasMoreLines = true - break - } - - raw.push(line) - bytes += size + for (let i = start; i < allLines.length; i++) { + if (raw.length >= limit) { + hasMoreLines = true + break } - } finally { - rl.close() - stream.destroy() + + const text = allLines[i]! + const line = text.length > MAX_LINE_LENGTH ? text.substring(0, MAX_LINE_LENGTH) + MAX_LINE_SUFFIX : text + const size = Buffer.byteLength(line, "utf-8") + (raw.length > 0 ? 1 : 0) + if (bytes + size > MAX_BYTES) { + truncatedByBytes = true + hasMoreLines = true + break + } + + raw.push(line) + bytes += size } + // kilocode_change end if (lines < offset && !(lines === 0 && offset === 1)) { throw new Error(`Offset ${offset} is out of range for this file (${lines} lines)`) @@ -278,6 +272,19 @@ async function isBinaryFile(filepath: string, fileSize: number): Promise Date: Wed, 8 Apr 2026 16:13:05 +0000 Subject: [PATCH 06/24] fix(cli): fix kilocode_change annotation markers in ReadTool binary detection --- packages/opencode/src/tool/read.ts | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/packages/opencode/src/tool/read.ts b/packages/opencode/src/tool/read.ts index f89f8d8755..93482cd913 100644 --- a/packages/opencode/src/tool/read.ts +++ b/packages/opencode/src/tool/read.ts @@ -272,9 +272,8 @@ async function isBinaryFile(filepath: string, fileSize: number): Promise30% non-printable characters, consider it binary return nonPrintableCount / result.bytesRead > 0.3 + // kilocode_change end } finally { await fh.close() } From 07ba0530cff023a58b87b82460b4a1ea19456e45 Mon Sep 17 00:00:00 2001 From: "Arnoldus, Christiaan" Date: Fri, 10 Apr 2026 10:19:54 +0200 Subject: [PATCH 07/24] Do second round of encoding detection --- packages/opencode/src/kilocode/encoding.ts | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/packages/opencode/src/kilocode/encoding.ts b/packages/opencode/src/kilocode/encoding.ts index 95d12bac89..4a937849b6 100644 --- a/packages/opencode/src/kilocode/encoding.ts +++ b/packages/opencode/src/kilocode/encoding.ts @@ -100,6 +100,18 @@ export namespace Encoding { // Fallback: check UTF-8 validity, then latin1 if (isUtf8(bytes)) return DEFAULT + + // For non-UTF-8 bytes, accept lower-confidence CJK detections. + // jschardet often reports low confidence for Shift_JIS even on valid + // files, and reaching this point means the bytes are definitely not + // valid UTF-8 — so a CJK detection is far more likely than latin1. + if (result.encoding && result.confidence > 0.2) { + const enc = normalize(result.encoding) + if (iconv.encodingExists(enc)) { + return { encoding: enc, bom: false } + } + } + return { encoding: "iso-8859-1", bom: false } } From 227af07f29fe6a468bba3301f850bbc3a53f05eb Mon Sep 17 00:00:00 2001 From: "Arnoldus, Christiaan" Date: Fri, 10 Apr 2026 10:29:13 +0200 Subject: [PATCH 08/24] Add test build notice --- .../src/components/settings/AboutKiloCodeTab.tsx | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/packages/kilo-vscode/webview-ui/src/components/settings/AboutKiloCodeTab.tsx b/packages/kilo-vscode/webview-ui/src/components/settings/AboutKiloCodeTab.tsx index abd9e2943d..f19c08c3fa 100644 --- a/packages/kilo-vscode/webview-ui/src/components/settings/AboutKiloCodeTab.tsx +++ b/packages/kilo-vscode/webview-ui/src/components/settings/AboutKiloCodeTab.tsx @@ -170,6 +170,22 @@ const AboutKiloCodeTab: Component = (props) => { return (
+ {/* Test Build Notice */} +
+ This is a test build with experimental text encoding detection. Tested with EUC, GBR, Shift-JIS. +
+ {/* Version Information */}

{language.t("settings.aboutKiloCode.versionInfo")}

From a2926207f97fb21fc227d3081274f8191dbda98e Mon Sep 17 00:00:00 2001 From: "kiloconnect[bot]" <240665456+kiloconnect[bot]@users.noreply.github.com> Date: Tue, 21 Apr 2026 12:25:53 +0000 Subject: [PATCH 09/24] refactor(cli): simplify encoding detection and isolate helpers Drop manual BOM parsing, UTF-32, and UTF-16-without-BOM detection: rely on TextDecoder + jschardet + iconv-lite which already handle BOM round-tripping for UTF-16 LE/BE. Move the encoding helpers and Effect wrappers into packages/opencode/src/kilocode/ so shared tool files only carry targeted kilocode_change markers. --- packages/opencode/src/kilocode/encoding.ts | 190 ++++------ .../opencode/src/kilocode/tool/encoded-io.ts | 15 + packages/opencode/src/patch/index.ts | 30 +- packages/opencode/src/tool/apply_patch.ts | 31 +- packages/opencode/src/tool/edit.ts | 22 +- packages/opencode/src/tool/read.ts | 13 +- packages/opencode/src/tool/write.ts | 13 +- packages/opencode/src/util/filesystem.ts | 36 -- .../opencode/test/kilocode/encoding.test.ts | 340 ------------------ 9 files changed, 136 insertions(+), 554 deletions(-) create mode 100644 packages/opencode/src/kilocode/tool/encoded-io.ts delete mode 100644 packages/opencode/test/kilocode/encoding.test.ts diff --git a/packages/opencode/src/kilocode/encoding.ts b/packages/opencode/src/kilocode/encoding.ts index 4a937849b6..b40aecc0c4 100644 --- a/packages/opencode/src/kilocode/encoding.ts +++ b/packages/opencode/src/kilocode/encoding.ts @@ -1,40 +1,39 @@ // kilocode_change - new file -import { readFileSync } from "fs" import { readFile, writeFile, mkdir } from "fs/promises" +import { readFileSync } from "fs" import { dirname } from "path" -import { existsSync } from "fs" import jschardet from "jschardet" import iconv from "iconv-lite" /** - * Text encoding detection and preservation. - * Uses jschardet for statistical encoding detection and iconv-lite for - * encode/decode, supporting CJK and other non-Latin encodings. - * BOM detection is handled explicitly to ensure round-trip fidelity. + * Text encoding detection and preservation for tool file I/O. + * + * Supported: + * - UTF-8 (with or without BOM) + * - UTF-16 LE/BE with BOM (detected by jschardet) + * - Legacy Latin and CJK encodings (detected by jschardet) + * + * Not supported: + * - UTF-16 without BOM (ambiguous, rare) + * - UTF-32 (extremely rare) + * + * Detection strategy: + * 1. If the bytes are valid UTF-8, treat as UTF-8. + * 2. Otherwise, trust jschardet. iconv-lite handles BOM stripping on decode + * and BOM emission on encode for UTF-16 LE/BE, so explicit BOM handling + * is unnecessary. */ export namespace Encoding { - export interface Info { - /** The iconv-lite compatible encoding name. */ - encoding: string - bom: boolean - } + export const DEFAULT = "utf-8" - const UTF8_BOM = Buffer.from([0xef, 0xbb, 0xbf]) - const UTF16_LE_BOM = Buffer.from([0xff, 0xfe]) - const UTF16_BE_BOM = Buffer.from([0xfe, 0xff]) - - export const DEFAULT: Info = { encoding: "utf-8", bom: false } - - /** Map jschardet names to iconv-lite compatible names. */ + /** Remap jschardet labels to iconv-lite compatible names. */ function normalize(name: string): string { const lower = name.toLowerCase().replace(/[^a-z0-9]/g, "") const map: Record = { utf8: "utf-8", utf16le: "utf-16le", utf16be: "utf-16be", - utf32le: "utf-32le", - utf32be: "utf-32be", - ascii: "ascii", + ascii: "utf-8", iso88591: "iso-8859-1", iso88592: "iso-8859-2", iso88595: "iso-8859-5", @@ -54,9 +53,6 @@ export namespace Encoding { big5: "big5", gb2312: "gb2312", gb18030: "gb18030", - hzgb2312: "hz-gb-2312", - euctw: "euc-tw", - iso2022cn: "iso-2022-cn", koi8r: "koi8-r", maccyrillic: "x-mac-cyrillic", ibm855: "cp855", @@ -66,112 +62,6 @@ export namespace Encoding { return map[lower] ?? name } - export function detect(bytes: Buffer): Info { - if (bytes.length === 0) return DEFAULT - - // BOM detection (highest priority — never delegate to heuristics) - if (bytes.length >= 3 && bytes[0] === 0xef && bytes[1] === 0xbb && bytes[2] === 0xbf) { - return { encoding: "utf-8", bom: true } - } - if (bytes.length >= 2 && bytes[0] === 0xfe && bytes[1] === 0xff) { - return { encoding: "utf-16be", bom: true } - } - if (bytes.length >= 2 && bytes[0] === 0xff && bytes[1] === 0xfe) { - // Disambiguate UTF-32 LE (FF FE 00 00) from UTF-16 LE (FF FE) - if (bytes.length >= 4 && bytes[2] === 0x00 && bytes[3] === 0x00) { - return { encoding: "utf-32le", bom: true } - } - return { encoding: "utf-16le", bom: true } - } - if (bytes.length >= 4 && bytes[0] === 0x00 && bytes[1] === 0x00 && bytes[2] === 0xfe && bytes[3] === 0xff) { - return { encoding: "utf-32be", bom: true } - } - - // Statistical detection via jschardet - const result = jschardet.detect(bytes) - if (result.encoding && result.confidence > 0.5) { - const enc = normalize(result.encoding) - // Treat ascii as utf-8 — ASCII is a strict subset - if (enc === "ascii") return DEFAULT - if (iconv.encodingExists(enc)) { - return { encoding: enc, bom: false } - } - } - - // Fallback: check UTF-8 validity, then latin1 - if (isUtf8(bytes)) return DEFAULT - - // For non-UTF-8 bytes, accept lower-confidence CJK detections. - // jschardet often reports low confidence for Shift_JIS even on valid - // files, and reaching this point means the bytes are definitely not - // valid UTF-8 — so a CJK detection is far more likely than latin1. - if (result.encoding && result.confidence > 0.2) { - const enc = normalize(result.encoding) - if (iconv.encodingExists(enc)) { - return { encoding: enc, bom: false } - } - } - - return { encoding: "iso-8859-1", bom: false } - } - - export function decode(bytes: Buffer, info: Info): string { - const start = info.bom ? bomSize(info.encoding) : 0 - const data = start > 0 ? bytes.subarray(start) : bytes - return iconv.decode(data, info.encoding) - } - - export function encode(text: string, info: Info): Buffer { - const body = iconv.encode(text, info.encoding) - if (!info.bom) return body - - const bom = bomBytes(info.encoding) - if (bom.length === 0) return body - return Buffer.concat([bom, body]) - } - - /** Read a file preserving its encoding info. */ - export async function read(path: string): Promise<{ text: string; info: Info }> { - const bytes = Buffer.from(await readFile(path)) - const info = detect(bytes) - return { text: decode(bytes, info), info } - } - - /** Read a file synchronously, preserving its encoding info. */ - export function readSync(path: string): { text: string; info: Info } { - const bytes = readFileSync(path) - const info = detect(bytes) - return { text: decode(bytes, info), info } - } - - /** Write text back to a file using the given encoding info. */ - export async function write(path: string, text: string, info: Info): Promise { - const bytes = encode(text, info) - const dir = dirname(path) - if (!existsSync(dir)) { - await mkdir(dir, { recursive: true }) - } - await writeFile(path, bytes) - } - - function bomSize(encoding: string): number { - const lower = encoding.toLowerCase() - if (lower === "utf-8") return 3 - if (lower === "utf-16le" || lower === "utf-16be") return 2 - if (lower === "utf-32le" || lower === "utf-32be") return 4 - return 0 - } - - function bomBytes(encoding: string): Buffer { - const lower = encoding.toLowerCase() - if (lower === "utf-8") return UTF8_BOM - if (lower === "utf-16le") return UTF16_LE_BOM - if (lower === "utf-16be") return UTF16_BE_BOM - if (lower === "utf-32le") return Buffer.from([0xff, 0xfe, 0x00, 0x00]) - if (lower === "utf-32be") return Buffer.from([0x00, 0x00, 0xfe, 0xff]) - return Buffer.alloc(0) - } - function isUtf8(bytes: Buffer): boolean { try { new TextDecoder("utf-8", { fatal: true }).decode(bytes) @@ -180,4 +70,44 @@ export namespace Encoding { return false } } + + export function detect(bytes: Buffer): string { + if (bytes.length === 0) return DEFAULT + if (isUtf8(bytes)) return DEFAULT + const result = jschardet.detect(bytes) + if (!result.encoding) return DEFAULT + const enc = normalize(result.encoding) + // Reject unsupported Unicode encodings + if (enc.startsWith("utf-32")) return DEFAULT + if (!iconv.encodingExists(enc)) return DEFAULT + return enc + } + + export function decode(bytes: Buffer, encoding: string): string { + return iconv.decode(bytes, encoding) + } + + export function encode(text: string, encoding: string): Buffer { + return iconv.encode(text, encoding) + } + + /** Read a file, detecting its encoding. */ + export async function read(path: string): Promise<{ text: string; encoding: string }> { + const bytes = Buffer.from(await readFile(path)) + const encoding = detect(bytes) + return { text: decode(bytes, encoding), encoding } + } + + /** Synchronous read, detecting encoding. */ + export function readSync(path: string): { text: string; encoding: string } { + const bytes = readFileSync(path) + const encoding = detect(bytes) + return { text: decode(bytes, encoding), encoding } + } + + /** Write text, ensuring parent directory exists, using the given encoding. */ + export async function write(path: string, text: string, encoding: string = DEFAULT): Promise { + await mkdir(dirname(path), { recursive: true }) + await writeFile(path, encode(text, encoding)) + } } diff --git a/packages/opencode/src/kilocode/tool/encoded-io.ts b/packages/opencode/src/kilocode/tool/encoded-io.ts new file mode 100644 index 0000000000..dab6e26e25 --- /dev/null +++ b/packages/opencode/src/kilocode/tool/encoded-io.ts @@ -0,0 +1,15 @@ +// kilocode_change - new file +import { Effect } from "effect" +import { Encoding } from "../encoding" + +/** + * Effect wrappers around {@link Encoding.read} and {@link Encoding.write} so + * tool code can preserve file encoding without leaking Node/async boilerplate + * into each call site. + */ +export namespace EncodedIO { + export const read = (path: string) => Effect.promise(() => Encoding.read(path)) + + export const write = (path: string, text: string, encoding: string = Encoding.DEFAULT) => + Effect.promise(() => Encoding.write(path, text, encoding)) +} diff --git a/packages/opencode/src/patch/index.ts b/packages/opencode/src/patch/index.ts index 7e2b0496f6..c116df0f94 100644 --- a/packages/opencode/src/patch/index.ts +++ b/packages/opencode/src/patch/index.ts @@ -1,7 +1,6 @@ import z from "zod" import * as path from "path" import * as fs from "fs/promises" -import { readFileSync } from "fs" import { Log } from "../util/log" import { Encoding } from "../kilocode/encoding" // kilocode_change @@ -307,17 +306,17 @@ export namespace Patch { interface ApplyPatchFileUpdate { unified_diff: string content: string - encoding: Encoding.Info // kilocode_change + encoding: string // kilocode_change } export function deriveNewContentsFromChunks(filePath: string, chunks: UpdateFileChunk[]): ApplyPatchFileUpdate { // kilocode_change start - encoding-aware read let originalContent: string - let enc: Encoding.Info + let encoding: string try { const result = Encoding.readSync(filePath) originalContent = result.text - enc = result.info + encoding = result.encoding } catch (error) { throw new Error(`Failed to read file ${filePath}: ${error}`) } @@ -346,7 +345,7 @@ export namespace Patch { return { unified_diff: unifiedDiff, content: newContent, - encoding: enc, // kilocode_change + encoding, // kilocode_change } } @@ -531,17 +530,10 @@ export namespace Patch { const modified: string[] = [] const deleted: string[] = [] - // kilocode_change start - encoding-aware writes for (const hunk of hunks) { switch (hunk.type) { case "add": - // Create parent directories - const addDir = path.dirname(hunk.path) - if (addDir !== "." && addDir !== "/") { - await fs.mkdir(addDir, { recursive: true }) - } - - await Encoding.write(hunk.path, hunk.contents, Encoding.DEFAULT) + await Encoding.write(hunk.path, hunk.contents) // kilocode_change - encoding-aware write added.push(hunk.path) log.info(`Added file: ${hunk.path}`) break @@ -556,26 +548,20 @@ export namespace Patch { const fileUpdate = deriveNewContentsFromChunks(hunk.path, hunk.chunks) if (hunk.move_path) { - // Handle file move - const moveDir = path.dirname(hunk.move_path) - if (moveDir !== "." && moveDir !== "/") { - await fs.mkdir(moveDir, { recursive: true }) - } - + // kilocode_change start - encoding-aware move await Encoding.write(hunk.move_path, fileUpdate.content, fileUpdate.encoding) await fs.unlink(hunk.path) + // kilocode_change end modified.push(hunk.move_path) log.info(`Moved file: ${hunk.path} -> ${hunk.move_path}`) } else { - // Regular update - await Encoding.write(hunk.path, fileUpdate.content, fileUpdate.encoding) + await Encoding.write(hunk.path, fileUpdate.content, fileUpdate.encoding) // kilocode_change modified.push(hunk.path) log.info(`Updated file: ${hunk.path}`) } break } } - // kilocode_change end return { added, modified, deleted } } diff --git a/packages/opencode/src/tool/apply_patch.ts b/packages/opencode/src/tool/apply_patch.ts index 3535d96955..be1cbbb9f3 100644 --- a/packages/opencode/src/tool/apply_patch.ts +++ b/packages/opencode/src/tool/apply_patch.ts @@ -15,6 +15,7 @@ import DESCRIPTION from "./apply_patch.txt" import { File } from "../file" import { filterDiagnostics } from "./diagnostics" // kilocode_change import { ConfigValidation } from "../kilocode/config-validation" // kilocode_change +import { EncodedIO } from "../kilocode/tool/encoded-io" // kilocode_change import { Format } from "../format" const PatchParams = z.object({ @@ -61,6 +62,7 @@ export const ApplyPatchTool = Tool.define( diff: string additions: number deletions: number + encoding: string // kilocode_change - preserved per-file encoding }> = [] let totalDiff = "" @@ -91,6 +93,7 @@ export const ApplyPatchTool = Tool.define( diff, additions, deletions, + encoding: "utf-8", // kilocode_change - new files default to utf-8 }) totalDiff += diff + "\n" @@ -106,13 +109,18 @@ export const ApplyPatchTool = Tool.define( ) } - const oldContent = yield* afs.readFileString(filePath) + // kilocode_change start - preserve existing file encoding + const readResult = yield* EncodedIO.read(filePath) + const oldContent = readResult.text + let encoding = readResult.encoding let newContent = oldContent + // kilocode_change end // Apply the update chunks to get new content try { const fileUpdate = Patch.deriveNewContentsFromChunks(filePath, hunk.chunks) newContent = fileUpdate.content + encoding = fileUpdate.encoding // kilocode_change } catch (error) { return yield* Effect.fail(new Error(`apply_patch verification failed: ${error}`)) } @@ -138,6 +146,7 @@ export const ApplyPatchTool = Tool.define( diff, additions, deletions, + encoding, // kilocode_change }) totalDiff += diff + "\n" @@ -145,9 +154,12 @@ export const ApplyPatchTool = Tool.define( } case "delete": { - const contentToDelete = yield* afs - .readFileString(filePath) - .pipe(Effect.catch((error) => Effect.fail(new Error(`apply_patch verification failed: ${error}`)))) + // kilocode_change start - encoding-aware read + const deleteRead = yield* EncodedIO.read(filePath).pipe( + Effect.catch((error) => Effect.fail(new Error(`apply_patch verification failed: ${error}`))), + ) + const contentToDelete = deleteRead.text + // kilocode_change end const deleteDiff = trimDiff(createTwoFilesPatch(filePath, filePath, contentToDelete, "")) const deletions = contentToDelete.split("\n").length @@ -160,6 +172,7 @@ export const ApplyPatchTool = Tool.define( diff: deleteDiff, additions: 0, deletions, + encoding: deleteRead.encoding, // kilocode_change }) totalDiff += deleteDiff + "\n" @@ -199,22 +212,18 @@ export const ApplyPatchTool = Tool.define( const edited = change.type === "delete" ? undefined : (change.movePath ?? change.filePath) switch (change.type) { case "add": - // Create parent directories (recursive: true is safe on existing/root dirs) - - yield* afs.writeWithDirs(change.filePath, change.newContent) + yield* EncodedIO.write(change.filePath, change.newContent, change.encoding) // kilocode_change updates.push({ file: change.filePath, event: "add" }) break case "update": - yield* afs.writeWithDirs(change.filePath, change.newContent) + yield* EncodedIO.write(change.filePath, change.newContent, change.encoding) // kilocode_change updates.push({ file: change.filePath, event: "change" }) break case "move": if (change.movePath) { - // Create parent directories (recursive: true is safe on existing/root dirs) - - yield* afs.writeWithDirs(change.movePath!, change.newContent) + yield* EncodedIO.write(change.movePath!, change.newContent, change.encoding) // kilocode_change yield* afs.remove(change.filePath) updates.push({ file: change.filePath, event: "unlink" }) updates.push({ file: change.movePath, event: "add" }) diff --git a/packages/opencode/src/tool/edit.ts b/packages/opencode/src/tool/edit.ts index 5982e5bf6d..b89417d1b5 100644 --- a/packages/opencode/src/tool/edit.ts +++ b/packages/opencode/src/tool/edit.ts @@ -22,6 +22,7 @@ import { assertExternalDirectoryEffect } from "./external-directory" import { AppFileSystem } from "../filesystem" import { filterDiagnostics } from "./diagnostics" // kilocode_change import { ConfigValidation } from "../kilocode/config-validation" // kilocode_change +import { EncodedIO } from "../kilocode/tool/encoded-io" // kilocode_change const MAX_DIFF_CONTENT = 500_000 // kilocode_change @@ -100,7 +101,14 @@ export const EditTool = Tool.define( Effect.gen(function* () { if (params.oldString === "") { const existed = yield* afs.existsSafe(filePath) - if (existed) contentOld = yield* afs.readFileString(filePath) // kilocode_change + // kilocode_change start - preserve existing file encoding + let encoding = "utf-8" + if (existed) { + const encoded = yield* EncodedIO.read(filePath) + contentOld = encoded.text + encoding = encoded.encoding + } + // kilocode_change end contentNew = params.newString diff = trimDiff(createTwoFilesPatch(filePath, filePath, contentOld, contentNew)) cachedFilediff = buildFileDiff(filePath, contentOld, contentNew) // kilocode_change @@ -114,7 +122,7 @@ export const EditTool = Tool.define( filediff: cachedFilediff, // kilocode_change }, }) - yield* afs.writeWithDirs(filePath, params.newString) + yield* EncodedIO.write(filePath, params.newString, encoding) // kilocode_change yield* format.file(filePath) yield* bus.publish(File.Event.Edited, { file: filePath }) yield* bus.publish(FileWatcher.Event.Updated, { @@ -129,7 +137,11 @@ export const EditTool = Tool.define( if (!info) throw new Error(`File ${filePath} not found`) if (info.type === "Directory") throw new Error(`Path is a directory, not a file: ${filePath}`) yield* filetime.assert(ctx.sessionID, filePath) - contentOld = yield* afs.readFileString(filePath) + // kilocode_change start - preserve existing file encoding + const encoded = yield* EncodedIO.read(filePath) + contentOld = encoded.text + const encoding = encoded.encoding + // kilocode_change end const ending = detectLineEnding(contentOld) const old = convertToLineEnding(normalizeLineEndings(params.oldString), ending) @@ -157,14 +169,14 @@ export const EditTool = Tool.define( }, }) - yield* afs.writeWithDirs(filePath, contentNew) + yield* EncodedIO.write(filePath, contentNew, encoding) // kilocode_change yield* format.file(filePath) yield* bus.publish(File.Event.Edited, { file: filePath }) yield* bus.publish(FileWatcher.Event.Updated, { file: filePath, event: "change", }) - contentNew = yield* afs.readFileString(filePath) + contentNew = (yield* EncodedIO.read(filePath)).text // kilocode_change diff = trimDiff( createTwoFilesPatch( filePath, diff --git a/packages/opencode/src/tool/read.ts b/packages/opencode/src/tool/read.ts index a6a17c709b..e01f87af73 100644 --- a/packages/opencode/src/tool/read.ts +++ b/packages/opencode/src/tool/read.ts @@ -318,15 +318,12 @@ export async function isBinaryFile(filepath: string, fileSize: number): Promise< if (result.bytesRead === 0) return false // kilocode_change start - encoding-aware binary detection + // If encoding detection identifies a known text encoding (including UTF-16 + // LE/BE with BOM or CJK), it's text — not binary. This prevents UTF-16 + // files with legitimate null bytes from being falsely rejected. const sample = bytes.subarray(0, result.bytesRead) - // If encoding detection recognizes a known text encoding, it's not binary. - // This prevents UTF-16 (with null bytes) and CJK encodings from being - // falsely flagged as binary. - const info = Encoding.detect(sample) - if (info.encoding !== "iso-8859-1" && info.encoding !== "utf-8") { - // jschardet confidently identified a non-default encoding → text file - return false - } + const enc = Encoding.detect(sample) + if (enc !== "utf-8") return false let nonPrintableCount = 0 for (let i = 0; i < result.bytesRead; i++) { diff --git a/packages/opencode/src/tool/write.ts b/packages/opencode/src/tool/write.ts index f12b32d308..289d09c872 100644 --- a/packages/opencode/src/tool/write.ts +++ b/packages/opencode/src/tool/write.ts @@ -16,6 +16,7 @@ import { trimDiff, buildFileDiff } from "./edit" // kilocode_change import { assertExternalDirectoryEffect } from "./external-directory" import { filterDiagnostics } from "./diagnostics" // kilocode_change import { ConfigValidation } from "../kilocode/config-validation" // kilocode_change +import { EncodedIO } from "../kilocode/tool/encoded-io" // kilocode_change const MAX_PROJECT_DIAGNOSTICS_FILES = 5 @@ -42,7 +43,15 @@ export const WriteTool = Tool.define( yield* assertExternalDirectoryEffect(ctx, filepath) const exists = yield* fs.existsSafe(filepath) - const contentOld = exists ? yield* fs.readFileString(filepath) : "" + // kilocode_change start - preserve existing file encoding + let contentOld = "" + let encoding = "utf-8" + if (exists) { + const encoded = yield* EncodedIO.read(filepath) + contentOld = encoded.text + encoding = encoded.encoding + } + // kilocode_change end if (exists) yield* filetime.assert(ctx.sessionID, filepath) const diff = trimDiff(createTwoFilesPatch(filepath, filepath, contentOld, params.content)) @@ -58,7 +67,7 @@ export const WriteTool = Tool.define( }, }) - yield* fs.writeWithDirs(filepath, params.content) + yield* EncodedIO.write(filepath, params.content, encoding) // kilocode_change yield* format.file(filepath) yield* bus.publish(File.Event.Edited, { file: filepath }) yield* bus.publish(FileWatcher.Event.Updated, { diff --git a/packages/opencode/src/util/filesystem.ts b/packages/opencode/src/util/filesystem.ts index 02a7bf6edf..5f50231b03 100644 --- a/packages/opencode/src/util/filesystem.ts +++ b/packages/opencode/src/util/filesystem.ts @@ -6,7 +6,6 @@ import { dirname, join, relative, resolve as pathResolve, win32 } from "path" import { Readable } from "stream" import { pipeline } from "stream/promises" import { Glob } from "./glob" -import { Encoding } from "../kilocode/encoding" // kilocode_change export namespace Filesystem { // Fast sync version for metadata checks @@ -50,41 +49,6 @@ export namespace Filesystem { return readFile(p) } - // kilocode_change start - encoding-aware read/write for tool file operations - export async function readEncoded(p: string): Promise<{ text: string; encoding: Encoding.Info }> { - const bytes = Buffer.from(await readFile(p)) - const info = Encoding.detect(bytes) - return { text: Encoding.decode(bytes, info), encoding: info } - } - - export async function writeEncoded( - p: string, - content: string, - encoding: Encoding.Info, - mode?: number, - ): Promise { - const bytes = Encoding.encode(content, encoding) - try { - if (mode) { - await writeFile(p, bytes, { mode }) - } else { - await writeFile(p, bytes) - } - } catch (e) { - if (isEnoent(e)) { - await mkdir(dirname(p), { recursive: true }) - if (mode) { - await writeFile(p, bytes, { mode }) - } else { - await writeFile(p, bytes) - } - return - } - throw e - } - } - // kilocode_change end - export async function readArrayBuffer(p: string): Promise { const buf = await readFile(p) return buf.buffer.slice(buf.byteOffset, buf.byteOffset + buf.byteLength) as ArrayBuffer diff --git a/packages/opencode/test/kilocode/encoding.test.ts b/packages/opencode/test/kilocode/encoding.test.ts deleted file mode 100644 index ae9dd6aa45..0000000000 --- a/packages/opencode/test/kilocode/encoding.test.ts +++ /dev/null @@ -1,340 +0,0 @@ -import { test, expect, describe } from "bun:test" -import { Encoding } from "../../src/kilocode/encoding" -import { tmpdir } from "../fixture/fixture" -import path from "path" -import fs from "fs/promises" -import iconv from "iconv-lite" - -describe("Encoding", () => { - describe("detect", () => { - test("detects UTF-8 BOM", () => { - const bytes = Buffer.concat([Buffer.from([0xef, 0xbb, 0xbf]), Buffer.from("hello", "utf-8")]) - const info = Encoding.detect(bytes) - expect(info.encoding).toBe("utf-8") - expect(info.bom).toBe(true) - }) - - test("detects UTF-16 LE BOM", () => { - const bytes = Buffer.concat([Buffer.from([0xff, 0xfe]), Buffer.from("hello", "utf16le")]) - const info = Encoding.detect(bytes) - expect(info.encoding).toBe("utf-16le") - expect(info.bom).toBe(true) - }) - - test("detects UTF-16 BE BOM", () => { - const bytes = Buffer.concat([Buffer.from([0xfe, 0xff]), iconv.encode("hello", "utf-16be")]) - const info = Encoding.detect(bytes) - expect(info.encoding).toBe("utf-16be") - expect(info.bom).toBe(true) - }) - - test("detects plain UTF-8 (no BOM)", () => { - const bytes = Buffer.from("hello world", "utf-8") - const info = Encoding.detect(bytes) - expect(info.encoding).toBe("utf-8") - expect(info.bom).toBe(false) - }) - - test("returns utf-8 for empty buffer", () => { - const info = Encoding.detect(Buffer.alloc(0)) - expect(info.encoding).toBe("utf-8") - expect(info.bom).toBe(false) - }) - - test("detects UTF-32 LE BOM", () => { - const bytes = Buffer.from([0xff, 0xfe, 0x00, 0x00, 0x41, 0x00, 0x00, 0x00]) - const info = Encoding.detect(bytes) - expect(info.encoding).toBe("utf-32le") - expect(info.bom).toBe(true) - }) - - test("detects UTF-32 BE BOM", () => { - const bytes = Buffer.from([0x00, 0x00, 0xfe, 0xff, 0x00, 0x00, 0x00, 0x41]) - const info = Encoding.detect(bytes) - expect(info.encoding).toBe("utf-32be") - expect(info.bom).toBe(true) - }) - }) - - describe("decode", () => { - test("decodes UTF-8 with BOM", () => { - const bytes = Buffer.concat([Buffer.from([0xef, 0xbb, 0xbf]), Buffer.from("hello", "utf-8")]) - const text = Encoding.decode(bytes, { encoding: "utf-8", bom: true }) - expect(text).toBe("hello") - }) - - test("decodes UTF-8 without BOM", () => { - const bytes = Buffer.from("hello", "utf-8") - const text = Encoding.decode(bytes, { encoding: "utf-8", bom: false }) - expect(text).toBe("hello") - }) - - test("decodes UTF-16 LE with BOM", () => { - const bytes = Buffer.concat([Buffer.from([0xff, 0xfe]), Buffer.from("hello", "utf16le")]) - const text = Encoding.decode(bytes, { encoding: "utf-16le", bom: true }) - expect(text).toBe("hello") - }) - - test("decodes UTF-16 BE with BOM", () => { - const bytes = Buffer.concat([Buffer.from([0xfe, 0xff]), iconv.encode("hello", "utf-16be")]) - const text = Encoding.decode(bytes, { encoding: "utf-16be", bom: true }) - expect(text).toBe("hello") - }) - - test("decodes Shift-JIS", () => { - const original = "こんにちは" - const bytes = iconv.encode(original, "Shift_JIS") - const text = Encoding.decode(bytes, { encoding: "Shift_JIS", bom: false }) - expect(text).toBe(original) - }) - - test("decodes GB2312", () => { - const original = "你好世界" - const bytes = iconv.encode(original, "gb2312") - const text = Encoding.decode(bytes, { encoding: "gb2312", bom: false }) - expect(text).toBe(original) - }) - - test("decodes EUC-KR", () => { - const original = "안녕하세요" - const bytes = iconv.encode(original, "euc-kr") - const text = Encoding.decode(bytes, { encoding: "euc-kr", bom: false }) - expect(text).toBe(original) - }) - }) - - describe("encode", () => { - test("encodes UTF-8 without BOM", () => { - const bytes = Encoding.encode("hello", { encoding: "utf-8", bom: false }) - expect(bytes).toEqual(Buffer.from("hello", "utf-8")) - }) - - test("encodes UTF-8 with BOM", () => { - const bytes = Encoding.encode("hello", { encoding: "utf-8", bom: true }) - expect(bytes[0]).toBe(0xef) - expect(bytes[1]).toBe(0xbb) - expect(bytes[2]).toBe(0xbf) - expect(bytes.subarray(3).toString("utf-8")).toBe("hello") - }) - - test("encodes UTF-16 LE with BOM", () => { - const bytes = Encoding.encode("hi", { encoding: "utf-16le", bom: true }) - expect(bytes[0]).toBe(0xff) - expect(bytes[1]).toBe(0xfe) - expect(bytes.subarray(2).toString("utf16le")).toBe("hi") - }) - - test("encodes Shift-JIS", () => { - const text = "こんにちは" - const bytes = Encoding.encode(text, { encoding: "Shift_JIS", bom: false }) - const expected = iconv.encode(text, "Shift_JIS") - expect(bytes).toEqual(expected) - }) - }) - - describe("round-trip", () => { - test("UTF-8 with BOM round-trips", () => { - const original = Buffer.concat([Buffer.from([0xef, 0xbb, 0xbf]), Buffer.from("hello world\n", "utf-8")]) - const info = Encoding.detect(original) - const text = Encoding.decode(original, info) - const result = Encoding.encode(text, info) - expect(result).toEqual(original) - }) - - test("UTF-16 LE with BOM round-trips", () => { - const original = Buffer.concat([Buffer.from([0xff, 0xfe]), Buffer.from("hello world\n", "utf16le")]) - const info = Encoding.detect(original) - const text = Encoding.decode(original, info) - const result = Encoding.encode(text, info) - expect(result).toEqual(original) - }) - - test("plain UTF-8 round-trips", () => { - const original = Buffer.from("hello world\n", "utf-8") - const info = Encoding.detect(original) - const text = Encoding.decode(original, info) - const result = Encoding.encode(text, info) - expect(result).toEqual(original) - }) - - test("Shift-JIS round-trips", () => { - const text = "日本語テスト\n" - const original = iconv.encode(text, "Shift_JIS") - const info = Encoding.detect(original) - expect(info.encoding).toBe("Shift_JIS") - const decoded = Encoding.decode(original, info) - expect(decoded).toBe(text) - const result = Encoding.encode(decoded, info) - expect(result).toEqual(original) - }) - - test("EUC-JP round-trips", () => { - const text = "日本語テスト\n" - const original = iconv.encode(text, "euc-jp") - const info = Encoding.detect(original) - expect(info.encoding).toBe("euc-jp") - const decoded = Encoding.decode(original, info) - expect(decoded).toBe(text) - const result = Encoding.encode(decoded, info) - expect(result).toEqual(original) - }) - - test("Big5 round-trips", () => { - // Longer sample required for reliable statistical detection - const text = "次常用國字標準字體表建議使用正體中文排版系統進行文件處理以維護傳統漢字文化\n" - const original = iconv.encode(text, "big5") - const info = Encoding.detect(original) - expect(info.encoding).toBe("big5") - const decoded = Encoding.decode(original, info) - expect(decoded).toBe(text) - const result = Encoding.encode(decoded, info) - expect(result).toEqual(original) - }) - - test("GB2312 round-trips", () => { - // Longer sample required for reliable statistical detection - const text = "你好世界测试文件内容这是一个很长的中文文本用于测试编码检测功能\n第二行也有中文内容\n" - const original = iconv.encode(text, "gb2312") - const info = Encoding.detect(original) - // jschardet detects as GB2312 which normalizes the same - const decoded = Encoding.decode(original, info) - expect(decoded).toBe(text) - const result = Encoding.encode(decoded, info) - expect(result).toEqual(original) - }) - - test("EUC-KR round-trips", () => { - // Longer sample for reliable detection - const text = "안녕하세요 세계 프로그래밍 테스트 문자열입니다\n두번째 줄도 있습니다\n" - const original = iconv.encode(text, "euc-kr") - const info = Encoding.detect(original) - expect(info.encoding).toBe("euc-kr") - const decoded = Encoding.decode(original, info) - expect(decoded).toBe(text) - const result = Encoding.encode(decoded, info) - expect(result).toEqual(original) - }) - - test("Windows-1251 (Cyrillic) round-trips", () => { - const text = "Привет мир\n" - const original = iconv.encode(text, "windows-1251") - const info = Encoding.detect(original) - expect(info.encoding).toBe("windows-1251") - const decoded = Encoding.decode(original, info) - expect(decoded).toBe(text) - const result = Encoding.encode(decoded, info) - expect(result).toEqual(original) - }) - - test("KOI8-R (Russian) round-trips", () => { - const text = "Привет мир\n" - const original = iconv.encode(text, "koi8-r") - const info = Encoding.detect(original) - // jschardet detects KOI8-R for this content - const decoded = Encoding.decode(original, info) - expect(decoded).toBe(text) - const result = Encoding.encode(decoded, info) - expect(result).toEqual(original) - }) - }) - - describe("file read/write", () => { - test("preserves UTF-8 BOM through file write/read cycle", async () => { - await using tmp = await tmpdir() - const file = path.join(tmp.path, "bom.txt") - const original = Buffer.concat([Buffer.from([0xef, 0xbb, 0xbf]), Buffer.from("line one\nline two\n", "utf-8")]) - await fs.writeFile(file, original) - - const { text, info } = await Encoding.read(file) - expect(info.encoding).toBe("utf-8") - expect(info.bom).toBe(true) - expect(text).toBe("line one\nline two\n") - - const modified = text.replace("one", "1") - await Encoding.write(file, modified, info) - - const raw = await fs.readFile(file) - expect(raw[0]).toBe(0xef) - expect(raw[1]).toBe(0xbb) - expect(raw[2]).toBe(0xbf) - expect(raw.subarray(3).toString("utf-8")).toBe("line 1\nline two\n") - }) - - test("preserves UTF-16 LE BOM through file write/read cycle", async () => { - await using tmp = await tmpdir() - const file = path.join(tmp.path, "utf16le.txt") - const content = "hello world\n" - const original = Buffer.concat([Buffer.from([0xff, 0xfe]), Buffer.from(content, "utf16le")]) - await fs.writeFile(file, original) - - const { text, info } = await Encoding.read(file) - expect(info.encoding).toBe("utf-16le") - expect(info.bom).toBe(true) - expect(text).toBe("hello world\n") - - await Encoding.write(file, text, info) - - const raw = await fs.readFile(file) - expect(raw).toEqual(original) - }) - - test("preserves Shift-JIS through file write/read cycle", async () => { - await using tmp = await tmpdir() - const file = path.join(tmp.path, "shiftjis.txt") - const text = "日本語テスト\nconst x = 1;\n" - const original = iconv.encode(text, "Shift_JIS") - await fs.writeFile(file, original) - - const { text: decoded, info } = await Encoding.read(file) - expect(info.encoding).toBe("Shift_JIS") - expect(decoded).toBe(text) - - // Modify ASCII part and write back - const modified = decoded.replace("const x = 1", "const x = 2") - await Encoding.write(file, modified, info) - - const raw = await fs.readFile(file) - const expected = iconv.encode(modified, "Shift_JIS") - expect(raw).toEqual(expected) - }) - - test("preserves Big5 through file write/read cycle", async () => { - await using tmp = await tmpdir() - const file = path.join(tmp.path, "big5.txt") - const text = "次常用國字標準字體表建議使用正體中文排版系統進行文件處理\n第二行正體中文\n" - const original = iconv.encode(text, "big5") - await fs.writeFile(file, original) - - const { text: decoded, info } = await Encoding.read(file) - expect(info.encoding).toBe("big5") - expect(decoded).toBe(text) - - await Encoding.write(file, decoded, info) - - const raw = await fs.readFile(file) - expect(raw).toEqual(original) - }) - - test("readSync works for UTF-8 BOM files", async () => { - await using tmp = await tmpdir() - const file = path.join(tmp.path, "sync.txt") - const original = Buffer.concat([Buffer.from([0xef, 0xbb, 0xbf]), Buffer.from("sync test", "utf-8")]) - await fs.writeFile(file, original) - - const { text, info } = Encoding.readSync(file) - expect(info.encoding).toBe("utf-8") - expect(info.bom).toBe(true) - expect(text).toBe("sync test") - }) - - test("creates parent directories when writing", async () => { - await using tmp = await tmpdir() - const file = path.join(tmp.path, "nested", "dir", "file.txt") - - await Encoding.write(file, "content", Encoding.DEFAULT) - - const raw = await fs.readFile(file, "utf-8") - expect(raw).toBe("content") - }) - }) -}) From 526f6807ffacfd0729e826ba9969383ab9cd5c0a Mon Sep 17 00:00:00 2001 From: "kiloconnect[bot]" <240665456+kiloconnect[bot]@users.noreply.github.com> Date: Tue, 21 Apr 2026 12:40:56 +0000 Subject: [PATCH 10/24] test(cli): add integration tests for tool encoding preservation Exercise Read/Write/Edit/ApplyPatch tools directly against files in UTF-8, UTF-16 LE/BE (with BOM), Shift_JIS, EUC-JP, GB2312, Big5, EUC-KR, Windows-1251, and KOI8-R to verify the tools decode input correctly and round-trip bytes back to the original encoding. Also ensure UTF-16 BOMs survive the write pipeline. --- packages/opencode/src/kilocode/encoding.ts | 6 + .../test/kilocode/tool-encoding.test.ts | 306 ++++++++++++++++++ 2 files changed, 312 insertions(+) create mode 100644 packages/opencode/test/kilocode/tool-encoding.test.ts diff --git a/packages/opencode/src/kilocode/encoding.ts b/packages/opencode/src/kilocode/encoding.ts index b40aecc0c4..eed7b00400 100644 --- a/packages/opencode/src/kilocode/encoding.ts +++ b/packages/opencode/src/kilocode/encoding.ts @@ -88,6 +88,12 @@ export namespace Encoding { } export function encode(text: string, encoding: string): Buffer { + // iconv-lite's utf-16le/utf-16be do not emit a BOM, but UTF-16 without a + // BOM is unsupported in this codebase. Prepend the appropriate BOM so the + // next detection pass can still recognise the encoding. + const lower = encoding.toLowerCase() + if (lower === "utf-16le") return Buffer.concat([Buffer.from([0xff, 0xfe]), iconv.encode(text, encoding)]) + if (lower === "utf-16be") return Buffer.concat([Buffer.from([0xfe, 0xff]), iconv.encode(text, encoding)]) return iconv.encode(text, encoding) } diff --git a/packages/opencode/test/kilocode/tool-encoding.test.ts b/packages/opencode/test/kilocode/tool-encoding.test.ts new file mode 100644 index 0000000000..b9c318c777 --- /dev/null +++ b/packages/opencode/test/kilocode/tool-encoding.test.ts @@ -0,0 +1,306 @@ +// kilocode_change - new file +// Integration tests verifying that the agent file tools (read, write, edit, +// apply_patch) detect and preserve the original encoding of files on disk. +// Tests exercise the real tool pipeline rather than the Encoding helper +// directly so we validate end-to-end behaviour. + +import { afterEach, describe, expect } from "bun:test" +import { Effect, Layer } from "effect" +import path from "path" +import fs from "fs/promises" +import iconv from "iconv-lite" +import { Agent } from "../../src/agent/agent" +import { AppFileSystem } from "../../src/filesystem" +import { ApplyPatchTool } from "../../src/tool/apply_patch" +import { Bus } from "../../src/bus" +import * as CrossSpawnSpawner from "../../src/effect/cross-spawn-spawner" +import { EditTool } from "../../src/tool/edit" +import { FileTime } from "../../src/file/time" +import { Format } from "../../src/format" +import { Instance } from "../../src/project/instance" +import { Instruction } from "../../src/session/instruction" +import { LSP } from "../../src/lsp" +import { MessageID, SessionID } from "../../src/session/schema" +import { ReadTool } from "../../src/tool/read" +import { Tool } from "../../src/tool/tool" +import { Truncate } from "../../src/tool/truncate" +import { WriteTool } from "../../src/tool/write" +import { provideTmpdirInstance } from "../fixture/fixture" +import { testEffect } from "../lib/effect" + +const ctx = { + sessionID: SessionID.make("ses_test-encoding"), + messageID: MessageID.make(""), + callID: "", + agent: "build", + abort: AbortSignal.any([]), + messages: [], + metadata: () => Effect.void, + ask: () => Effect.void, +} + +afterEach(async () => { + await Instance.disposeAll() +}) + +const it = testEffect( + Layer.mergeAll( + Agent.defaultLayer, + AppFileSystem.defaultLayer, + CrossSpawnSpawner.defaultLayer, + FileTime.defaultLayer, + Instruction.defaultLayer, + LSP.defaultLayer, + Bus.layer, + Format.defaultLayer, + Truncate.defaultLayer, + ), +) + +const runRead = (args: Tool.InferParameters) => + Effect.gen(function* () { + const info = yield* ReadTool + const tool = yield* info.init() + return yield* tool.execute(args, ctx) + }) + +const runWrite = (args: Tool.InferParameters) => + Effect.gen(function* () { + const info = yield* WriteTool + const tool = yield* info.init() + return yield* tool.execute(args, ctx) + }) + +const runEdit = (args: Tool.InferParameters) => + Effect.gen(function* () { + const info = yield* EditTool + const tool = yield* info.init() + return yield* tool.execute(args, ctx) + }) + +const runPatch = (args: Tool.InferParameters) => + Effect.gen(function* () { + const info = yield* ApplyPatchTool + const tool = yield* info.init() + return yield* tool.execute(args, ctx) + }) + +const markRead = (filepath: string) => + Effect.gen(function* () { + const ft = yield* FileTime.Service + yield* ft.read(ctx.sessionID, filepath) + }) + +// iconv-lite's utf-16le/utf-16be do not emit a BOM on their own, but this +// codebase only supports UTF-16 with BOM. Prepend one for fixture files. +const encodeBytes = (text: string, encoding: string): Buffer => { + const lower = encoding.toLowerCase() + if (lower === "utf-16le") return Buffer.concat([Buffer.from([0xff, 0xfe]), iconv.encode(text, encoding)]) + if (lower === "utf-16be") return Buffer.concat([Buffer.from([0xfe, 0xff]), iconv.encode(text, encoding)]) + return iconv.encode(text, encoding) +} + +// Create a file with the given encoding by writing raw bytes. +const putEncoded = (filepath: string, text: string, encoding: string) => + Effect.promise(async () => { + await fs.mkdir(path.dirname(filepath), { recursive: true }) + await fs.writeFile(filepath, encodeBytes(text, encoding)) + }) + +const loadDecoded = (filepath: string, encoding: string) => + Effect.promise(async () => { + const bytes = await fs.readFile(filepath) + return iconv.decode(bytes, encoding) + }) + +const loadBytes = (filepath: string) => Effect.promise(() => fs.readFile(filepath)) + +// Sample phrases chosen to exercise each encoding's characteristic byte patterns. +const samples = { + utf8: "Hello, world! — £100", + shiftJis: "こんにちは、世界！日本語のテストです。", + eucJp: "日本語のEUC-JPテスト文字列です。", + gb2312: "你好，世界！这是简体中文测试。", + big5: "你好，世界！這是繁體中文測試。", + eucKr: "안녕하세요, 세계! 한국어 테스트입니다.", + windows1251: "Привет, мир! Это тест кириллицы.", + koi8r: "Привет, мир! КОИ-8 Р тест.", + latin1: "Caf\u00e9 na\u00efve r\u00e9sum\u00e9 — \u00a3100", +} + +describe("tool encoding preservation", () => { + describe("ReadTool decodes files with non-UTF-8 encodings", () => { + const cases: Array<[string, string, string]> = [ + ["UTF-8", "utf-8", samples.utf8], + ["UTF-16 LE with BOM", "utf-16le", samples.utf8], + ["UTF-16 BE with BOM", "utf-16be", samples.utf8], + ["Shift_JIS", "Shift_JIS", samples.shiftJis], + ["EUC-JP", "euc-jp", samples.eucJp], + ["GB2312", "gb2312", samples.gb2312], + ["Big5", "big5", samples.big5], + ["EUC-KR", "euc-kr", samples.eucKr], + ["Windows-1251", "windows-1251", samples.windows1251], + ["KOI8-R", "koi8-r", samples.koi8r], + ] + + for (const [label, encoding, text] of cases) { + it.live(`decodes ${label} content for the model`, () => + providEncoded(encoding, text, (filepath) => + Effect.gen(function* () { + const result = yield* runRead({ filePath: filepath }) + expect(result.output).toContain(text) + }), + ), + ) + } + }) + + describe("ReadTool does not flag non-Latin text files as binary", () => { + it.live("accepts Shift_JIS", () => + providEncoded("Shift_JIS", samples.shiftJis, (filepath) => + Effect.gen(function* () { + const result = yield* runRead({ filePath: filepath }) + expect(result.output).toContain(samples.shiftJis) + }), + ), + ) + + it.live("accepts UTF-16 LE with BOM (contains NUL bytes)", () => + providEncoded("utf-16le", samples.utf8, (filepath) => + Effect.gen(function* () { + const result = yield* runRead({ filePath: filepath }) + expect(result.output).toContain(samples.utf8) + }), + ), + ) + }) + + describe("WriteTool preserves existing file encoding when overwriting", () => { + const cases: Array<[string, string, string]> = [ + ["Shift_JIS", "Shift_JIS", samples.shiftJis], + ["GB2312", "gb2312", samples.gb2312], + ["Windows-1251", "windows-1251", samples.windows1251], + ["UTF-16 LE", "utf-16le", samples.utf8], + ] + + for (const [label, encoding, original] of cases) { + it.live(`preserves ${label} encoding on overwrite`, () => + provideTmpdirInstance((dir) => + Effect.gen(function* () { + const filepath = path.join(dir, "file.txt") + yield* putEncoded(filepath, original, encoding) + yield* markRead(filepath) + + const replacement = original + " updated" + yield* runWrite({ filePath: filepath, content: replacement }) + + const decoded = yield* loadDecoded(filepath, encoding) + expect(decoded).toBe(replacement) + + // Bytes should still match the original encoding (and differ from UTF-8). + const bytes = yield* loadBytes(filepath) + expect(bytes.equals(encodeBytes(replacement, encoding))).toBe(true) + }), + ), + ) + } + + it.live("defaults new files to UTF-8", () => + provideTmpdirInstance((dir) => + Effect.gen(function* () { + const filepath = path.join(dir, "new.txt") + yield* runWrite({ filePath: filepath, content: samples.utf8 }) + + const bytes = yield* loadBytes(filepath) + expect(bytes.equals(Buffer.from(samples.utf8, "utf-8"))).toBe(true) + }), + ), + ) + }) + + describe("EditTool preserves existing file encoding across edits", () => { + const cases: Array<[string, string, string, string, string]> = [ + ["Shift_JIS", "Shift_JIS", samples.shiftJis, "日本語", "ニホンゴ"], + ["GB2312", "gb2312", samples.gb2312, "简体中文", "中文简体"], + ["Windows-1251", "windows-1251", samples.windows1251, "мир", "планета"], + ["UTF-16 LE", "utf-16le", samples.utf8 + "\n second line", "world", "earth"], + ] + + for (const [label, encoding, original, oldString, newString] of cases) { + it.live(`preserves ${label} through edit`, () => + provideTmpdirInstance((dir) => + Effect.gen(function* () { + const filepath = path.join(dir, "doc.txt") + yield* putEncoded(filepath, original, encoding) + yield* markRead(filepath) + + yield* runEdit({ filePath: filepath, oldString, newString }) + + const decoded = yield* loadDecoded(filepath, encoding) + const expected = original.replace(oldString, newString) + expect(decoded).toBe(expected) + + const bytes = yield* loadBytes(filepath) + expect(bytes.equals(encodeBytes(expected, encoding))).toBe(true) + }), + ), + ) + } + }) + + describe("ApplyPatchTool preserves encoding", () => { + it.live("preserves Shift_JIS through an update hunk", () => + provideTmpdirInstance((dir) => + Effect.gen(function* () { + const filepath = path.join(dir, "doc.txt") + const original = "line1\n" + samples.shiftJis + "\nline3\n" + yield* putEncoded(filepath, original, "Shift_JIS") + + const patch = [ + "*** Begin Patch", + "*** Update File: doc.txt", + "@@", + " line1", + "-" + samples.shiftJis, + "+" + samples.eucJp.replace(/[^\u3000-\u30ff\u4e00-\u9fff]/g, ""), + " line3", + "*** End Patch", + ].join("\n") + + yield* runPatch({ patchText: patch }) + + const decoded = yield* loadDecoded(filepath, "Shift_JIS") + expect(decoded).toContain("line1") + expect(decoded).toContain("line3") + + // File must still decode as Shift_JIS — UTF-8 bytes would mojibake here. + const bytes = yield* loadBytes(filepath) + expect(bytes.includes(Buffer.from([0xe3, 0x81]))).toBe(false) + }), + ), + ) + + it.live("new files added via apply_patch are UTF-8", () => + provideTmpdirInstance((dir) => + Effect.gen(function* () { + const patch = ["*** Begin Patch", "*** Add File: new.txt", "+hello world", "*** End Patch"].join("\n") + yield* runPatch({ patchText: patch }) + const bytes = yield* loadBytes(path.join(dir, "new.txt")) + expect(bytes.equals(Buffer.from("hello world\n", "utf-8"))).toBe(true) + }), + ), + ) + }) +}) + +// Shared helper to set up a temp instance with an encoded file at `file.txt`. +function providEncoded(encoding: string, text: string, body: (filepath: string) => Effect.Effect) { + return provideTmpdirInstance((dir) => + Effect.gen(function* () { + const filepath = path.join(dir, "file.txt") + yield* putEncoded(filepath, text, encoding) + yield* markRead(filepath) + return yield* body(filepath) + }), + ) +} From 010a94698e449bdd9270f44e53aa209dd4c7a248 Mon Sep 17 00:00:00 2001 From: "kiloconnect[bot]" <240665456+kiloconnect[bot]@users.noreply.github.com> Date: Tue, 21 Apr 2026 12:41:26 +0000 Subject: [PATCH 11/24] chore: add changeset for encoding preservation --- .changeset/preserve-file-encoding.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 .changeset/preserve-file-encoding.md diff --git a/.changeset/preserve-file-encoding.md b/.changeset/preserve-file-encoding.md new file mode 100644 index 0000000000..a454443b56 --- /dev/null +++ b/.changeset/preserve-file-encoding.md @@ -0,0 +1,16 @@ +--- +"@kilocode/cli": minor +--- + +Preserve the original text encoding when reading and editing files. The read, edit, write, and apply_patch tools now detect each file's encoding and write it back unchanged, so non-UTF-8 source trees no longer get corrupted when the agent touches them. + +Supported: + +- UTF-8 +- UTF-16 with BOM +- Legacy Latin and CJK encodings (Shift_JIS, EUC-JP, GB2312, Big5, EUC-KR, Windows-1251, KOI8-R, ISO-8859 family, and others) + +Not supported: + +- UTF-16 without BOM +- UTF-32 From ce80c09356d00d9bff440cbd05d237f440faf014 Mon Sep 17 00:00:00 2001 From: "kiloconnect[bot]" <240665456+kiloconnect[bot]@users.noreply.github.com> Date: Tue, 21 Apr 2026 12:54:26 +0000 Subject: [PATCH 12/24] wip --- .kilo/package-lock.json | 31 +++++++++++++++++++++++++++++++ .kilocode/package-lock.json | 31 +++++++++++++++++++++++++++++++ 2 files changed, 62 insertions(+) create mode 100644 .kilo/package-lock.json create mode 100644 .kilocode/package-lock.json diff --git a/.kilo/package-lock.json b/.kilo/package-lock.json new file mode 100644 index 0000000000..7013f0cbd7 --- /dev/null +++ b/.kilo/package-lock.json @@ -0,0 +1,31 @@ +{ + "name": ".kilo", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "dependencies": { + "@kilocode/plugin": "*" + } + }, + "node_modules/@kilocode/plugin": { + "version": "7.1.23", + "license": "MIT", + "dependencies": { + "@kilocode/sdk": "7.1.23", + "zod": "4.1.8" + } + }, + "node_modules/@kilocode/sdk": { + "version": "7.1.23", + "license": "MIT" + }, + "node_modules/zod": { + "version": "4.1.8", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/colinhacks" + } + } + } +} diff --git a/.kilocode/package-lock.json b/.kilocode/package-lock.json new file mode 100644 index 0000000000..a83a334755 --- /dev/null +++ b/.kilocode/package-lock.json @@ -0,0 +1,31 @@ +{ + "name": ".kilocode", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "dependencies": { + "@kilocode/plugin": "*" + } + }, + "node_modules/@kilocode/plugin": { + "version": "7.1.23", + "license": "MIT", + "dependencies": { + "@kilocode/sdk": "7.1.23", + "zod": "4.1.8" + } + }, + "node_modules/@kilocode/sdk": { + "version": "7.1.23", + "license": "MIT" + }, + "node_modules/zod": { + "version": "4.1.8", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/colinhacks" + } + } + } +} From a26b4c34b26311cc8dd9068cd14d60f5eb2bad7d Mon Sep 17 00:00:00 2001 From: "kiloconnect[bot]" <240665456+kiloconnect[bot]@users.noreply.github.com> Date: Tue, 21 Apr 2026 13:01:42 +0000 Subject: [PATCH 13/24] Revert "Add test build notice" This reverts commit 227af07f29fe6a468bba3301f850bbc3a53f05eb. --- .../src/components/settings/AboutKiloCodeTab.tsx | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/packages/kilo-vscode/webview-ui/src/components/settings/AboutKiloCodeTab.tsx b/packages/kilo-vscode/webview-ui/src/components/settings/AboutKiloCodeTab.tsx index f19c08c3fa..abd9e2943d 100644 --- a/packages/kilo-vscode/webview-ui/src/components/settings/AboutKiloCodeTab.tsx +++ b/packages/kilo-vscode/webview-ui/src/components/settings/AboutKiloCodeTab.tsx @@ -170,22 +170,6 @@ const AboutKiloCodeTab: Component = (props) => { return (
- {/* Test Build Notice */} -
- This is a test build with experimental text encoding detection. Tested with EUC, GBR, Shift-JIS. -
- {/* Version Information */}

{language.t("settings.aboutKiloCode.versionInfo")}

From 25bec4bf2b9c1cd521209cace11b944eda6169e5 Mon Sep 17 00:00:00 2001 From: "kiloconnect[bot]" <240665456+kiloconnect[bot]@users.noreply.github.com> Date: Tue, 21 Apr 2026 13:01:47 +0000 Subject: [PATCH 14/24] Revert "wip" This reverts commit ce80c09356d00d9bff440cbd05d237f440faf014. --- .kilo/package-lock.json | 31 ------------------------------- .kilocode/package-lock.json | 31 ------------------------------- 2 files changed, 62 deletions(-) delete mode 100644 .kilo/package-lock.json delete mode 100644 .kilocode/package-lock.json diff --git a/.kilo/package-lock.json b/.kilo/package-lock.json deleted file mode 100644 index 7013f0cbd7..0000000000 --- a/.kilo/package-lock.json +++ /dev/null @@ -1,31 +0,0 @@ -{ - "name": ".kilo", - "lockfileVersion": 3, - "requires": true, - "packages": { - "": { - "dependencies": { - "@kilocode/plugin": "*" - } - }, - "node_modules/@kilocode/plugin": { - "version": "7.1.23", - "license": "MIT", - "dependencies": { - "@kilocode/sdk": "7.1.23", - "zod": "4.1.8" - } - }, - "node_modules/@kilocode/sdk": { - "version": "7.1.23", - "license": "MIT" - }, - "node_modules/zod": { - "version": "4.1.8", - "license": "MIT", - "funding": { - "url": "https://github.com/sponsors/colinhacks" - } - } - } -} diff --git a/.kilocode/package-lock.json b/.kilocode/package-lock.json deleted file mode 100644 index a83a334755..0000000000 --- a/.kilocode/package-lock.json +++ /dev/null @@ -1,31 +0,0 @@ -{ - "name": ".kilocode", - "lockfileVersion": 3, - "requires": true, - "packages": { - "": { - "dependencies": { - "@kilocode/plugin": "*" - } - }, - "node_modules/@kilocode/plugin": { - "version": "7.1.23", - "license": "MIT", - "dependencies": { - "@kilocode/sdk": "7.1.23", - "zod": "4.1.8" - } - }, - "node_modules/@kilocode/sdk": { - "version": "7.1.23", - "license": "MIT" - }, - "node_modules/zod": { - "version": "4.1.8", - "license": "MIT", - "funding": { - "url": "https://github.com/sponsors/colinhacks" - } - } - } -} From fa08250d73b2b3d66851622b59481437c71ecd5d Mon Sep 17 00:00:00 2001 From: "kiloconnect[bot]" <240665456+kiloconnect[bot]@users.noreply.github.com> Date: Tue, 21 Apr 2026 13:15:30 +0000 Subject: [PATCH 15/24] refactor(cli): restore streaming line read and tidy markers - Restore the streaming createInterface loop in the `lines` helper so the diff against main stays small; decode bytes with iconv-lite up front and feed the resulting text through Readable.from. - Drop 'new file' kilocode_change markers from files under kilocode/ directories (those paths are exempt from the annotation check). - Tighten the changeset copy to sound like a user-facing release note. --- .changeset/preserve-file-encoding.md | 13 +---- packages/opencode/src/kilocode/encoding.ts | 1 - .../opencode/src/kilocode/tool/encoded-io.ts | 1 - packages/opencode/src/tool/read.ts | 57 +++++++++++-------- .../test/kilocode/tool-encoding.test.ts | 1 - 5 files changed, 36 insertions(+), 37 deletions(-) diff --git a/.changeset/preserve-file-encoding.md b/.changeset/preserve-file-encoding.md index a454443b56..28221dde06 100644 --- a/.changeset/preserve-file-encoding.md +++ b/.changeset/preserve-file-encoding.md @@ -2,15 +2,8 @@ "@kilocode/cli": minor --- -Preserve the original text encoding when reading and editing files. The read, edit, write, and apply_patch tools now detect each file's encoding and write it back unchanged, so non-UTF-8 source trees no longer get corrupted when the agent touches them. +Detect and preserve the original text encoding of files when the agent reads or edits them. Source trees in Japanese, Chinese, Korean, Cyrillic, or Western European encodings no longer get mangled when Kilo touches them, and non-UTF-8 files are displayed correctly to the model instead of as garbled text. -Supported: +Supported: UTF-8, UTF-16 with BOM, and common legacy Latin and CJK encodings (Shift_JIS, EUC-JP, GB2312, Big5, EUC-KR, Windows-1251, KOI8-R, ISO-8859, and others). -- UTF-8 -- UTF-16 with BOM -- Legacy Latin and CJK encodings (Shift_JIS, EUC-JP, GB2312, Big5, EUC-KR, Windows-1251, KOI8-R, ISO-8859 family, and others) - -Not supported: - -- UTF-16 without BOM -- UTF-32 +Not supported: UTF-16 without BOM, UTF-32. diff --git a/packages/opencode/src/kilocode/encoding.ts b/packages/opencode/src/kilocode/encoding.ts index eed7b00400..3b5a17b7f5 100644 --- a/packages/opencode/src/kilocode/encoding.ts +++ b/packages/opencode/src/kilocode/encoding.ts @@ -1,4 +1,3 @@ -// kilocode_change - new file import { readFile, writeFile, mkdir } from "fs/promises" import { readFileSync } from "fs" import { dirname } from "path" diff --git a/packages/opencode/src/kilocode/tool/encoded-io.ts b/packages/opencode/src/kilocode/tool/encoded-io.ts index dab6e26e25..8aa6569235 100644 --- a/packages/opencode/src/kilocode/tool/encoded-io.ts +++ b/packages/opencode/src/kilocode/tool/encoded-io.ts @@ -1,4 +1,3 @@ -// kilocode_change - new file import { Effect } from "effect" import { Encoding } from "../encoding" diff --git a/packages/opencode/src/tool/read.ts b/packages/opencode/src/tool/read.ts index e01f87af73..e00c5d7bf3 100644 --- a/packages/opencode/src/tool/read.ts +++ b/packages/opencode/src/tool/read.ts @@ -2,6 +2,8 @@ import z from "zod" import { Effect, Scope } from "effect" import { open } from "fs/promises" import * as path from "path" +import { Readable } from "stream" // kilocode_change +import { createInterface } from "readline" import { Tool } from "./tool" import { AppFileSystem } from "../filesystem" import { LSP } from "../lsp" @@ -233,45 +235,53 @@ export const ReadTool = Tool.define( }), ) -// kilocode_change start - encoding-aware file reading +// kilocode_change start export async function lines(filepath: string, opts: { limit: number; offset: number }) { const encoded = await Encoding.read(filepath) - const all = encoded.text.split(/\r\n|\r|\n/) - // Remove trailing empty element from split when file ends with newline - if (all.length > 0 && all[all.length - 1] === "") all.pop() + const rl = createInterface({ + input: Readable.from([encoded.text]), + // Note: we use the crlfDelay option to recognize all instances of CR LF + // ('\r\n') in file as a single line break. + crlfDelay: Infinity, + }) const start = opts.offset - 1 const raw: string[] = [] let bytes = 0 + let count = 0 let cut = false let more = false + try { + for await (const text of rl) { + count += 1 + if (count <= start) continue - for (let i = start; i < all.length; i++) { - if (raw.length >= opts.limit) { - more = true - break + if (raw.length >= opts.limit) { + more = true + continue + } + + const line = text.length > MAX_LINE_LENGTH ? text.substring(0, MAX_LINE_LENGTH) + MAX_LINE_SUFFIX : text + const size = Buffer.byteLength(line, "utf-8") + (raw.length > 0 ? 1 : 0) + if (bytes + size > MAX_BYTES) { + cut = true + more = true + break + } + + raw.push(line) + bytes += size } - - const text = all[i]! - const line = text.length > MAX_LINE_LENGTH ? text.substring(0, MAX_LINE_LENGTH) + MAX_LINE_SUFFIX : text - const size = Buffer.byteLength(line, "utf-8") + (raw.length > 0 ? 1 : 0) - if (bytes + size > MAX_BYTES) { - cut = true - more = true - break - } - - raw.push(line) - bytes += size + } finally { + rl.close() } - return { raw, count: all.length, cut, more, offset: opts.offset } + return { raw, count, cut, more, offset: opts.offset } } // kilocode_change end // kilocode_change start export async function isBinaryFile(filepath: string, fileSize: number): Promise { - // kilocode_change end const ext = path.extname(filepath).toLowerCase() // binary check for common non-text extensions switch (ext) { @@ -317,7 +327,6 @@ export async function isBinaryFile(filepath: string, fileSize: number): Promise< const result = await fh.read(bytes, 0, sampleSize, 0) if (result.bytesRead === 0) return false - // kilocode_change start - encoding-aware binary detection // If encoding detection identifies a known text encoding (including UTF-16 // LE/BE with BOM or CJK), it's text — not binary. This prevents UTF-16 // files with legitimate null bytes from being falsely rejected. @@ -334,8 +343,8 @@ export async function isBinaryFile(filepath: string, fileSize: number): Promise< } // If >30% non-printable characters, consider it binary return nonPrintableCount / result.bytesRead > 0.3 - // kilocode_change end } finally { await fh.close() } } +// kilocode_change end diff --git a/packages/opencode/test/kilocode/tool-encoding.test.ts b/packages/opencode/test/kilocode/tool-encoding.test.ts index b9c318c777..c1324cd567 100644 --- a/packages/opencode/test/kilocode/tool-encoding.test.ts +++ b/packages/opencode/test/kilocode/tool-encoding.test.ts @@ -1,4 +1,3 @@ -// kilocode_change - new file // Integration tests verifying that the agent file tools (read, write, edit, // apply_patch) detect and preserve the original encoding of files on disk. // Tests exercise the real tool pipeline rather than the Encoding helper From 0cef72517c1883b83472d7542e6229739a3b17fa Mon Sep 17 00:00:00 2001 From: "kiloconnect[bot]" <240665456+kiloconnect[bot]@users.noreply.github.com> Date: Tue, 21 Apr 2026 13:16:03 +0000 Subject: [PATCH 16/24] wip --- .kilo/package-lock.json | 31 +++++++++++++++++++++++++++++++ .kilocode/package-lock.json | 31 +++++++++++++++++++++++++++++++ 2 files changed, 62 insertions(+) create mode 100644 .kilo/package-lock.json create mode 100644 .kilocode/package-lock.json diff --git a/.kilo/package-lock.json b/.kilo/package-lock.json new file mode 100644 index 0000000000..7013f0cbd7 --- /dev/null +++ b/.kilo/package-lock.json @@ -0,0 +1,31 @@ +{ + "name": ".kilo", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "dependencies": { + "@kilocode/plugin": "*" + } + }, + "node_modules/@kilocode/plugin": { + "version": "7.1.23", + "license": "MIT", + "dependencies": { + "@kilocode/sdk": "7.1.23", + "zod": "4.1.8" + } + }, + "node_modules/@kilocode/sdk": { + "version": "7.1.23", + "license": "MIT" + }, + "node_modules/zod": { + "version": "4.1.8", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/colinhacks" + } + } + } +} diff --git a/.kilocode/package-lock.json b/.kilocode/package-lock.json new file mode 100644 index 0000000000..a83a334755 --- /dev/null +++ b/.kilocode/package-lock.json @@ -0,0 +1,31 @@ +{ + "name": ".kilocode", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "dependencies": { + "@kilocode/plugin": "*" + } + }, + "node_modules/@kilocode/plugin": { + "version": "7.1.23", + "license": "MIT", + "dependencies": { + "@kilocode/sdk": "7.1.23", + "zod": "4.1.8" + } + }, + "node_modules/@kilocode/sdk": { + "version": "7.1.23", + "license": "MIT" + }, + "node_modules/zod": { + "version": "4.1.8", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/colinhacks" + } + } + } +} From 5538fc9c3299f63fc5e0d43aa5db5de85128cb42 Mon Sep 17 00:00:00 2001 From: "kiloconnect[bot]" <240665456+kiloconnect[bot]@users.noreply.github.com> Date: Tue, 21 Apr 2026 13:32:48 +0000 Subject: [PATCH 17/24] feat(cli): preserve UTF-8 BOM through read/write round-trip iconv-lite's utf-8 codec strips any leading BOM on decode and never emits one on encode, so files that started with EF BB BF would lose their BOM after an edit. Track UTF-8-with-BOM as a distinct synthetic encoding label and re-prepend the BOM bytes on write so the round-trip keeps the file byte-identical (modulo any actual edits). --- .changeset/preserve-file-encoding.md | 4 +-- packages/opencode/src/kilocode/encoding.ts | 31 ++++++++++++++----- .../test/kilocode/tool-encoding.test.ts | 14 +++++++-- 3 files changed, 37 insertions(+), 12 deletions(-) diff --git a/.changeset/preserve-file-encoding.md b/.changeset/preserve-file-encoding.md index 28221dde06..034e6540f3 100644 --- a/.changeset/preserve-file-encoding.md +++ b/.changeset/preserve-file-encoding.md @@ -2,8 +2,8 @@ "@kilocode/cli": minor --- -Detect and preserve the original text encoding of files when the agent reads or edits them. Source trees in Japanese, Chinese, Korean, Cyrillic, or Western European encodings no longer get mangled when Kilo touches them, and non-UTF-8 files are displayed correctly to the model instead of as garbled text. +The agent now detects and preserves the original text encoding of files when reading and editing them, so non-UTF-8 files are displayed correctly to the model and written back in their original encoding. -Supported: UTF-8, UTF-16 with BOM, and common legacy Latin and CJK encodings (Shift_JIS, EUC-JP, GB2312, Big5, EUC-KR, Windows-1251, KOI8-R, ISO-8859, and others). +Supported: UTF-8 (with or without BOM), UTF-16 with BOM, and common legacy Latin and CJK encodings (Shift_JIS, EUC-JP, GB2312, Big5, EUC-KR, Windows-1251, KOI8-R, ISO-8859, and others). Not supported: UTF-16 without BOM, UTF-32. diff --git a/packages/opencode/src/kilocode/encoding.ts b/packages/opencode/src/kilocode/encoding.ts index 3b5a17b7f5..d4c21e5678 100644 --- a/packages/opencode/src/kilocode/encoding.ts +++ b/packages/opencode/src/kilocode/encoding.ts @@ -17,13 +17,26 @@ import iconv from "iconv-lite" * - UTF-32 (extremely rare) * * Detection strategy: - * 1. If the bytes are valid UTF-8, treat as UTF-8. - * 2. Otherwise, trust jschardet. iconv-lite handles BOM stripping on decode - * and BOM emission on encode for UTF-16 LE/BE, so explicit BOM handling - * is unnecessary. + * 1. If the bytes are valid UTF-8, treat as UTF-8 (tracking the presence of a + * BOM so it can be written back). + * 2. Otherwise, trust jschardet. + * + * iconv-lite's UTF codecs strip BOMs on decode and do not emit them on encode, + * so UTF BOMs are handled explicitly in {@link encode} to round-trip cleanly. */ export namespace Encoding { export const DEFAULT = "utf-8" + /** + * Synthetic label for UTF-8 files that start with a BOM. iconv-lite's utf-8 + * codec always strips BOMs on decode and never emits one on encode, so we + * track the "with BOM" case explicitly to round-trip it faithfully. + */ + export const UTF8_BOM = "utf-8-bom" + const UTF8_BOM_BYTES = Buffer.from([0xef, 0xbb, 0xbf]) + + function hasUtf8Bom(bytes: Buffer): boolean { + return bytes.length >= 3 && bytes[0] === 0xef && bytes[1] === 0xbb && bytes[2] === 0xbf + } /** Remap jschardet labels to iconv-lite compatible names. */ function normalize(name: string): string { @@ -72,7 +85,7 @@ export namespace Encoding { export function detect(bytes: Buffer): string { if (bytes.length === 0) return DEFAULT - if (isUtf8(bytes)) return DEFAULT + if (isUtf8(bytes)) return hasUtf8Bom(bytes) ? UTF8_BOM : DEFAULT const result = jschardet.detect(bytes) if (!result.encoding) return DEFAULT const enc = normalize(result.encoding) @@ -83,13 +96,15 @@ export namespace Encoding { } export function decode(bytes: Buffer, encoding: string): string { + if (encoding === UTF8_BOM) return iconv.decode(bytes, "utf-8") return iconv.decode(bytes, encoding) } export function encode(text: string, encoding: string): Buffer { - // iconv-lite's utf-16le/utf-16be do not emit a BOM, but UTF-16 without a - // BOM is unsupported in this codebase. Prepend the appropriate BOM so the - // next detection pass can still recognise the encoding. + // iconv-lite's UTF codecs strip/ignore BOMs, but we support "UTF-X with BOM" + // as a distinct variant. Prepend the BOM manually so round-tripping keeps + // the original byte signature intact. + if (encoding === UTF8_BOM) return Buffer.concat([UTF8_BOM_BYTES, iconv.encode(text, "utf-8")]) const lower = encoding.toLowerCase() if (lower === "utf-16le") return Buffer.concat([Buffer.from([0xff, 0xfe]), iconv.encode(text, encoding)]) if (lower === "utf-16be") return Buffer.concat([Buffer.from([0xfe, 0xff]), iconv.encode(text, encoding)]) diff --git a/packages/opencode/test/kilocode/tool-encoding.test.ts b/packages/opencode/test/kilocode/tool-encoding.test.ts index c1324cd567..e05d8ff4ff 100644 --- a/packages/opencode/test/kilocode/tool-encoding.test.ts +++ b/packages/opencode/test/kilocode/tool-encoding.test.ts @@ -90,9 +90,12 @@ const markRead = (filepath: string) => yield* ft.read(ctx.sessionID, filepath) }) -// iconv-lite's utf-16le/utf-16be do not emit a BOM on their own, but this -// codebase only supports UTF-16 with BOM. Prepend one for fixture files. +// iconv-lite's UTF codecs don't emit BOMs, but this codebase supports +// "UTF-X with BOM" as a distinct variant. Prepend one here for fixture files +// that are meant to have one. +const UTF8_BOM = "utf-8-bom" const encodeBytes = (text: string, encoding: string): Buffer => { + if (encoding === UTF8_BOM) return Buffer.concat([Buffer.from([0xef, 0xbb, 0xbf]), iconv.encode(text, "utf-8")]) const lower = encoding.toLowerCase() if (lower === "utf-16le") return Buffer.concat([Buffer.from([0xff, 0xfe]), iconv.encode(text, encoding)]) if (lower === "utf-16be") return Buffer.concat([Buffer.from([0xfe, 0xff]), iconv.encode(text, encoding)]) @@ -109,6 +112,10 @@ const putEncoded = (filepath: string, text: string, encoding: string) => const loadDecoded = (filepath: string, encoding: string) => Effect.promise(async () => { const bytes = await fs.readFile(filepath) + if (encoding === UTF8_BOM) { + const stripped = bytes.length >= 3 && bytes[0] === 0xef && bytes[1] === 0xbb && bytes[2] === 0xbf + return iconv.decode(stripped ? bytes.subarray(3) : bytes, "utf-8") + } return iconv.decode(bytes, encoding) }) @@ -131,6 +138,7 @@ describe("tool encoding preservation", () => { describe("ReadTool decodes files with non-UTF-8 encodings", () => { const cases: Array<[string, string, string]> = [ ["UTF-8", "utf-8", samples.utf8], + ["UTF-8 with BOM", UTF8_BOM, samples.utf8], ["UTF-16 LE with BOM", "utf-16le", samples.utf8], ["UTF-16 BE with BOM", "utf-16be", samples.utf8], ["Shift_JIS", "Shift_JIS", samples.shiftJis], @@ -176,6 +184,7 @@ describe("tool encoding preservation", () => { describe("WriteTool preserves existing file encoding when overwriting", () => { const cases: Array<[string, string, string]> = [ + ["UTF-8 with BOM", UTF8_BOM, samples.utf8], ["Shift_JIS", "Shift_JIS", samples.shiftJis], ["GB2312", "gb2312", samples.gb2312], ["Windows-1251", "windows-1251", samples.windows1251], @@ -219,6 +228,7 @@ describe("tool encoding preservation", () => { describe("EditTool preserves existing file encoding across edits", () => { const cases: Array<[string, string, string, string, string]> = [ + ["UTF-8 with BOM", UTF8_BOM, samples.utf8 + "\n second line", "world", "earth"], ["Shift_JIS", "Shift_JIS", samples.shiftJis, "日本語", "ニホンゴ"], ["GB2312", "gb2312", samples.gb2312, "简体中文", "中文简体"], ["Windows-1251", "windows-1251", samples.windows1251, "мир", "планета"], From 6f55a3061d75a81454d993a43fe82f6eafeca3d4 Mon Sep 17 00:00:00 2001 From: "kiloconnect[bot]" <240665456+kiloconnect[bot]@users.noreply.github.com> Date: Tue, 21 Apr 2026 14:04:35 +0000 Subject: [PATCH 18/24] refactor(cli): restore stream.destroy and keep original comments - Wire Readable.from through a named `stream` variable so the `lines` helper can destroy it in the finally block, matching the previous createReadStream pattern. - Restore the 'Create parent directories', 'Handle file move', 'Regular update', 'Read original file content', and 'For delete, we need to read the current content' comments that were dropped when switching to Encoding.write / Encoding.read. The explicit fs.mkdir calls are no longer needed because Encoding.write mkdirs recursively, but the intent comments still apply. --- packages/opencode/src/patch/index.ts | 20 +++++++++++++------- packages/opencode/src/tool/apply_patch.ts | 10 +++++++--- packages/opencode/src/tool/read.ts | 4 +++- 3 files changed, 23 insertions(+), 11 deletions(-) diff --git a/packages/opencode/src/patch/index.ts b/packages/opencode/src/patch/index.ts index c116df0f94..3e6ab4bb1a 100644 --- a/packages/opencode/src/patch/index.ts +++ b/packages/opencode/src/patch/index.ts @@ -311,6 +311,7 @@ export namespace Patch { export function deriveNewContentsFromChunks(filePath: string, chunks: UpdateFileChunk[]): ApplyPatchFileUpdate { // kilocode_change start - encoding-aware read + // Read original file content let originalContent: string let encoding: string try { @@ -342,11 +343,13 @@ export namespace Patch { // Generate unified diff const unifiedDiff = generateUnifiedDiff(originalContent, newContent) + // kilocode_change start - include detected encoding for round-trip write return { unified_diff: unifiedDiff, content: newContent, - encoding, // kilocode_change + encoding, } + // kilocode_change end } function computeReplacements( @@ -530,10 +533,12 @@ export namespace Patch { const modified: string[] = [] const deleted: string[] = [] + // kilocode_change start - encoding-aware writes (Encoding.write mkdirs recursively) for (const hunk of hunks) { switch (hunk.type) { case "add": - await Encoding.write(hunk.path, hunk.contents) // kilocode_change - encoding-aware write + // Create parent directories + await Encoding.write(hunk.path, hunk.contents) added.push(hunk.path) log.info(`Added file: ${hunk.path}`) break @@ -548,20 +553,21 @@ export namespace Patch { const fileUpdate = deriveNewContentsFromChunks(hunk.path, hunk.chunks) if (hunk.move_path) { - // kilocode_change start - encoding-aware move + // Handle file move await Encoding.write(hunk.move_path, fileUpdate.content, fileUpdate.encoding) await fs.unlink(hunk.path) - // kilocode_change end modified.push(hunk.move_path) log.info(`Moved file: ${hunk.path} -> ${hunk.move_path}`) } else { - await Encoding.write(hunk.path, fileUpdate.content, fileUpdate.encoding) // kilocode_change + // Regular update + await Encoding.write(hunk.path, fileUpdate.content, fileUpdate.encoding) modified.push(hunk.path) log.info(`Updated file: ${hunk.path}`) } break } } + // kilocode_change end return { added, modified, deleted } } @@ -608,6 +614,7 @@ export namespace Patch { hunk.type === "update" && hunk.move_path ? hunk.move_path : hunk.path, ) + // kilocode_change start - encoding-aware read for delete hunks switch (hunk.type) { case "add": changes.set(resolvedPath, { @@ -620,19 +627,18 @@ export namespace Patch { // For delete, we need to read the current content const deletePath = path.resolve(effectiveCwd, hunk.path) try { - // kilocode_change start - encoding-aware read const result = await Encoding.read(deletePath) changes.set(resolvedPath, { type: "delete", content: result.text, }) - // kilocode_change end } catch (error) { return { type: MaybeApplyPatchVerified.CorrectnessError, error: new Error(`Failed to read file for deletion: ${deletePath}`), } } + // kilocode_change end break case "update": diff --git a/packages/opencode/src/tool/apply_patch.ts b/packages/opencode/src/tool/apply_patch.ts index be1cbbb9f3..823359e8f0 100644 --- a/packages/opencode/src/tool/apply_patch.ts +++ b/packages/opencode/src/tool/apply_patch.ts @@ -208,22 +208,25 @@ export const ApplyPatchTool = Tool.define( // Apply the changes const updates: Array<{ file: string; event: "add" | "change" | "unlink" }> = [] + // kilocode_change start - encoding-aware writes (EncodedIO.write mkdirs recursively) for (const change of fileChanges) { const edited = change.type === "delete" ? undefined : (change.movePath ?? change.filePath) switch (change.type) { case "add": - yield* EncodedIO.write(change.filePath, change.newContent, change.encoding) // kilocode_change + // Create parent directories (recursive: true is safe on existing/root dirs) + yield* EncodedIO.write(change.filePath, change.newContent, change.encoding) updates.push({ file: change.filePath, event: "add" }) break case "update": - yield* EncodedIO.write(change.filePath, change.newContent, change.encoding) // kilocode_change + yield* EncodedIO.write(change.filePath, change.newContent, change.encoding) updates.push({ file: change.filePath, event: "change" }) break case "move": if (change.movePath) { - yield* EncodedIO.write(change.movePath!, change.newContent, change.encoding) // kilocode_change + // Create parent directories (recursive: true is safe on existing/root dirs) + yield* EncodedIO.write(change.movePath!, change.newContent, change.encoding) yield* afs.remove(change.filePath) updates.push({ file: change.filePath, event: "unlink" }) updates.push({ file: change.movePath, event: "add" }) @@ -235,6 +238,7 @@ export const ApplyPatchTool = Tool.define( updates.push({ file: change.filePath, event: "unlink" }) break } + // kilocode_change end if (edited) { yield* format.file(edited) diff --git a/packages/opencode/src/tool/read.ts b/packages/opencode/src/tool/read.ts index e00c5d7bf3..471508fe63 100644 --- a/packages/opencode/src/tool/read.ts +++ b/packages/opencode/src/tool/read.ts @@ -238,8 +238,9 @@ export const ReadTool = Tool.define( // kilocode_change start export async function lines(filepath: string, opts: { limit: number; offset: number }) { const encoded = await Encoding.read(filepath) + const stream = Readable.from([encoded.text]) const rl = createInterface({ - input: Readable.from([encoded.text]), + input: stream, // Note: we use the crlfDelay option to recognize all instances of CR LF // ('\r\n') in file as a single line break. crlfDelay: Infinity, @@ -274,6 +275,7 @@ export async function lines(filepath: string, opts: { limit: number; offset: num } } finally { rl.close() + stream.destroy() } return { raw, count, cut, more, offset: opts.offset } From cce5e459bc9f01fa4b716301b4b23c0598753fbc Mon Sep 17 00:00:00 2001 From: "kiloconnect[bot]" <240665456+kiloconnect[bot]@users.noreply.github.com> Date: Tue, 21 Apr 2026 16:18:47 +0000 Subject: [PATCH 19/24] =?UTF-8?q?refactor(cli):=20self-review=20=E2=80=94?= =?UTF-8?q?=20tighten=20markers,=20fix=20detection=20edge=20cases?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Encoding.detect: lowercase-compare UTF-32 rejection so jschardet's uppercase 'UTF-32'/'UTF-32BE'/'UTF-32LE' labels no longer slip through (iconv-lite happens to have utf-32 codecs that would then be accepted). - Encoding.read: drop redundant Buffer.from wrap around readFile's already-Buffer return value. - EncodedIO: switch to Effect.tryPromise so I/O failures surface as typed errors that .pipe(Effect.catch(...)) can recover from; the apply_patch delete branch relies on this to translate read errors into 'apply_patch verification failed'. - Shrink kilocode_change blocks in edit.ts, write.ts, apply_patch.ts, patch/index.ts, and read.ts to per-line inline markers where each block was only wrapping 1-3 changed lines. - Collapse the 'if (exists) { let contentOld; let encoding; ... }' blocks in edit.ts and write.ts into a single ternary + destructure that keeps all three lines self-contained and each independently annotated. - Remove the unused 'latin1' sample and fix the 'providEncoded' typo in the encoding integration test file. - Strengthen the apply_patch Shift_JIS test to assert exact bytes instead of the weak 'does not contain these two UTF-8 bytes' check. - Drop the stray .kilo/ and .kilocode/ lockfiles that were re-added by an upstream 'wip' commit. --- .kilo/package-lock.json | 31 ------------------- .kilocode/package-lock.json | 31 ------------------- packages/opencode/src/kilocode/encoding.ts | 6 ++-- .../opencode/src/kilocode/tool/encoded-io.ts | 9 ++++-- packages/opencode/src/patch/index.ts | 28 ++++++----------- packages/opencode/src/tool/apply_patch.ts | 8 ++--- packages/opencode/src/tool/edit.ts | 19 ++++-------- packages/opencode/src/tool/read.ts | 13 ++++---- packages/opencode/src/tool/write.ts | 12 ++----- .../test/kilocode/tool-encoding.test.ts | 20 ++++++------ 10 files changed, 47 insertions(+), 130 deletions(-) delete mode 100644 .kilo/package-lock.json delete mode 100644 .kilocode/package-lock.json diff --git a/.kilo/package-lock.json b/.kilo/package-lock.json deleted file mode 100644 index 7013f0cbd7..0000000000 --- a/.kilo/package-lock.json +++ /dev/null @@ -1,31 +0,0 @@ -{ - "name": ".kilo", - "lockfileVersion": 3, - "requires": true, - "packages": { - "": { - "dependencies": { - "@kilocode/plugin": "*" - } - }, - "node_modules/@kilocode/plugin": { - "version": "7.1.23", - "license": "MIT", - "dependencies": { - "@kilocode/sdk": "7.1.23", - "zod": "4.1.8" - } - }, - "node_modules/@kilocode/sdk": { - "version": "7.1.23", - "license": "MIT" - }, - "node_modules/zod": { - "version": "4.1.8", - "license": "MIT", - "funding": { - "url": "https://github.com/sponsors/colinhacks" - } - } - } -} diff --git a/.kilocode/package-lock.json b/.kilocode/package-lock.json deleted file mode 100644 index a83a334755..0000000000 --- a/.kilocode/package-lock.json +++ /dev/null @@ -1,31 +0,0 @@ -{ - "name": ".kilocode", - "lockfileVersion": 3, - "requires": true, - "packages": { - "": { - "dependencies": { - "@kilocode/plugin": "*" - } - }, - "node_modules/@kilocode/plugin": { - "version": "7.1.23", - "license": "MIT", - "dependencies": { - "@kilocode/sdk": "7.1.23", - "zod": "4.1.8" - } - }, - "node_modules/@kilocode/sdk": { - "version": "7.1.23", - "license": "MIT" - }, - "node_modules/zod": { - "version": "4.1.8", - "license": "MIT", - "funding": { - "url": "https://github.com/sponsors/colinhacks" - } - } - } -} diff --git a/packages/opencode/src/kilocode/encoding.ts b/packages/opencode/src/kilocode/encoding.ts index d4c21e5678..9345dd56a9 100644 --- a/packages/opencode/src/kilocode/encoding.ts +++ b/packages/opencode/src/kilocode/encoding.ts @@ -89,8 +89,8 @@ export namespace Encoding { const result = jschardet.detect(bytes) if (!result.encoding) return DEFAULT const enc = normalize(result.encoding) - // Reject unsupported Unicode encodings - if (enc.startsWith("utf-32")) return DEFAULT + // Reject unsupported Unicode encodings (UTF-32 and anything iconv-lite cannot decode) + if (enc.toLowerCase().startsWith("utf-32")) return DEFAULT if (!iconv.encodingExists(enc)) return DEFAULT return enc } @@ -113,7 +113,7 @@ export namespace Encoding { /** Read a file, detecting its encoding. */ export async function read(path: string): Promise<{ text: string; encoding: string }> { - const bytes = Buffer.from(await readFile(path)) + const bytes = await readFile(path) const encoding = detect(bytes) return { text: decode(bytes, encoding), encoding } } diff --git a/packages/opencode/src/kilocode/tool/encoded-io.ts b/packages/opencode/src/kilocode/tool/encoded-io.ts index 8aa6569235..19c74b4e4b 100644 --- a/packages/opencode/src/kilocode/tool/encoded-io.ts +++ b/packages/opencode/src/kilocode/tool/encoded-io.ts @@ -4,11 +4,14 @@ import { Encoding } from "../encoding" /** * Effect wrappers around {@link Encoding.read} and {@link Encoding.write} so * tool code can preserve file encoding without leaking Node/async boilerplate - * into each call site. + * into each call site. Uses {@link Effect.tryPromise} so I/O failures surface + * as typed errors that can be recovered with `.pipe(Effect.catch(...))`. */ export namespace EncodedIO { - export const read = (path: string) => Effect.promise(() => Encoding.read(path)) + const wrap = (cause: unknown) => (cause instanceof Error ? cause : new Error(String(cause))) + + export const read = (path: string) => Effect.tryPromise({ try: () => Encoding.read(path), catch: wrap }) export const write = (path: string, text: string, encoding: string = Encoding.DEFAULT) => - Effect.promise(() => Encoding.write(path, text, encoding)) + Effect.tryPromise({ try: () => Encoding.write(path, text, encoding), catch: wrap }) } diff --git a/packages/opencode/src/patch/index.ts b/packages/opencode/src/patch/index.ts index 3e6ab4bb1a..323ce403fc 100644 --- a/packages/opencode/src/patch/index.ts +++ b/packages/opencode/src/patch/index.ts @@ -310,18 +310,16 @@ export namespace Patch { } export function deriveNewContentsFromChunks(filePath: string, chunks: UpdateFileChunk[]): ApplyPatchFileUpdate { - // kilocode_change start - encoding-aware read // Read original file content let originalContent: string - let encoding: string + let encoding: string // kilocode_change try { - const result = Encoding.readSync(filePath) - originalContent = result.text - encoding = result.encoding + const result = Encoding.readSync(filePath) // kilocode_change - encoding-aware read + originalContent = result.text // kilocode_change + encoding = result.encoding // kilocode_change } catch (error) { throw new Error(`Failed to read file ${filePath}: ${error}`) } - // kilocode_change end let originalLines = originalContent.split("\n") @@ -343,13 +341,11 @@ export namespace Patch { // Generate unified diff const unifiedDiff = generateUnifiedDiff(originalContent, newContent) - // kilocode_change start - include detected encoding for round-trip write return { unified_diff: unifiedDiff, content: newContent, - encoding, + encoding, // kilocode_change - include detected encoding for round-trip write } - // kilocode_change end } function computeReplacements( @@ -533,12 +529,11 @@ export namespace Patch { const modified: string[] = [] const deleted: string[] = [] - // kilocode_change start - encoding-aware writes (Encoding.write mkdirs recursively) for (const hunk of hunks) { switch (hunk.type) { case "add": // Create parent directories - await Encoding.write(hunk.path, hunk.contents) + await Encoding.write(hunk.path, hunk.contents) // kilocode_change - encoding-aware write (mkdirs) added.push(hunk.path) log.info(`Added file: ${hunk.path}`) break @@ -554,20 +549,19 @@ export namespace Patch { if (hunk.move_path) { // Handle file move - await Encoding.write(hunk.move_path, fileUpdate.content, fileUpdate.encoding) + await Encoding.write(hunk.move_path, fileUpdate.content, fileUpdate.encoding) // kilocode_change await fs.unlink(hunk.path) modified.push(hunk.move_path) log.info(`Moved file: ${hunk.path} -> ${hunk.move_path}`) } else { // Regular update - await Encoding.write(hunk.path, fileUpdate.content, fileUpdate.encoding) + await Encoding.write(hunk.path, fileUpdate.content, fileUpdate.encoding) // kilocode_change modified.push(hunk.path) log.info(`Updated file: ${hunk.path}`) } break } } - // kilocode_change end return { added, modified, deleted } } @@ -614,7 +608,6 @@ export namespace Patch { hunk.type === "update" && hunk.move_path ? hunk.move_path : hunk.path, ) - // kilocode_change start - encoding-aware read for delete hunks switch (hunk.type) { case "add": changes.set(resolvedPath, { @@ -627,10 +620,10 @@ export namespace Patch { // For delete, we need to read the current content const deletePath = path.resolve(effectiveCwd, hunk.path) try { - const result = await Encoding.read(deletePath) + const result = await Encoding.read(deletePath) // kilocode_change - encoding-aware read changes.set(resolvedPath, { type: "delete", - content: result.text, + content: result.text, // kilocode_change }) } catch (error) { return { @@ -638,7 +631,6 @@ export namespace Patch { error: new Error(`Failed to read file for deletion: ${deletePath}`), } } - // kilocode_change end break case "update": diff --git a/packages/opencode/src/tool/apply_patch.ts b/packages/opencode/src/tool/apply_patch.ts index 823359e8f0..deb5d6a6de 100644 --- a/packages/opencode/src/tool/apply_patch.ts +++ b/packages/opencode/src/tool/apply_patch.ts @@ -109,12 +109,10 @@ export const ApplyPatchTool = Tool.define( ) } - // kilocode_change start - preserve existing file encoding - const readResult = yield* EncodedIO.read(filePath) - const oldContent = readResult.text - let encoding = readResult.encoding + const pre = yield* EncodedIO.read(filePath) // kilocode_change - preserve file encoding + const oldContent = pre.text // kilocode_change + let encoding = pre.encoding // kilocode_change let newContent = oldContent - // kilocode_change end // Apply the update chunks to get new content try { diff --git a/packages/opencode/src/tool/edit.ts b/packages/opencode/src/tool/edit.ts index b89417d1b5..5ff46065a2 100644 --- a/packages/opencode/src/tool/edit.ts +++ b/packages/opencode/src/tool/edit.ts @@ -101,14 +101,9 @@ export const EditTool = Tool.define( Effect.gen(function* () { if (params.oldString === "") { const existed = yield* afs.existsSafe(filePath) - // kilocode_change start - preserve existing file encoding - let encoding = "utf-8" - if (existed) { - const encoded = yield* EncodedIO.read(filePath) - contentOld = encoded.text - encoding = encoded.encoding - } - // kilocode_change end + const pre = existed ? yield* EncodedIO.read(filePath) : { text: "", encoding: "utf-8" } // kilocode_change + contentOld = pre.text // kilocode_change + const encoding = pre.encoding // kilocode_change - preserve file encoding on write contentNew = params.newString diff = trimDiff(createTwoFilesPatch(filePath, filePath, contentOld, contentNew)) cachedFilediff = buildFileDiff(filePath, contentOld, contentNew) // kilocode_change @@ -137,11 +132,9 @@ export const EditTool = Tool.define( if (!info) throw new Error(`File ${filePath} not found`) if (info.type === "Directory") throw new Error(`Path is a directory, not a file: ${filePath}`) yield* filetime.assert(ctx.sessionID, filePath) - // kilocode_change start - preserve existing file encoding - const encoded = yield* EncodedIO.read(filePath) - contentOld = encoded.text - const encoding = encoded.encoding - // kilocode_change end + const pre = yield* EncodedIO.read(filePath) // kilocode_change - preserve file encoding + contentOld = pre.text // kilocode_change + const encoding = pre.encoding // kilocode_change const ending = detectLineEnding(contentOld) const old = convertToLineEnding(normalizeLineEndings(params.oldString), ending) diff --git a/packages/opencode/src/tool/read.ts b/packages/opencode/src/tool/read.ts index 471508fe63..5d596fd4ec 100644 --- a/packages/opencode/src/tool/read.ts +++ b/packages/opencode/src/tool/read.ts @@ -237,8 +237,9 @@ export const ReadTool = Tool.define( // kilocode_change start export async function lines(filepath: string, opts: { limit: number; offset: number }) { - const encoded = await Encoding.read(filepath) - const stream = Readable.from([encoded.text]) + // kilocode_change end + const encoded = await Encoding.read(filepath) // kilocode_change - decode with detected encoding + const stream = Readable.from([encoded.text]) // kilocode_change - replaces createReadStream const rl = createInterface({ input: stream, // Note: we use the crlfDelay option to recognize all instances of CR LF @@ -280,10 +281,10 @@ export async function lines(filepath: string, opts: { limit: number; offset: num return { raw, count, cut, more, offset: opts.offset } } -// kilocode_change end // kilocode_change start export async function isBinaryFile(filepath: string, fileSize: number): Promise { + // kilocode_change end const ext = path.extname(filepath).toLowerCase() // binary check for common non-text extensions switch (ext) { @@ -329,12 +330,11 @@ export async function isBinaryFile(filepath: string, fileSize: number): Promise< const result = await fh.read(bytes, 0, sampleSize, 0) if (result.bytesRead === 0) return false - // If encoding detection identifies a known text encoding (including UTF-16 - // LE/BE with BOM or CJK), it's text — not binary. This prevents UTF-16 - // files with legitimate null bytes from being falsely rejected. + // kilocode_change start - treat detected non-UTF-8 text (CJK, UTF-16 with BOM) as text, not binary const sample = bytes.subarray(0, result.bytesRead) const enc = Encoding.detect(sample) if (enc !== "utf-8") return false + // kilocode_change end let nonPrintableCount = 0 for (let i = 0; i < result.bytesRead; i++) { @@ -349,4 +349,3 @@ export async function isBinaryFile(filepath: string, fileSize: number): Promise< await fh.close() } } -// kilocode_change end diff --git a/packages/opencode/src/tool/write.ts b/packages/opencode/src/tool/write.ts index 289d09c872..bf1c27dfe7 100644 --- a/packages/opencode/src/tool/write.ts +++ b/packages/opencode/src/tool/write.ts @@ -43,15 +43,9 @@ export const WriteTool = Tool.define( yield* assertExternalDirectoryEffect(ctx, filepath) const exists = yield* fs.existsSafe(filepath) - // kilocode_change start - preserve existing file encoding - let contentOld = "" - let encoding = "utf-8" - if (exists) { - const encoded = yield* EncodedIO.read(filepath) - contentOld = encoded.text - encoding = encoded.encoding - } - // kilocode_change end + const pre = exists ? yield* EncodedIO.read(filepath) : { text: "", encoding: "utf-8" } // kilocode_change + const contentOld = pre.text // kilocode_change + const encoding = pre.encoding // kilocode_change - preserve file encoding on write if (exists) yield* filetime.assert(ctx.sessionID, filepath) const diff = trimDiff(createTwoFilesPatch(filepath, filepath, contentOld, params.content)) diff --git a/packages/opencode/test/kilocode/tool-encoding.test.ts b/packages/opencode/test/kilocode/tool-encoding.test.ts index e05d8ff4ff..19a8b7e4b0 100644 --- a/packages/opencode/test/kilocode/tool-encoding.test.ts +++ b/packages/opencode/test/kilocode/tool-encoding.test.ts @@ -131,7 +131,6 @@ const samples = { eucKr: "안녕하세요, 세계! 한국어 테스트입니다.", windows1251: "Привет, мир! Это тест кириллицы.", koi8r: "Привет, мир! КОИ-8 Р тест.", - latin1: "Caf\u00e9 na\u00efve r\u00e9sum\u00e9 — \u00a3100", } describe("tool encoding preservation", () => { @@ -152,7 +151,7 @@ describe("tool encoding preservation", () => { for (const [label, encoding, text] of cases) { it.live(`decodes ${label} content for the model`, () => - providEncoded(encoding, text, (filepath) => + provideEncoded(encoding, text, (filepath) => Effect.gen(function* () { const result = yield* runRead({ filePath: filepath }) expect(result.output).toContain(text) @@ -164,7 +163,7 @@ describe("tool encoding preservation", () => { describe("ReadTool does not flag non-Latin text files as binary", () => { it.live("accepts Shift_JIS", () => - providEncoded("Shift_JIS", samples.shiftJis, (filepath) => + provideEncoded("Shift_JIS", samples.shiftJis, (filepath) => Effect.gen(function* () { const result = yield* runRead({ filePath: filepath }) expect(result.output).toContain(samples.shiftJis) @@ -173,7 +172,7 @@ describe("tool encoding preservation", () => { ) it.live("accepts UTF-16 LE with BOM (contains NUL bytes)", () => - providEncoded("utf-16le", samples.utf8, (filepath) => + provideEncoded("utf-16le", samples.utf8, (filepath) => Effect.gen(function* () { const result = yield* runRead({ filePath: filepath }) expect(result.output).toContain(samples.utf8) @@ -262,7 +261,9 @@ describe("tool encoding preservation", () => { provideTmpdirInstance((dir) => Effect.gen(function* () { const filepath = path.join(dir, "doc.txt") + const replacement = "日本語" const original = "line1\n" + samples.shiftJis + "\nline3\n" + const expected = original.replace(samples.shiftJis, replacement) yield* putEncoded(filepath, original, "Shift_JIS") const patch = [ @@ -271,7 +272,7 @@ describe("tool encoding preservation", () => { "@@", " line1", "-" + samples.shiftJis, - "+" + samples.eucJp.replace(/[^\u3000-\u30ff\u4e00-\u9fff]/g, ""), + "+" + replacement, " line3", "*** End Patch", ].join("\n") @@ -279,12 +280,11 @@ describe("tool encoding preservation", () => { yield* runPatch({ patchText: patch }) const decoded = yield* loadDecoded(filepath, "Shift_JIS") - expect(decoded).toContain("line1") - expect(decoded).toContain("line3") + expect(decoded).toBe(expected) - // File must still decode as Shift_JIS — UTF-8 bytes would mojibake here. + // Bytes must still be Shift_JIS, not silently promoted to UTF-8. const bytes = yield* loadBytes(filepath) - expect(bytes.includes(Buffer.from([0xe3, 0x81]))).toBe(false) + expect(bytes.equals(encodeBytes(expected, "Shift_JIS"))).toBe(true) }), ), ) @@ -303,7 +303,7 @@ describe("tool encoding preservation", () => { }) // Shared helper to set up a temp instance with an encoded file at `file.txt`. -function providEncoded(encoding: string, text: string, body: (filepath: string) => Effect.Effect) { +function provideEncoded(encoding: string, text: string, body: (filepath: string) => Effect.Effect) { return provideTmpdirInstance((dir) => Effect.gen(function* () { const filepath = path.join(dir, "file.txt") From dfbdc97cd38e962d2c8ea2233737a45071e2f359 Mon Sep 17 00:00:00 2001 From: "kiloconnect[bot]" <240665456+kiloconnect[bot]@users.noreply.github.com> Date: Wed, 22 Apr 2026 13:04:30 +0000 Subject: [PATCH 20/24] refactor(cli): simplify isBinaryFile check to UTF-16 BOM only Per review: CJK and legacy single-byte encodings already passed the old control-char heuristic (their bytes are all >= 0x80, never NUL), so the full Encoding.detect pass in isBinaryFile was solving a non-problem for them. The only realistic regression in the old heuristic is UTF-16 with BOM, where the second byte of every ASCII character is 0x00 and fires the NUL-byte early-return. Replace the detect call with a 2-byte BOM check via a new Encoding.hasUtf16Bom helper. Also reword the move/add inline markers in patch/index.ts to note that Encoding.write handles mkdir. --- packages/opencode/src/kilocode/encoding.ts | 6 ++++++ packages/opencode/src/patch/index.ts | 2 +- packages/opencode/src/tool/read.ts | 6 ++---- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/packages/opencode/src/kilocode/encoding.ts b/packages/opencode/src/kilocode/encoding.ts index 9345dd56a9..f846da9871 100644 --- a/packages/opencode/src/kilocode/encoding.ts +++ b/packages/opencode/src/kilocode/encoding.ts @@ -38,6 +38,12 @@ export namespace Encoding { return bytes.length >= 3 && bytes[0] === 0xef && bytes[1] === 0xbb && bytes[2] === 0xbf } + /** True if `bytes[0..limit]` starts with a UTF-16 LE or BE byte-order mark. */ + export function hasUtf16Bom(bytes: Buffer, limit = bytes.length): boolean { + if (limit < 2) return false + return (bytes[0] === 0xff && bytes[1] === 0xfe) || (bytes[0] === 0xfe && bytes[1] === 0xff) + } + /** Remap jschardet labels to iconv-lite compatible names. */ function normalize(name: string): string { const lower = name.toLowerCase().replace(/[^a-z0-9]/g, "") diff --git a/packages/opencode/src/patch/index.ts b/packages/opencode/src/patch/index.ts index 323ce403fc..20c46de136 100644 --- a/packages/opencode/src/patch/index.ts +++ b/packages/opencode/src/patch/index.ts @@ -549,7 +549,7 @@ export namespace Patch { if (hunk.move_path) { // Handle file move - await Encoding.write(hunk.move_path, fileUpdate.content, fileUpdate.encoding) // kilocode_change + await Encoding.write(hunk.move_path, fileUpdate.content, fileUpdate.encoding) // kilocode_change - encoding-aware write (mkdirs) await fs.unlink(hunk.path) modified.push(hunk.move_path) log.info(`Moved file: ${hunk.path} -> ${hunk.move_path}`) diff --git a/packages/opencode/src/tool/read.ts b/packages/opencode/src/tool/read.ts index 5d596fd4ec..2dfb1f6587 100644 --- a/packages/opencode/src/tool/read.ts +++ b/packages/opencode/src/tool/read.ts @@ -330,10 +330,8 @@ export async function isBinaryFile(filepath: string, fileSize: number): Promise< const result = await fh.read(bytes, 0, sampleSize, 0) if (result.bytesRead === 0) return false - // kilocode_change start - treat detected non-UTF-8 text (CJK, UTF-16 with BOM) as text, not binary - const sample = bytes.subarray(0, result.bytesRead) - const enc = Encoding.detect(sample) - if (enc !== "utf-8") return false + // kilocode_change start - UTF-16 BOM: NUL bytes are legitimate, skip the NUL/control-char heuristic + if (Encoding.hasUtf16Bom(bytes, result.bytesRead)) return false // kilocode_change end let nonPrintableCount = 0 From 8c47af8af402f2251ed32e17810db71f40e707b9 Mon Sep 17 00:00:00 2001 From: "kiloconnect[bot]" <240665456+kiloconnect[bot]@users.noreply.github.com> Date: Wed, 22 Apr 2026 15:59:41 +0000 Subject: [PATCH 21/24] refactor(cli): reapply encoding changes after main merge Main moved the Patch namespace out of packages/opencode/src/patch/index.ts into patch/patch.ts and switched tool imports to the new @opencode-ai/shared/filesystem and ../../src/tool barrels. Reapply the encoding-aware read/write changes on top of the refactored patch.ts, and update the integration test imports to match the new module layout. --- packages/opencode/test/kilocode/tool-encoding.test.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/opencode/test/kilocode/tool-encoding.test.ts b/packages/opencode/test/kilocode/tool-encoding.test.ts index 19a8b7e4b0..2ac56a8d00 100644 --- a/packages/opencode/test/kilocode/tool-encoding.test.ts +++ b/packages/opencode/test/kilocode/tool-encoding.test.ts @@ -9,7 +9,7 @@ import path from "path" import fs from "fs/promises" import iconv from "iconv-lite" import { Agent } from "../../src/agent/agent" -import { AppFileSystem } from "../../src/filesystem" +import { AppFileSystem } from "@opencode-ai/shared/filesystem" import { ApplyPatchTool } from "../../src/tool/apply_patch" import { Bus } from "../../src/bus" import * as CrossSpawnSpawner from "../../src/effect/cross-spawn-spawner" @@ -21,8 +21,8 @@ import { Instruction } from "../../src/session/instruction" import { LSP } from "../../src/lsp" import { MessageID, SessionID } from "../../src/session/schema" import { ReadTool } from "../../src/tool/read" -import { Tool } from "../../src/tool/tool" -import { Truncate } from "../../src/tool/truncate" +import * as Tool from "../../src/tool/tool" +import { Truncate } from "../../src/tool" import { WriteTool } from "../../src/tool/write" import { provideTmpdirInstance } from "../fixture/fixture" import { testEffect } from "../lib/effect" From 8eaff58330992d56e0c0a4177c22efd954b12326 Mon Sep 17 00:00:00 2001 From: "kiloconnect[bot]" <240665456+kiloconnect[bot]@users.noreply.github.com> Date: Wed, 22 Apr 2026 16:00:19 +0000 Subject: [PATCH 22/24] chore: restore .kilo/.kilocode lockfiles that main re-tracked --- .kilo/package-lock.json | 115 ++++++++++++++++++++++++++++++++++++ .kilocode/package-lock.json | 115 ++++++++++++++++++++++++++++++++++++ 2 files changed, 230 insertions(+) create mode 100644 .kilo/package-lock.json create mode 100644 .kilocode/package-lock.json diff --git a/.kilo/package-lock.json b/.kilo/package-lock.json new file mode 100644 index 0000000000..f78a49868c --- /dev/null +++ b/.kilo/package-lock.json @@ -0,0 +1,115 @@ +{ + "name": ".kilo", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "dependencies": { + "@kilocode/plugin": "7.2.14" + } + }, + "node_modules/@kilocode/plugin": { + "version": "7.2.14", + "resolved": "https://registry.npmjs.org/@kilocode/plugin/-/plugin-7.2.14.tgz", + "integrity": "sha512-mS+WA9HZIBH2qQ9ARA+v0q4MdQTSdfOvKbe4AOSkjP+P5hVA70OM/UVM9DVcvmjSOxU+wuUxmOy+j/EQIrgFmw==", + "license": "MIT", + "dependencies": { + "@kilocode/sdk": "7.2.14", + "zod": "4.1.8" + }, + "peerDependencies": { + "@opentui/core": ">=0.1.97", + "@opentui/solid": ">=0.1.97" + }, + "peerDependenciesMeta": { + "@opentui/core": { + "optional": true + }, + "@opentui/solid": { + "optional": true + } + } + }, + "node_modules/@kilocode/sdk": { + "version": "7.2.14", + "resolved": "https://registry.npmjs.org/@kilocode/sdk/-/sdk-7.2.14.tgz", + "integrity": "sha512-Naz83lFrsbavuDp6UwxRuglOaSNvRBsZfcRNvb7RpWYAwbuJP0dBdhpXj6uO3ta5qxeQ2JzxKNC9Ffz+LCLLDg==", + "license": "MIT", + "dependencies": { + "cross-spawn": "7.0.6" + } + }, + "node_modules/cross-spawn": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", + "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==", + "license": "MIT", + "dependencies": { + "path-key": "^3.1.0", + "shebang-command": "^2.0.0", + "which": "^2.0.1" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/isexe": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", + "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", + "license": "ISC" + }, + "node_modules/path-key": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", + "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/shebang-command": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", + "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", + "license": "MIT", + "dependencies": { + "shebang-regex": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/shebang-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", + "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/which": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", + "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", + "license": "ISC", + "dependencies": { + "isexe": "^2.0.0" + }, + "bin": { + "node-which": "bin/node-which" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/zod": { + "version": "4.1.8", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/colinhacks" + } + } + } +} diff --git a/.kilocode/package-lock.json b/.kilocode/package-lock.json new file mode 100644 index 0000000000..65452b7e4d --- /dev/null +++ b/.kilocode/package-lock.json @@ -0,0 +1,115 @@ +{ + "name": ".kilocode", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "dependencies": { + "@kilocode/plugin": "7.2.14" + } + }, + "node_modules/@kilocode/plugin": { + "version": "7.2.14", + "resolved": "https://registry.npmjs.org/@kilocode/plugin/-/plugin-7.2.14.tgz", + "integrity": "sha512-mS+WA9HZIBH2qQ9ARA+v0q4MdQTSdfOvKbe4AOSkjP+P5hVA70OM/UVM9DVcvmjSOxU+wuUxmOy+j/EQIrgFmw==", + "license": "MIT", + "dependencies": { + "@kilocode/sdk": "7.2.14", + "zod": "4.1.8" + }, + "peerDependencies": { + "@opentui/core": ">=0.1.97", + "@opentui/solid": ">=0.1.97" + }, + "peerDependenciesMeta": { + "@opentui/core": { + "optional": true + }, + "@opentui/solid": { + "optional": true + } + } + }, + "node_modules/@kilocode/sdk": { + "version": "7.2.14", + "resolved": "https://registry.npmjs.org/@kilocode/sdk/-/sdk-7.2.14.tgz", + "integrity": "sha512-Naz83lFrsbavuDp6UwxRuglOaSNvRBsZfcRNvb7RpWYAwbuJP0dBdhpXj6uO3ta5qxeQ2JzxKNC9Ffz+LCLLDg==", + "license": "MIT", + "dependencies": { + "cross-spawn": "7.0.6" + } + }, + "node_modules/cross-spawn": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", + "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==", + "license": "MIT", + "dependencies": { + "path-key": "^3.1.0", + "shebang-command": "^2.0.0", + "which": "^2.0.1" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/isexe": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", + "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", + "license": "ISC" + }, + "node_modules/path-key": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", + "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/shebang-command": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", + "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", + "license": "MIT", + "dependencies": { + "shebang-regex": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/shebang-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", + "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/which": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", + "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", + "license": "ISC", + "dependencies": { + "isexe": "^2.0.0" + }, + "bin": { + "node-which": "bin/node-which" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/zod": { + "version": "4.1.8", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/colinhacks" + } + } + } +} From e84ea3afc677e2a18f26e2d2168e7122edfb41a9 Mon Sep 17 00:00:00 2001 From: "kiloconnect[bot]" <240665456+kiloconnect[bot]@users.noreply.github.com> Date: Wed, 22 Apr 2026 16:04:12 +0000 Subject: [PATCH 23/24] refactor(cli): self-review post-merge cleanup in apply_patch - Drop the dead 'encoding = pre.encoding' initial value that was only read if deriveNewContentsFromChunks threw, and that code path returns before encoding is ever used. - Inline pre.text into the read line so we don't bind a 'pre' local just to destructure it once. --- packages/opencode/src/tool/apply_patch.ts | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/packages/opencode/src/tool/apply_patch.ts b/packages/opencode/src/tool/apply_patch.ts index d5e9ea3247..d6a1be855e 100644 --- a/packages/opencode/src/tool/apply_patch.ts +++ b/packages/opencode/src/tool/apply_patch.ts @@ -109,10 +109,9 @@ export const ApplyPatchTool = Tool.define( ) } - const pre = yield* EncodedIO.read(filePath) // kilocode_change - preserve file encoding - const oldContent = pre.text // kilocode_change - let encoding = pre.encoding // kilocode_change + const oldContent = (yield* EncodedIO.read(filePath)).text // kilocode_change - encoding-aware read let newContent = oldContent + let encoding: string // kilocode_change - filled in by the patch helper below // Apply the update chunks to get new content try { From eb52bda015f268f676dab40fa97ff8a5a6208d1a Mon Sep 17 00:00:00 2001 From: "kiloconnect[bot]" <240665456+kiloconnect[bot]@users.noreply.github.com> Date: Thu, 23 Apr 2026 10:04:01 +0000 Subject: [PATCH 24/24] fix(cli): prevent double BOM when content starts with U+FEFF Strip a leading U+FEFF from text before prepending a UTF-8/UTF-16 BOM so round-tripping content that already carries a BOM character emits exactly one BOM. Also clarifies the changeset that new files default to UTF-8. --- .changeset/preserve-file-encoding.md | 2 +- packages/opencode/src/kilocode/encoding.ts | 11 ++++--- .../test/kilocode/tool-encoding.test.ts | 29 +++++++++++++++++++ 3 files changed, 37 insertions(+), 5 deletions(-) diff --git a/.changeset/preserve-file-encoding.md b/.changeset/preserve-file-encoding.md index 034e6540f3..2bbfe07f03 100644 --- a/.changeset/preserve-file-encoding.md +++ b/.changeset/preserve-file-encoding.md @@ -2,7 +2,7 @@ "@kilocode/cli": minor --- -The agent now detects and preserves the original text encoding of files when reading and editing them, so non-UTF-8 files are displayed correctly to the model and written back in their original encoding. +The agent now detects and preserves the original text encoding of files when reading and editing them, so non-UTF-8 files are displayed correctly to the model and written back in their original encoding. New files are still created as UTF-8 without BOM — detection only applies when overwriting or editing an existing file. Supported: UTF-8 (with or without BOM), UTF-16 with BOM, and common legacy Latin and CJK encodings (Shift_JIS, EUC-JP, GB2312, Big5, EUC-KR, Windows-1251, KOI8-R, ISO-8859, and others). diff --git a/packages/opencode/src/kilocode/encoding.ts b/packages/opencode/src/kilocode/encoding.ts index f846da9871..70fc51330e 100644 --- a/packages/opencode/src/kilocode/encoding.ts +++ b/packages/opencode/src/kilocode/encoding.ts @@ -109,11 +109,14 @@ export namespace Encoding { export function encode(text: string, encoding: string): Buffer { // iconv-lite's UTF codecs strip/ignore BOMs, but we support "UTF-X with BOM" // as a distinct variant. Prepend the BOM manually so round-tripping keeps - // the original byte signature intact. - if (encoding === UTF8_BOM) return Buffer.concat([UTF8_BOM_BYTES, iconv.encode(text, "utf-8")]) + // the original byte signature intact. Strip a leading U+FEFF from `text` + // first so we never emit a double BOM when the decoded text already + // contains one (e.g. if a tool round-trips content verbatim). + const body = text.charCodeAt(0) === 0xfeff ? text.slice(1) : text + if (encoding === UTF8_BOM) return Buffer.concat([UTF8_BOM_BYTES, iconv.encode(body, "utf-8")]) const lower = encoding.toLowerCase() - if (lower === "utf-16le") return Buffer.concat([Buffer.from([0xff, 0xfe]), iconv.encode(text, encoding)]) - if (lower === "utf-16be") return Buffer.concat([Buffer.from([0xfe, 0xff]), iconv.encode(text, encoding)]) + if (lower === "utf-16le") return Buffer.concat([Buffer.from([0xff, 0xfe]), iconv.encode(body, encoding)]) + if (lower === "utf-16be") return Buffer.concat([Buffer.from([0xfe, 0xff]), iconv.encode(body, encoding)]) return iconv.encode(text, encoding) } diff --git a/packages/opencode/test/kilocode/tool-encoding.test.ts b/packages/opencode/test/kilocode/tool-encoding.test.ts index 2ac56a8d00..1e03493fd6 100644 --- a/packages/opencode/test/kilocode/tool-encoding.test.ts +++ b/packages/opencode/test/kilocode/tool-encoding.test.ts @@ -223,6 +223,35 @@ describe("tool encoding preservation", () => { }), ), ) + + // Guard against double-BOM regressions: if the model ever hands back content + // that already starts with U+FEFF (e.g. by round-tripping literal bytes), + // writing it to a BOM-encoded file must still produce exactly one BOM. + const bomCases: Array<[string, string, Buffer]> = [ + ["UTF-8 with BOM", UTF8_BOM, Buffer.from([0xef, 0xbb, 0xbf])], + ["UTF-16 LE", "utf-16le", Buffer.from([0xff, 0xfe])], + ["UTF-16 BE", "utf-16be", Buffer.from([0xfe, 0xff])], + ] + for (const [label, encoding, bom] of bomCases) { + it.live(`does not emit a double BOM for ${label} when content starts with U+FEFF`, () => + provideTmpdirInstance((dir) => + Effect.gen(function* () { + const filepath = path.join(dir, "file.txt") + yield* putEncoded(filepath, "hello", encoding) + yield* markRead(filepath) + + yield* runWrite({ filePath: filepath, content: "\uFEFFgoodbye" }) + + const bytes = yield* loadBytes(filepath) + // Exactly one BOM prefix, immediately followed by encoded payload. + expect(bytes.subarray(0, bom.length).equals(bom)).toBe(true) + expect(bytes.subarray(bom.length, bom.length * 2).equals(bom)).toBe(false) + const decoded = yield* loadDecoded(filepath, encoding) + expect(decoded).toBe("goodbye") + }), + ), + ) + } }) describe("EditTool preserves existing file encoding across edits", () => {