diff --git a/.changeset/preserve-file-encoding.md b/.changeset/preserve-file-encoding.md index 034e6540f3..2bbfe07f03 100644 --- a/.changeset/preserve-file-encoding.md +++ b/.changeset/preserve-file-encoding.md @@ -2,7 +2,7 @@ "@kilocode/cli": minor --- -The agent now detects and preserves the original text encoding of files when reading and editing them, so non-UTF-8 files are displayed correctly to the model and written back in their original encoding. +The agent now detects and preserves the original text encoding of files when reading and editing them, so non-UTF-8 files are displayed correctly to the model and written back in their original encoding. New files are still created as UTF-8 without BOM — detection only applies when overwriting or editing an existing file. Supported: UTF-8 (with or without BOM), UTF-16 with BOM, and common legacy Latin and CJK encodings (Shift_JIS, EUC-JP, GB2312, Big5, EUC-KR, Windows-1251, KOI8-R, ISO-8859, and others). diff --git a/packages/opencode/src/kilocode/encoding.ts b/packages/opencode/src/kilocode/encoding.ts index f846da9871..70fc51330e 100644 --- a/packages/opencode/src/kilocode/encoding.ts +++ b/packages/opencode/src/kilocode/encoding.ts @@ -109,11 +109,14 @@ export namespace Encoding { export function encode(text: string, encoding: string): Buffer { // iconv-lite's UTF codecs strip/ignore BOMs, but we support "UTF-X with BOM" // as a distinct variant. Prepend the BOM manually so round-tripping keeps - // the original byte signature intact. - if (encoding === UTF8_BOM) return Buffer.concat([UTF8_BOM_BYTES, iconv.encode(text, "utf-8")]) + // the original byte signature intact. Strip a leading U+FEFF from `text` + // first so we never emit a double BOM when the decoded text already + // contains one (e.g. if a tool round-trips content verbatim). + const body = text.charCodeAt(0) === 0xfeff ? text.slice(1) : text + if (encoding === UTF8_BOM) return Buffer.concat([UTF8_BOM_BYTES, iconv.encode(body, "utf-8")]) const lower = encoding.toLowerCase() - if (lower === "utf-16le") return Buffer.concat([Buffer.from([0xff, 0xfe]), iconv.encode(text, encoding)]) - if (lower === "utf-16be") return Buffer.concat([Buffer.from([0xfe, 0xff]), iconv.encode(text, encoding)]) + if (lower === "utf-16le") return Buffer.concat([Buffer.from([0xff, 0xfe]), iconv.encode(body, encoding)]) + if (lower === "utf-16be") return Buffer.concat([Buffer.from([0xfe, 0xff]), iconv.encode(body, encoding)]) return iconv.encode(text, encoding) } diff --git a/packages/opencode/test/kilocode/tool-encoding.test.ts b/packages/opencode/test/kilocode/tool-encoding.test.ts index 2ac56a8d00..1e03493fd6 100644 --- a/packages/opencode/test/kilocode/tool-encoding.test.ts +++ b/packages/opencode/test/kilocode/tool-encoding.test.ts @@ -223,6 +223,35 @@ describe("tool encoding preservation", () => { }), ), ) + + // Guard against double-BOM regressions: if the model ever hands back content + // that already starts with U+FEFF (e.g. by round-tripping literal bytes), + // writing it to a BOM-encoded file must still produce exactly one BOM. + const bomCases: Array<[string, string, Buffer]> = [ + ["UTF-8 with BOM", UTF8_BOM, Buffer.from([0xef, 0xbb, 0xbf])], + ["UTF-16 LE", "utf-16le", Buffer.from([0xff, 0xfe])], + ["UTF-16 BE", "utf-16be", Buffer.from([0xfe, 0xff])], + ] + for (const [label, encoding, bom] of bomCases) { + it.live(`does not emit a double BOM for ${label} when content starts with U+FEFF`, () => + provideTmpdirInstance((dir) => + Effect.gen(function* () { + const filepath = path.join(dir, "file.txt") + yield* putEncoded(filepath, "hello", encoding) + yield* markRead(filepath) + + yield* runWrite({ filePath: filepath, content: "\uFEFFgoodbye" }) + + const bytes = yield* loadBytes(filepath) + // Exactly one BOM prefix, immediately followed by encoded payload. + expect(bytes.subarray(0, bom.length).equals(bom)).toBe(true) + expect(bytes.subarray(bom.length, bom.length * 2).equals(bom)).toBe(false) + const decoded = yield* loadDecoded(filepath, encoding) + expect(decoded).toBe("goodbye") + }), + ), + ) + } }) describe("EditTool preserves existing file encoding across edits", () => {