From 9ffd047962039d6b73d301d5d4e67560cd501c4f Mon Sep 17 00:00:00 2001 From: kilo-agent Date: Tue, 5 May 2026 13:02:21 +0000 Subject: [PATCH] feat(cli): support UTF-32 with BOM for file encoding detection UTF-16 and UTF-32 without a BOM remain unsupported (ambiguous). Aligns with upstream opencode PR chrarnoldus/opencode#1. --- .changeset/utf-32-bom-encoding.md | 6 + .../code-with-ai/features/file-encoding.md | 4 +- packages/opencode/src/kilocode/encoding.ts | 31 ++++- packages/opencode/src/tool/read.ts | 6 +- .../opencode/test/kilocode/encoding.test.ts | 113 +++++++++++++++++- .../test/kilocode/tool-encoding.test.ts | 18 +++ 6 files changed, 163 insertions(+), 15 deletions(-) create mode 100644 .changeset/utf-32-bom-encoding.md diff --git a/.changeset/utf-32-bom-encoding.md b/.changeset/utf-32-bom-encoding.md new file mode 100644 index 00000000000..176380079fc --- /dev/null +++ b/.changeset/utf-32-bom-encoding.md @@ -0,0 +1,6 @@ +--- +"kilo-code": minor +"@kilocode/cli": minor +--- + +Detect and preserve UTF-32 (LE and BE) with BOM when reading and editing files. UTF-16 and UTF-32 without a BOM remain unsupported. diff --git a/packages/kilo-docs/pages/code-with-ai/features/file-encoding.md b/packages/kilo-docs/pages/code-with-ai/features/file-encoding.md index a6d22e2c3cd..416d8cac39a 100644 --- a/packages/kilo-docs/pages/code-with-ai/features/file-encoding.md +++ b/packages/kilo-docs/pages/code-with-ai/features/file-encoding.md @@ -11,6 +11,7 @@ Kilo automatically detects the text encoding of each file it reads and preserves - UTF-8, with or without BOM - UTF-16 LE and UTF-16 BE, **with a BOM** +- UTF-32 LE and UTF-32 BE, **with a BOM** - Shift_JIS, EUC-JP, GB2312, Big5, EUC-KR - Windows-1251, KOI8-R - The ISO-8859 family @@ -20,8 +21,7 @@ New files Kilo creates are always UTF-8 without a BOM. Encoding detection only r ## Not Supported -- **UTF-16 without a BOM.** The byte pattern is ambiguous and cannot be distinguished reliably from other encodings. Save the file with a BOM or convert it to UTF-8. -- **UTF-32.** Extremely rare in practice; convert to UTF-8 if you need Kilo to work with it. +- **UTF-16 or UTF-32 without a BOM.** The byte pattern is ambiguous and cannot be distinguished reliably from other encodings. Save the file with a BOM or convert it to UTF-8. {% callout type="info" %} Encoding detection is statistical. Very short files, or files whose byte patterns happen to look like a different encoding, may occasionally be misidentified. If that happens, converting the file to UTF-8 is the most reliable workaround. diff --git a/packages/opencode/src/kilocode/encoding.ts b/packages/opencode/src/kilocode/encoding.ts index 70fc51330e6..92ac12fb831 100644 --- a/packages/opencode/src/kilocode/encoding.ts +++ b/packages/opencode/src/kilocode/encoding.ts @@ -10,16 +10,17 @@ import iconv from "iconv-lite" * Supported: * - UTF-8 (with or without BOM) * - UTF-16 LE/BE with BOM (detected by jschardet) + * - UTF-32 LE/BE with BOM (detected by jschardet) * - Legacy Latin and CJK encodings (detected by jschardet) * * Not supported: - * - UTF-16 without BOM (ambiguous, rare) - * - UTF-32 (extremely rare) + * - UTF-16 or UTF-32 without BOM (ambiguous, rare) * * Detection strategy: * 1. If the bytes are valid UTF-8, treat as UTF-8 (tracking the presence of a * BOM so it can be written back). - * 2. Otherwise, trust jschardet. + * 2. Otherwise, trust jschardet. jschardet only reports the wide UTF variants + * when a BOM is present, which aligns with the contract above. * * iconv-lite's UTF codecs strip BOMs on decode and do not emit them on encode, * so UTF BOMs are handled explicitly in {@link encode} to round-trip cleanly. @@ -41,9 +42,19 @@ export namespace Encoding { /** True if `bytes[0..limit]` starts with a UTF-16 LE or BE byte-order mark. */ export function hasUtf16Bom(bytes: Buffer, limit = bytes.length): boolean { if (limit < 2) return false + // UTF-32 LE starts with FF FE 00 00, so exclude that to avoid misclassifying it as UTF-16 LE. + if (hasUtf32Bom(bytes, limit)) return false return (bytes[0] === 0xff && bytes[1] === 0xfe) || (bytes[0] === 0xfe && bytes[1] === 0xff) } + /** True if `bytes[0..limit]` starts with a UTF-32 LE or BE byte-order mark. */ + export function hasUtf32Bom(bytes: Buffer, limit = bytes.length): boolean { + if (limit < 4) return false + const le = bytes[0] === 0xff && bytes[1] === 0xfe && bytes[2] === 0x00 && bytes[3] === 0x00 + const be = bytes[0] === 0x00 && bytes[1] === 0x00 && bytes[2] === 0xfe && bytes[3] === 0xff + return le || be + } + /** Remap jschardet labels to iconv-lite compatible names. */ function normalize(name: string): string { const lower = name.toLowerCase().replace(/[^a-z0-9]/g, "") @@ -51,6 +62,8 @@ export namespace Encoding { utf8: "utf-8", utf16le: "utf-16le", utf16be: "utf-16be", + utf32le: "utf-32le", + utf32be: "utf-32be", ascii: "utf-8", iso88591: "iso-8859-1", iso88592: "iso-8859-2", @@ -95,8 +108,12 @@ export namespace Encoding { const result = jschardet.detect(bytes) if (!result.encoding) return DEFAULT const enc = normalize(result.encoding) - // Reject unsupported Unicode encodings (UTF-32 and anything iconv-lite cannot decode) - if (enc.toLowerCase().startsWith("utf-32")) return DEFAULT + const lower = enc.toLowerCase() + // Wide UTF variants are only accepted when a BOM is present. jschardet already + // enforces this in practice (it reports "ascii" for BOM-less samples), but gate + // explicitly so a future detector change can't silently promote ambiguous bytes. + if ((lower === "utf-16le" || lower === "utf-16be") && !hasUtf16Bom(bytes)) return DEFAULT + if ((lower === "utf-32le" || lower === "utf-32be") && !hasUtf32Bom(bytes)) return DEFAULT if (!iconv.encodingExists(enc)) return DEFAULT return enc } @@ -117,6 +134,10 @@ export namespace Encoding { const lower = encoding.toLowerCase() if (lower === "utf-16le") return Buffer.concat([Buffer.from([0xff, 0xfe]), iconv.encode(body, encoding)]) if (lower === "utf-16be") return Buffer.concat([Buffer.from([0xfe, 0xff]), iconv.encode(body, encoding)]) + if (lower === "utf-32le") + return Buffer.concat([Buffer.from([0xff, 0xfe, 0x00, 0x00]), iconv.encode(body, encoding)]) + if (lower === "utf-32be") + return Buffer.concat([Buffer.from([0x00, 0x00, 0xfe, 0xff]), iconv.encode(body, encoding)]) return iconv.encode(text, encoding) } diff --git a/packages/opencode/src/tool/read.ts b/packages/opencode/src/tool/read.ts index 3f54597c98b..a02ab17fc00 100644 --- a/packages/opencode/src/tool/read.ts +++ b/packages/opencode/src/tool/read.ts @@ -143,9 +143,9 @@ export const ReadTool = Tool.define( if (bytes.length === 0) return false - // kilocode_change start - UTF-16 BOM: NUL bytes are legitimate, skip the NUL/control-char heuristic - if (Encoding.hasUtf16Bom(Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength), bytes.length)) - return false + // kilocode_change start - UTF-16/32 BOM: NUL bytes are legitimate, skip the NUL/control-char heuristic + const buf = Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength) + if (Encoding.hasUtf16Bom(buf, bytes.length) || Encoding.hasUtf32Bom(buf, bytes.length)) return false // kilocode_change end let nonPrintableCount = 0 diff --git a/packages/opencode/test/kilocode/encoding.test.ts b/packages/opencode/test/kilocode/encoding.test.ts index bd00379c12f..a5fad72d928 100644 --- a/packages/opencode/test/kilocode/encoding.test.ts +++ b/packages/opencode/test/kilocode/encoding.test.ts @@ -17,6 +17,8 @@ const BOM = { utf8: Buffer.from([0xef, 0xbb, 0xbf]), utf16le: Buffer.from([0xff, 0xfe]), utf16be: Buffer.from([0xfe, 0xff]), + utf32le: Buffer.from([0xff, 0xfe, 0x00, 0x00]), + utf32be: Buffer.from([0x00, 0x00, 0xfe, 0xff]), } async function tmp(body: (dir: string) => Promise): Promise { @@ -65,14 +67,37 @@ describe("Encoding.detect", () => { expect(Encoding.detect(bytes)).toBe("utf-16be") }) - test("UTF-32 (detected by jschardet) is rejected and falls back to utf-8", () => { - // Build a sample starting with a UTF-32 LE BOM. jschardet will report - // UTF-32*; the namespace explicitly strips that because iconv-lite can't - // round-trip it. - const bytes = Buffer.concat([Buffer.from([0xff, 0xfe, 0x00, 0x00]), Buffer.alloc(32)]) + test("UTF-32 LE with BOM detects as utf-32le", () => { + const bytes = Buffer.concat([BOM.utf32le, iconv.encode("hello world", "utf-32le")]) + expect(Encoding.detect(bytes)).toBe("utf-32le") + }) + + test("UTF-32 BE with BOM detects as utf-32be", () => { + const bytes = Buffer.concat([BOM.utf32be, iconv.encode("hello world", "utf-32be")]) + expect(Encoding.detect(bytes)).toBe("utf-32be") + }) + + test("UTF-32 without BOM falls back to utf-8 (contract: wide UTF requires BOM)", () => { + // iconv-produced UTF-32 bytes without any BOM prefix. jschardet reports + // "ascii" for these because the buffer is dominated by NUL bytes that + // coincidentally look like padded ASCII, so detect() falls back to utf-8. + // The important contract is that we do NOT silently promote to utf-32*. + const bytes = iconv.encode("hello", "utf-32le") expect(Encoding.detect(bytes)).toBe(Encoding.DEFAULT) }) + test("UTF-16 without BOM falls back to utf-8 (contract: wide UTF requires BOM)", () => { + const bytes = iconv.encode("hello", "utf-16le") + expect(Encoding.detect(bytes)).toBe(Encoding.DEFAULT) + }) + + test("UTF-32 LE BOM is not misdetected as UTF-16 LE (shared FF FE prefix)", () => { + // UTF-32 LE BOM is FF FE 00 00; its first two bytes are identical to the + // UTF-16 LE BOM, so the order of BOM checks matters. + const bytes = Buffer.concat([BOM.utf32le, iconv.encode("hi", "utf-32le")]) + expect(Encoding.detect(bytes)).toBe("utf-32le") + }) + test("Shift_JIS bytes detect as Shift_JIS (case-insensitive, iconv-compatible label)", () => { const bytes = iconv.encode("こんにちは、世界！日本語のテストです。", "Shift_JIS") const detected = Encoding.detect(bytes) @@ -94,6 +119,8 @@ describe("Encoding.decode / Encoding.encode", () => { ["utf-8-bom synthetic label", Encoding.UTF8_BOM, "hello"], ["utf-16le", "utf-16le", "Hello 世界"], ["utf-16be", "utf-16be", "Hello 世界"], + ["utf-32le", "utf-32le", "Hello 世界"], + ["utf-32be", "utf-32be", "Hello 世界"], ["Shift_JIS", "Shift_JIS", "日本語"], ["windows-1251", "windows-1251", "Привет"], ["gb2312", "gb2312", "你好"], @@ -134,6 +161,25 @@ describe("Encoding.decode / Encoding.encode", () => { expect(bytes[3]).toBe(0x68) }) + test("utf-32le encode emits exactly one BOM even if input starts with U+FEFF", () => { + const bytes = Encoding.encode("\uFEFFhi", "utf-32le") + expect(bytes.subarray(0, 4).equals(BOM.utf32le)).toBe(true) + // Next four bytes must be the 'h' code point (0x68 0x00 0x00 0x00), not another BOM. + expect(bytes[4]).toBe(0x68) + expect(bytes[5]).toBe(0x00) + expect(bytes[6]).toBe(0x00) + expect(bytes[7]).toBe(0x00) + }) + + test("utf-32be encode emits exactly one BOM even if input starts with U+FEFF", () => { + const bytes = Encoding.encode("\uFEFFhi", "utf-32be") + expect(bytes.subarray(0, 4).equals(BOM.utf32be)).toBe(true) + expect(bytes[4]).toBe(0x00) + expect(bytes[5]).toBe(0x00) + expect(bytes[6]).toBe(0x00) + expect(bytes[7]).toBe(0x68) + }) + test("decode of utf-8-bom produces text without leading U+FEFF", () => { // iconv-lite's utf-8 codec is documented to strip BOMs; guard against // regressions if the underlying behaviour changes. @@ -165,6 +211,37 @@ describe("Encoding.hasUtf16Bom", () => { test("returns false for a one-byte buffer", () => { expect(Encoding.hasUtf16Bom(Buffer.from([0xff]))).toBe(false) }) + test("returns false for a UTF-32 LE BOM (distinct encoding)", () => { + // UTF-32 LE starts with FF FE 00 00; a naive check on the first two bytes + // would misclassify it as UTF-16 LE. hasUtf16Bom must reject it so the + // caller can disambiguate. + expect(Encoding.hasUtf16Bom(BOM.utf32le)).toBe(false) + }) +}) + +describe("Encoding.hasUtf32Bom", () => { + test("detects LE BOM (FF FE 00 00)", () => { + expect(Encoding.hasUtf32Bom(BOM.utf32le)).toBe(true) + }) + test("detects BE BOM (00 00 FE FF)", () => { + expect(Encoding.hasUtf32Bom(BOM.utf32be)).toBe(true) + }) + test("returns false for UTF-16 LE BOM (shorter, distinct encoding)", () => { + expect(Encoding.hasUtf32Bom(BOM.utf16le)).toBe(false) + }) + test("returns false for UTF-16 BE BOM", () => { + expect(Encoding.hasUtf32Bom(BOM.utf16be)).toBe(false) + }) + test("returns false for UTF-8 BOM", () => { + expect(Encoding.hasUtf32Bom(BOM.utf8)).toBe(false) + }) + test("returns false for a three-byte buffer (too short)", () => { + expect(Encoding.hasUtf32Bom(Buffer.from([0xff, 0xfe, 0x00]))).toBe(false) + }) + test("respects an explicit limit smaller than the buffer", () => { + expect(Encoding.hasUtf32Bom(BOM.utf32le, 3)).toBe(false) + expect(Encoding.hasUtf32Bom(BOM.utf32le, 4)).toBe(true) + }) }) describe("Encoding.read / Encoding.readSync / Encoding.write", () => { @@ -242,4 +319,30 @@ describe("Encoding.read / Encoding.readSync / Encoding.write", () => { expect(result.text).toBe(text) }) }) + + test("write + read round-trips utf-32le with BOM", async () => { + await tmp(async (dir) => { + const filepath = path.join(dir, "u32.txt") + const text = "Hello 世界" + await Encoding.write(filepath, text, "utf-32le") + const bytes = await fs.readFile(filepath) + expect(bytes.subarray(0, 4).equals(BOM.utf32le)).toBe(true) + const result = await Encoding.read(filepath) + expect(result.encoding).toBe("utf-32le") + expect(result.text).toBe(text) + }) + }) + + test("write + read round-trips utf-32be with BOM", async () => { + await tmp(async (dir) => { + const filepath = path.join(dir, "u32be.txt") + const text = "Hello 世界" + await Encoding.write(filepath, text, "utf-32be") + const bytes = await fs.readFile(filepath) + expect(bytes.subarray(0, 4).equals(BOM.utf32be)).toBe(true) + const result = await Encoding.read(filepath) + expect(result.encoding).toBe("utf-32be") + expect(result.text).toBe(text) + }) + }) }) diff --git a/packages/opencode/test/kilocode/tool-encoding.test.ts b/packages/opencode/test/kilocode/tool-encoding.test.ts index c49b218e847..a200670d48b 100644 --- a/packages/opencode/test/kilocode/tool-encoding.test.ts +++ b/packages/opencode/test/kilocode/tool-encoding.test.ts @@ -94,6 +94,8 @@ const encodeBytes = (text: string, encoding: string): Buffer => { const lower = encoding.toLowerCase() if (lower === "utf-16le") return Buffer.concat([Buffer.from([0xff, 0xfe]), iconv.encode(text, encoding)]) if (lower === "utf-16be") return Buffer.concat([Buffer.from([0xfe, 0xff]), iconv.encode(text, encoding)]) + if (lower === "utf-32le") return Buffer.concat([Buffer.from([0xff, 0xfe, 0x00, 0x00]), iconv.encode(text, encoding)]) + if (lower === "utf-32be") return Buffer.concat([Buffer.from([0x00, 0x00, 0xfe, 0xff]), iconv.encode(text, encoding)]) return iconv.encode(text, encoding) } @@ -135,6 +137,8 @@ describe("tool encoding preservation", () => { ["UTF-8 with BOM", UTF8_BOM, samples.utf8], ["UTF-16 LE with BOM", "utf-16le", samples.utf8], ["UTF-16 BE with BOM", "utf-16be", samples.utf8], + ["UTF-32 LE with BOM", "utf-32le", samples.utf8], + ["UTF-32 BE with BOM", "utf-32be", samples.utf8], ["Shift_JIS", "Shift_JIS", samples.shiftJis], ["EUC-JP", "euc-jp", samples.eucJp], ["GB2312", "gb2312", samples.gb2312], @@ -174,6 +178,15 @@ describe("tool encoding preservation", () => { }), ), ) + + it.live("accepts UTF-32 LE with BOM (3 of every 4 bytes are NUL)", () => + provideEncoded("utf-32le", samples.utf8, (filepath) => + Effect.gen(function* () { + const result = yield* runRead({ filePath: filepath }) + expect(result.output).toContain(samples.utf8) + }), + ), + ) }) describe("WriteTool preserves existing file encoding when overwriting", () => { @@ -183,6 +196,8 @@ describe("tool encoding preservation", () => { ["GB2312", "gb2312", samples.gb2312], ["Windows-1251", "windows-1251", samples.windows1251], ["UTF-16 LE", "utf-16le", samples.utf8], + ["UTF-32 LE", "utf-32le", samples.utf8], + ["UTF-32 BE", "utf-32be", samples.utf8], ] for (const [label, encoding, original] of cases) { @@ -226,6 +241,8 @@ describe("tool encoding preservation", () => { ["UTF-8 with BOM", UTF8_BOM, Buffer.from([0xef, 0xbb, 0xbf])], ["UTF-16 LE", "utf-16le", Buffer.from([0xff, 0xfe])], ["UTF-16 BE", "utf-16be", Buffer.from([0xfe, 0xff])], + ["UTF-32 LE", "utf-32le", Buffer.from([0xff, 0xfe, 0x00, 0x00])], + ["UTF-32 BE", "utf-32be", Buffer.from([0x00, 0x00, 0xfe, 0xff])], ] for (const [label, encoding, bom] of bomCases) { it.live(`does not emit a double BOM for ${label} when content starts with U+FEFF`, () => @@ -256,6 +273,7 @@ describe("tool encoding preservation", () => { ["GB2312", "gb2312", samples.gb2312, "简体中文", "中文简体"], ["Windows-1251", "windows-1251", samples.windows1251, "мир", "планета"], ["UTF-16 LE", "utf-16le", samples.utf8 + "\n second line", "world", "earth"], + ["UTF-32 LE", "utf-32le", samples.utf8 + "\n second line", "world", "earth"], ] for (const [label, encoding, original, oldString, newString] of cases) {