opensource/kilocode
3
0
Fork 0
mirror of https://github.com/Kilo-Org/kilocode.git synced 2026-08-30 17:14:40 +08:00
Code Issues Packages Projects Releases Wiki Activity

remove redundant wide-UTF BOM gate from detect()

Browse Source 
Users that hand us BOM-less UTF-16/32 are violating the documented
contract; trust jschardet + iconv.encodingExists and let the result
be whatever it is.
This commit is contained in:
kilo-agent
2026-05-05 13:06:57 +00:00
parent 9ffd047962
commit 988d080f96
2 changed files with 0 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/opencode/src/kilocode/encoding.ts
-6
View File
@@ -108,12 +108,6 @@ export namespace Encoding {
const result = jschardet.detect(bytes)
if (!result.encoding) return DEFAULT
const enc = normalize(result.encoding)
const lower = enc.toLowerCase()
// Wide UTF variants are only accepted when a BOM is present. jschardet already
// enforces this in practice (it reports "ascii" for BOM-less samples), but gate
// explicitly so a future detector change can't silently promote ambiguous bytes.
if ((lower === "utf-16le" || lower === "utf-16be") && !hasUtf16Bom(bytes)) return DEFAULT
if ((lower === "utf-32le" || lower === "utf-32be") && !hasUtf32Bom(bytes)) return DEFAULT
if (!iconv.encodingExists(enc)) return DEFAULT
return enc
}
packages/opencode/test/kilocode/encoding.test.ts
-14
View File
@@ -77,20 +77,6 @@ describe("Encoding.detect", () => {
expect(Encoding.detect(bytes)).toBe("utf-32be")
})
test("UTF-32 without BOM falls back to utf-8 (contract: wide UTF requires BOM)", () => {
// iconv-produced UTF-32 bytes without any BOM prefix. jschardet reports
// "ascii" for these because the buffer is dominated by NUL bytes that
// coincidentally look like padded ASCII, so detect() falls back to utf-8.
// The important contract is that we do NOT silently promote to utf-32*.
const bytes = iconv.encode("hello", "utf-32le")
expect(Encoding.detect(bytes)).toBe(Encoding.DEFAULT)
})
test("UTF-16 without BOM falls back to utf-8 (contract: wide UTF requires BOM)", () => {
const bytes = iconv.encode("hello", "utf-16le")
expect(Encoding.detect(bytes)).toBe(Encoding.DEFAULT)
})
test("UTF-32 LE BOM is not misdetected as UTF-16 LE (shared FF FE prefix)", () => {
// UTF-32 LE BOM is FF FE 00 00; its first two bytes are identical to the
// UTF-16 LE BOM, so the order of BOM checks matters.