Merge pull request #9923 from Kilo-Org/chore/replace-jschardet-with-chardet

chore(cli): replace jschardet with chardet

bun.lock

@@ -358,6 +358,7 @@
         "ai-gateway-provider": "3.1.2",
         "bonjour-service": "1.3.0",
         "bun-pty": "0.4.8",
+        "chardet": "2.1.1",
         "chokidar": "4.0.3",
         "cli-sound": "1.1.3",
         "clipboardy": "4.0.0",
@@ -376,7 +377,6 @@
         "iconv-lite": "0.7.2",
         "ignore": "7.0.5",
         "immer": "11.1.4",
-        "jschardet": "3.1.4",
         "jsonc-parser": "3.3.1",
         "mime-types": "3.0.2",
         "minimatch": "10.2.5",
@@ -3300,8 +3300,6 @@
 
     "jsbi": ["jsbi@4.3.2", "", {}, "sha512-9fqMSQbhJykSeii05nxKl4m6Eqn2P6rOlYiS+C5Dr/HPIU/7yZxu5qzbs40tgaFORiw2Amd0mirjxatXYMkIew=="],
 
-    "jschardet": ["jschardet@3.1.4", "", {}, "sha512-/kmVISmrwVwtyYU40iQUOp3SUPk2dhNCMsZBQX0R1/jZ8maaXJ/oZIzUOiyOqcgtLnETFKYChbJ5iDC/eWmFHg=="],
-
     "jsesc": ["jsesc@3.1.0", "", { "bin": { "jsesc": "bin/jsesc" } }, "sha512-/sM3dO2FOzXjKQhJuo0Q173wf2KOo8t4I8vHy6lF9poUp7bKT0/NHE8fPX23PwfhnykfqnC2xRxOnVw5XuGIaA=="],
 
     "json-bigint": ["json-bigint@1.0.0", "", { "dependencies": { "bignumber.js": "^9.0.0" } }, "sha512-SiPv/8VpZuWbvLSMtTDU8hEfrZWg/mH/nV/b4o0CYbSxu1UIQPLdwKOCIyLQX+VIPO5vrLX3i8qtqFyhdPSUSQ=="],

packages/kilo-docs/pages/code-with-ai/features/file-encoding.md

@@ -15,7 +15,7 @@ Kilo automatically detects the text encoding of each file it reads and preserves
 - Shift_JIS, EUC-JP, GB2312, Big5, EUC-KR
 - Windows-1251, KOI8-R
 - The ISO-8859 family
-- Other common legacy Latin and CJK encodings detected by [jschardet](https://github.com/aadsm/jschardet) and decoded by [iconv-lite](https://github.com/ashtuchkin/iconv-lite)
+- Other common legacy Latin and CJK encodings detected by [chardet](https://github.com/runk/node-chardet) and decoded by [iconv-lite](https://github.com/ashtuchkin/iconv-lite)
 
 New files Kilo creates are always UTF-8 without a BOM. Encoding detection only runs when Kilo reads or overwrites an existing file.
 

packages/opencode/package.json

@@ -142,6 +142,7 @@
     "ai-gateway-provider": "3.1.2",
     "bonjour-service": "1.3.0",
     "bun-pty": "0.4.8",
+    "chardet": "2.1.1",
     "chokidar": "4.0.3",
     "cli-sound": "1.1.3",
     "clipboardy": "4.0.0",
@@ -160,7 +161,6 @@
     "iconv-lite": "0.7.2",
     "ignore": "7.0.5",
     "immer": "11.1.4",
-    "jschardet": "3.1.4",
     "jsonc-parser": "3.3.1",
     "mime-types": "3.0.2",
     "minimatch": "10.2.5",

packages/opencode/src/jschardet.d.ts

@@ -1,14 +0,0 @@
-declare module "jschardet" {
-  export interface Result {
-    encoding?: string
-    confidence?: number
-  }
-
-  export function detect(input: ArrayLike<number>): Result
-
-  const api: {
-    detect(input: ArrayLike<number>): Result
-  }
-
-  export default api
-}

packages/opencode/src/kilocode/encoding.ts

@@ -1,7 +1,7 @@
 import { readFile, writeFile, mkdir } from "fs/promises"
 import { readFileSync } from "fs"
 import { dirname } from "path"
-import jschardet from "jschardet"
+import chardet from "chardet"
 import iconv from "iconv-lite"
 
 /**
@@ -9,9 +9,9 @@ import iconv from "iconv-lite"
  *
  * Supported:
  *  - UTF-8 (with or without BOM)
- *  - UTF-16 LE/BE with BOM (detected by jschardet)
- *  - UTF-32 LE/BE with BOM (detected by jschardet)
- *  - Legacy Latin and CJK encodings (detected by jschardet)
+ *  - UTF-16 LE/BE with BOM (detected by chardet)
+ *  - UTF-32 LE/BE with BOM (detected by chardet)
+ *  - Legacy Latin and CJK encodings (detected by chardet)
  *
  * Not supported:
  *  - UTF-16 or UTF-32 without BOM (ambiguous, rare)
@@ -19,7 +19,7 @@ import iconv from "iconv-lite"
  * Detection strategy:
  *  1. If the bytes are valid UTF-8, treat as UTF-8 (tracking the presence of a
  *     BOM so it can be written back).
- *  2. Otherwise, trust jschardet. jschardet only reports the wide UTF variants
+ *  2. Otherwise, trust chardet. chardet only reports the wide UTF variants
  *     when a BOM is present, which aligns with the contract above.
  *
  * iconv-lite's UTF codecs strip BOMs on decode and do not emit them on encode,
@@ -55,7 +55,19 @@ export namespace Encoding {
     return le || be
   }
 
-  /** Remap jschardet labels to iconv-lite compatible names. */
+  /**
+   * Canonicalize chardet labels to a stable lowercase-hyphenated form.
+   *
+   * iconv-lite already accepts every label chardet emits (e.g. "UTF-16 LE",
+   * "Shift_JIS", "KOI8-R"), so this map is not required to make decode/encode
+   * work. Its job is to give the rest of the codebase a consistent label —
+   * callers compare against `"utf-16le"`, `"windows-1251"`, etc., and should
+   * not have to account for chardet's casing or whitespace conventions.
+   *
+   * (ISO-2022-* is the one family iconv-lite does not support under any
+   * alias; those labels fall through to the `encodingExists` guard in
+   * `detect()` and are rejected to UTF-8.)
+   */
   function normalize(name: string): string {
     const lower = name.toLowerCase().replace(/[^a-z0-9]/g, "")
     const map: Record<string, string> = {
@@ -64,7 +76,6 @@ export namespace Encoding {
       utf16be: "utf-16be",
       utf32le: "utf-32le",
       utf32be: "utf-32be",
-      ascii: "utf-8",
       iso88591: "iso-8859-1",
       iso88592: "iso-8859-2",
       iso88595: "iso-8859-5",
@@ -82,13 +93,8 @@ export namespace Encoding {
       euckr: "euc-kr",
       iso2022kr: "iso-2022-kr",
       big5: "big5",
-      gb2312: "gb2312",
       gb18030: "gb18030",
       koi8r: "koi8-r",
-      maccyrillic: "x-mac-cyrillic",
-      ibm855: "cp855",
-      ibm866: "cp866",
-      tis620: "tis-620",
     }
     return map[lower] ?? name
   }
@@ -105,9 +111,9 @@ export namespace Encoding {
   export function detect(bytes: Buffer): string {
     if (bytes.length === 0) return DEFAULT
     if (isUtf8(bytes)) return hasUtf8Bom(bytes) ? UTF8_BOM : DEFAULT
-    const result = jschardet.detect(bytes)
-    if (!result.encoding) return DEFAULT
-    const enc = normalize(result.encoding)
+    const result = chardet.detect(bytes)
+    if (!result) return DEFAULT
+    const enc = normalize(result)
     if (!iconv.encodingExists(enc)) return DEFAULT
     return enc
   }

packages/opencode/test/kilocode/encoding.test.ts

@@ -3,7 +3,7 @@
 // by exercising detect/decode/encode/read/write/readSync directly, without
 // going through the Effect runtime, agent harness, or tool pipeline. They are
 // cheap, fast, and cover the internal branches (BOM handling, ASCII/UTF-8
-// normalization, jschardet fallback, unsupported encoding rejection) that the
+// normalization, chardet fallback, unsupported encoding rejection) that the
 // integration tests cannot hit deterministically.
 
 import { describe, expect, test } from "bun:test"
@@ -35,10 +35,10 @@ describe("Encoding.detect", () => {
     expect(Encoding.detect(Buffer.alloc(0))).toBe(Encoding.DEFAULT)
   })
 
-  test("plain ASCII is normalized to utf-8 (not 'ascii')", () => {
-    // jschardet reports "ascii" for pure-ASCII input; the namespace treats
-    // that as UTF-8 because UTF-8 is an ASCII superset and iconv-lite doesn't
-    // expose an "ascii" label that round-trips identically.
+  test("plain ASCII is reported as utf-8", () => {
+    // Plain ASCII is valid UTF-8, so the detector short-circuits on the
+    // isUtf8 check and never reaches chardet. UTF-8 is an ASCII superset,
+    // so this label round-trips identically through iconv-lite.
     expect(Encoding.detect(Buffer.from("plain ascii text\n"))).toBe("utf-8")
   })
 
@@ -52,8 +52,8 @@ describe("Encoding.detect", () => {
   })
 
   test("BOM-less UTF-8 containing multi-byte chars is not misdetected", () => {
-    // Regression guard: bytes that are valid UTF-8 must skip the jschardet
-    // branch. jschardet has been known to misfire on short CJK samples.
+    // Regression guard: bytes that are valid UTF-8 must skip the chardet
+    // branch. Encoding detectors have been known to misfire on short CJK samples.
     expect(Encoding.detect(Buffer.from("한글 テスト 中文", "utf-8"))).toBe("utf-8")
   })
 
@@ -231,13 +231,23 @@ describe("Encoding.hasUtf32Bom", () => {
 })
 
 describe("Encoding.read / Encoding.readSync / Encoding.write", () => {
+  // chardet is noticeably more conservative than other detectors (jschardet,
+  // ICU) on tiny samples: a 12-byte Shift_JIS phrase collides with the
+  // windows-1252 profile and is misclassified. In practice this is fine,
+  // because the tool pipeline only runs detection on files the agent is about
+  // to read or patch — real source files and documents carry far more than 12
+  // bytes of characteristic content, which is plenty for chardet to lock
+  // onto the right encoding. The short-sample cliff only matters for
+  // synthetic fixtures like this one, so we pad the sample to the same body
+  // of Japanese text the rest of the suite already relies on.
+  const shiftJisSample = "こんにちは、世界！日本語のテストです。"
+
   test("read detects and decodes Shift_JIS asynchronously", async () => {
     await tmp(async (dir) => {
       const filepath = path.join(dir, "sj.txt")
-      const text = "日本語テスト"
-      await fs.writeFile(filepath, iconv.encode(text, "Shift_JIS"))
+      await fs.writeFile(filepath, iconv.encode(shiftJisSample, "Shift_JIS"))
       const result = await Encoding.read(filepath)
-      expect(result.text).toBe(text)
+      expect(result.text).toBe(shiftJisSample)
       expect(result.encoding.toLowerCase()).toBe("shift_jis")
     })
   })
@@ -245,8 +255,7 @@ describe("Encoding.read / Encoding.readSync / Encoding.write", () => {
   test("readSync mirrors read for the same input", async () => {
     await tmp(async (dir) => {
       const filepath = path.join(dir, "sj.txt")
-      const text = "日本語テスト"
-      await fs.writeFile(filepath, iconv.encode(text, "Shift_JIS"))
+      await fs.writeFile(filepath, iconv.encode(shiftJisSample, "Shift_JIS"))
       const sync = Encoding.readSync(filepath)
       const async_ = await Encoding.read(filepath)
       expect(sync).toEqual(async_)

packages/opencode/test/kilocode/patch.test.ts

@@ -107,7 +107,7 @@ describe("Patch encoding preservation", () => {
 
   test("preserves Shift_JIS encoding through update", async () => {
     const file = path.join(dir, "jp.txt")
-    // jschardet needs enough characteristic bytes to identify Shift_JIS. A
+    // chardet needs enough characteristic bytes to identify Shift_JIS. A
     // single 19-byte phrase looks like windows-1252, so the sample is padded
     // to match the body of Japanese text the tool tests already rely on.
     const sample = "こんにちは、世界！日本語のテストです。"

packages/opencode/test/kilocode/tool-encoding.test.ts

@@ -372,7 +372,7 @@ describe("tool encoding preservation", () => {
       provideTmpdirInstance((dir) =>
         Effect.gen(function* () {
           const filepath = path.join(dir, "doc.txt")
-          // Pad with additional Shift_JIS text so jschardet has enough bytes
+          // Pad with additional Shift_JIS text so chardet has enough bytes
           // to confidently identify the encoding.
           const pad = samples.shiftJis + "\n"
           const original = pad + "日本語\n日本語\n日本語\n" + pad