Merge branch 'main' into markijbema/kilo-opencode-v1.14.30

packages/opencode/test/kilocode/encoding.test.ts

@@ -3,7 +3,7 @@
 // by exercising detect/decode/encode/read/write/readSync directly, without
 // going through the Effect runtime, agent harness, or tool pipeline. They are
 // cheap, fast, and cover the internal branches (BOM handling, ASCII/UTF-8
-// normalization, jschardet fallback, unsupported encoding rejection) that the
+// normalization, chardet fallback, unsupported encoding rejection) that the
 // integration tests cannot hit deterministically.
 
 import { describe, expect, test } from "bun:test"
@@ -35,10 +35,10 @@ describe("Encoding.detect", () => {
     expect(Encoding.detect(Buffer.alloc(0))).toBe(Encoding.DEFAULT)
   })
 
-  test("plain ASCII is normalized to utf-8 (not 'ascii')", () => {
-    // jschardet reports "ascii" for pure-ASCII input; the namespace treats
-    // that as UTF-8 because UTF-8 is an ASCII superset and iconv-lite doesn't
-    // expose an "ascii" label that round-trips identically.
+  test("plain ASCII is reported as utf-8", () => {
+    // Plain ASCII is valid UTF-8, so the detector short-circuits on the
+    // isUtf8 check and never reaches chardet. UTF-8 is an ASCII superset,
+    // so this label round-trips identically through iconv-lite.
     expect(Encoding.detect(Buffer.from("plain ascii text\n"))).toBe("utf-8")
   })
 
@@ -52,8 +52,8 @@ describe("Encoding.detect", () => {
   })
 
   test("BOM-less UTF-8 containing multi-byte chars is not misdetected", () => {
-    // Regression guard: bytes that are valid UTF-8 must skip the jschardet
-    // branch. jschardet has been known to misfire on short CJK samples.
+    // Regression guard: bytes that are valid UTF-8 must skip the chardet
+    // branch. Encoding detectors have been known to misfire on short CJK samples.
     expect(Encoding.detect(Buffer.from("한글 テスト 中文", "utf-8"))).toBe("utf-8")
   })
 
@@ -231,13 +231,23 @@ describe("Encoding.hasUtf32Bom", () => {
 })
 
 describe("Encoding.read / Encoding.readSync / Encoding.write", () => {
+  // chardet is noticeably more conservative than other detectors (jschardet,
+  // ICU) on tiny samples: a 12-byte Shift_JIS phrase collides with the
+  // windows-1252 profile and is misclassified. In practice this is fine,
+  // because the tool pipeline only runs detection on files the agent is about
+  // to read or patch — real source files and documents carry far more than 12
+  // bytes of characteristic content, which is plenty for chardet to lock
+  // onto the right encoding. The short-sample cliff only matters for
+  // synthetic fixtures like this one, so we pad the sample to the same body
+  // of Japanese text the rest of the suite already relies on.
+  const shiftJisSample = "こんにちは、世界！日本語のテストです。"
+
   test("read detects and decodes Shift_JIS asynchronously", async () => {
     await tmp(async (dir) => {
       const filepath = path.join(dir, "sj.txt")
-      const text = "日本語テスト"
-      await fs.writeFile(filepath, iconv.encode(text, "Shift_JIS"))
+      await fs.writeFile(filepath, iconv.encode(shiftJisSample, "Shift_JIS"))
       const result = await Encoding.read(filepath)
-      expect(result.text).toBe(text)
+      expect(result.text).toBe(shiftJisSample)
       expect(result.encoding.toLowerCase()).toBe("shift_jis")
     })
   })
@@ -245,8 +255,7 @@ describe("Encoding.read / Encoding.readSync / Encoding.write", () => {
   test("readSync mirrors read for the same input", async () => {
     await tmp(async (dir) => {
       const filepath = path.join(dir, "sj.txt")
-      const text = "日本語テスト"
-      await fs.writeFile(filepath, iconv.encode(text, "Shift_JIS"))
+      await fs.writeFile(filepath, iconv.encode(shiftJisSample, "Shift_JIS"))
       const sync = Encoding.readSync(filepath)
       const async_ = await Encoding.read(filepath)
       expect(sync).toEqual(async_)

packages/opencode/test/kilocode/patch.test.ts

@@ -107,7 +107,7 @@ describe("Patch encoding preservation", () => {
 
   test("preserves Shift_JIS encoding through update", async () => {
     const file = path.join(dir, "jp.txt")
-    // jschardet needs enough characteristic bytes to identify Shift_JIS. A
+    // chardet needs enough characteristic bytes to identify Shift_JIS. A
     // single 19-byte phrase looks like windows-1252, so the sample is padded
     // to match the body of Japanese text the tool tests already rely on.
     const sample = "こんにちは、世界！日本語のテストです。"

packages/opencode/test/kilocode/tool-encoding.test.ts

@@ -372,7 +372,7 @@ describe("tool encoding preservation", () => {
       provideTmpdirInstance((dir) =>
         Effect.gen(function* () {
           const filepath = path.join(dir, "doc.txt")
-          // Pad with additional Shift_JIS text so jschardet has enough bytes
+          // Pad with additional Shift_JIS text so chardet has enough bytes
           // to confidently identify the encoding.
           const pad = samples.shiftJis + "\n"
           const original = pad + "日本語\n日本語\n日本語\n" + pad