Merge pull request #8587 from Kilo-Org/feat/encoding-preservation

feat(cli): preserve original text encoding when editing files

.changeset/preserve-file-encoding.md

@@ -0,0 +1,9 @@
+---
+"@kilocode/cli": minor
+---
+
+The agent now detects and preserves the original text encoding of files when reading and editing them, so non-UTF-8 files are displayed correctly to the model and written back in their original encoding. New files are still created as UTF-8 without BOM — detection only applies when overwriting or editing an existing file.
+
+Supported: UTF-8 (with or without BOM), UTF-16 with BOM, and common legacy Latin and CJK encodings (Shift_JIS, EUC-JP, GB2312, Big5, EUC-KR, Windows-1251, KOI8-R, ISO-8859, and others).
+
+Not supported: UTF-16 without BOM, UTF-32.

bun.lock

@@ -449,8 +449,10 @@
         "gray-matter": "4.0.3",
         "hono": "catalog:",
         "hono-openapi": "catalog:",
+        "iconv-lite": "0.7.2",
         "ignore": "7.0.5",
         "immer": "11.1.4",
+        "jschardet": "3.1.4",
         "jsonc-parser": "3.3.1",
         "mime-types": "3.0.2",
         "minimatch": "10.2.5",
@@ -3588,6 +3590,8 @@
 
     "jsbi": ["jsbi@4.3.2", "", {}, "sha512-9fqMSQbhJykSeii05nxKl4m6Eqn2P6rOlYiS+C5Dr/HPIU/7yZxu5qzbs40tgaFORiw2Amd0mirjxatXYMkIew=="],
 
+    "jschardet": ["jschardet@3.1.4", "", {}, "sha512-/kmVISmrwVwtyYU40iQUOp3SUPk2dhNCMsZBQX0R1/jZ8maaXJ/oZIzUOiyOqcgtLnETFKYChbJ5iDC/eWmFHg=="],
+
     "jsesc": ["jsesc@3.1.0", "", { "bin": { "jsesc": "bin/jsesc" } }, "sha512-/sM3dO2FOzXjKQhJuo0Q173wf2KOo8t4I8vHy6lF9poUp7bKT0/NHE8fPX23PwfhnykfqnC2xRxOnVw5XuGIaA=="],
 
     "json-bigint": ["json-bigint@1.0.0", "", { "dependencies": { "bignumber.js": "^9.0.0" } }, "sha512-SiPv/8VpZuWbvLSMtTDU8hEfrZWg/mH/nV/b4o0CYbSxu1UIQPLdwKOCIyLQX+VIPO5vrLX3i8qtqFyhdPSUSQ=="],

packages/opencode/package.json

@@ -154,8 +154,10 @@
     "gray-matter": "4.0.3",
     "hono": "catalog:",
     "hono-openapi": "catalog:",
+    "iconv-lite": "0.7.2",
     "ignore": "7.0.5",
     "immer": "11.1.4",
+    "jschardet": "3.1.4",
     "jsonc-parser": "3.3.1",
     "mime-types": "3.0.2",
     "minimatch": "10.2.5",

packages/opencode/src/kilocode/encoding.ts

@@ -0,0 +1,142 @@
+import { readFile, writeFile, mkdir } from "fs/promises"
+import { readFileSync } from "fs"
+import { dirname } from "path"
+import jschardet from "jschardet"
+import iconv from "iconv-lite"
+
+/**
+ * Text encoding detection and preservation for tool file I/O.
+ *
+ * Supported:
+ *  - UTF-8 (with or without BOM)
+ *  - UTF-16 LE/BE with BOM (detected by jschardet)
+ *  - Legacy Latin and CJK encodings (detected by jschardet)
+ *
+ * Not supported:
+ *  - UTF-16 without BOM (ambiguous, rare)
+ *  - UTF-32 (extremely rare)
+ *
+ * Detection strategy:
+ *  1. If the bytes are valid UTF-8, treat as UTF-8 (tracking the presence of a
+ *     BOM so it can be written back).
+ *  2. Otherwise, trust jschardet.
+ *
+ * iconv-lite's UTF codecs strip BOMs on decode and do not emit them on encode,
+ * so UTF BOMs are handled explicitly in {@link encode} to round-trip cleanly.
+ */
+export namespace Encoding {
+  export const DEFAULT = "utf-8"
+  /**
+   * Synthetic label for UTF-8 files that start with a BOM. iconv-lite's utf-8
+   * codec always strips BOMs on decode and never emits one on encode, so we
+   * track the "with BOM" case explicitly to round-trip it faithfully.
+   */
+  export const UTF8_BOM = "utf-8-bom"
+  const UTF8_BOM_BYTES = Buffer.from([0xef, 0xbb, 0xbf])
+
+  function hasUtf8Bom(bytes: Buffer): boolean {
+    return bytes.length >= 3 && bytes[0] === 0xef && bytes[1] === 0xbb && bytes[2] === 0xbf
+  }
+
+  /** True if `bytes[0..limit]` starts with a UTF-16 LE or BE byte-order mark. */
+  export function hasUtf16Bom(bytes: Buffer, limit = bytes.length): boolean {
+    if (limit < 2) return false
+    return (bytes[0] === 0xff && bytes[1] === 0xfe) || (bytes[0] === 0xfe && bytes[1] === 0xff)
+  }
+
+  /** Remap jschardet labels to iconv-lite compatible names. */
+  function normalize(name: string): string {
+    const lower = name.toLowerCase().replace(/[^a-z0-9]/g, "")
+    const map: Record<string, string> = {
+      utf8: "utf-8",
+      utf16le: "utf-16le",
+      utf16be: "utf-16be",
+      ascii: "utf-8",
+      iso88591: "iso-8859-1",
+      iso88592: "iso-8859-2",
+      iso88595: "iso-8859-5",
+      iso88597: "iso-8859-7",
+      iso88598: "iso-8859-8",
+      iso88599: "iso-8859-9",
+      windows1250: "windows-1250",
+      windows1251: "windows-1251",
+      windows1252: "windows-1252",
+      windows1253: "windows-1253",
+      windows1255: "windows-1255",
+      shiftjis: "Shift_JIS",
+      eucjp: "euc-jp",
+      iso2022jp: "iso-2022-jp",
+      euckr: "euc-kr",
+      iso2022kr: "iso-2022-kr",
+      big5: "big5",
+      gb2312: "gb2312",
+      gb18030: "gb18030",
+      koi8r: "koi8-r",
+      maccyrillic: "x-mac-cyrillic",
+      ibm855: "cp855",
+      ibm866: "cp866",
+      tis620: "tis-620",
+    }
+    return map[lower] ?? name
+  }
+
+  function isUtf8(bytes: Buffer): boolean {
+    try {
+      new TextDecoder("utf-8", { fatal: true }).decode(bytes)
+      return true
+    } catch {
+      return false
+    }
+  }
+
+  export function detect(bytes: Buffer): string {
+    if (bytes.length === 0) return DEFAULT
+    if (isUtf8(bytes)) return hasUtf8Bom(bytes) ? UTF8_BOM : DEFAULT
+    const result = jschardet.detect(bytes)
+    if (!result.encoding) return DEFAULT
+    const enc = normalize(result.encoding)
+    // Reject unsupported Unicode encodings (UTF-32 and anything iconv-lite cannot decode)
+    if (enc.toLowerCase().startsWith("utf-32")) return DEFAULT
+    if (!iconv.encodingExists(enc)) return DEFAULT
+    return enc
+  }
+
+  export function decode(bytes: Buffer, encoding: string): string {
+    if (encoding === UTF8_BOM) return iconv.decode(bytes, "utf-8")
+    return iconv.decode(bytes, encoding)
+  }
+
+  export function encode(text: string, encoding: string): Buffer {
+    // iconv-lite's UTF codecs strip/ignore BOMs, but we support "UTF-X with BOM"
+    // as a distinct variant. Prepend the BOM manually so round-tripping keeps
+    // the original byte signature intact. Strip a leading U+FEFF from `text`
+    // first so we never emit a double BOM when the decoded text already
+    // contains one (e.g. if a tool round-trips content verbatim).
+    const body = text.charCodeAt(0) === 0xfeff ? text.slice(1) : text
+    if (encoding === UTF8_BOM) return Buffer.concat([UTF8_BOM_BYTES, iconv.encode(body, "utf-8")])
+    const lower = encoding.toLowerCase()
+    if (lower === "utf-16le") return Buffer.concat([Buffer.from([0xff, 0xfe]), iconv.encode(body, encoding)])
+    if (lower === "utf-16be") return Buffer.concat([Buffer.from([0xfe, 0xff]), iconv.encode(body, encoding)])
+    return iconv.encode(text, encoding)
+  }
+
+  /** Read a file, detecting its encoding. */
+  export async function read(path: string): Promise<{ text: string; encoding: string }> {
+    const bytes = await readFile(path)
+    const encoding = detect(bytes)
+    return { text: decode(bytes, encoding), encoding }
+  }
+
+  /** Synchronous read, detecting encoding. */
+  export function readSync(path: string): { text: string; encoding: string } {
+    const bytes = readFileSync(path)
+    const encoding = detect(bytes)
+    return { text: decode(bytes, encoding), encoding }
+  }
+
+  /** Write text, ensuring parent directory exists, using the given encoding. */
+  export async function write(path: string, text: string, encoding: string = DEFAULT): Promise<void> {
+    await mkdir(dirname(path), { recursive: true })
+    await writeFile(path, encode(text, encoding))
+  }
+}

packages/opencode/src/kilocode/tool/encoded-io.ts

@@ -0,0 +1,17 @@
+import { Effect } from "effect"
+import { Encoding } from "../encoding"
+
+/**
+ * Effect wrappers around {@link Encoding.read} and {@link Encoding.write} so
+ * tool code can preserve file encoding without leaking Node/async boilerplate
+ * into each call site. Uses {@link Effect.tryPromise} so I/O failures surface
+ * as typed errors that can be recovered with `.pipe(Effect.catch(...))`.
+ */
+export namespace EncodedIO {
+  const wrap = (cause: unknown) => (cause instanceof Error ? cause : new Error(String(cause)))
+
+  export const read = (path: string) => Effect.tryPromise({ try: () => Encoding.read(path), catch: wrap })
+
+  export const write = (path: string, text: string, encoding: string = Encoding.DEFAULT) =>
+    Effect.tryPromise({ try: () => Encoding.write(path, text, encoding), catch: wrap })
+}

packages/opencode/src/patch/index.ts

@@ -1,8 +1,8 @@
 import z from "zod"
 import * as path from "path"
 import * as fs from "fs/promises"
-import { readFileSync } from "fs"
 import { Log } from "../util"
+import { Encoding } from "../kilocode/encoding" // kilocode_change
 
 const log = Log.create({ service: "patch" })
 
@@ -305,13 +305,17 @@ export function maybeParseApplyPatch(
 interface ApplyPatchFileUpdate {
   unified_diff: string
   content: string
+  encoding: string // kilocode_change
 }
 
 export function deriveNewContentsFromChunks(filePath: string, chunks: UpdateFileChunk[]): ApplyPatchFileUpdate {
   // Read original file content
   let originalContent: string
+  let encoding: string // kilocode_change
   try {
-    originalContent = readFileSync(filePath, "utf-8")
+    const result = Encoding.readSync(filePath) // kilocode_change - encoding-aware read
+    originalContent = result.text // kilocode_change
+    encoding = result.encoding // kilocode_change
   } catch (error) {
     throw new Error(`Failed to read file ${filePath}: ${error}`, { cause: error })
   }
@@ -339,6 +343,7 @@ export function deriveNewContentsFromChunks(filePath: string, chunks: UpdateFile
   return {
     unified_diff: unifiedDiff,
     content: newContent,
+    encoding, // kilocode_change - include detected encoding for round-trip write
   }
 }
 
@@ -526,13 +531,7 @@ export async function applyHunksToFiles(hunks: Hunk[]): Promise<AffectedPaths> {
   for (const hunk of hunks) {
     switch (hunk.type) {
       case "add":
-        // Create parent directories
-        const addDir = path.dirname(hunk.path)
-        if (addDir !== "." && addDir !== "/") {
-          await fs.mkdir(addDir, { recursive: true })
-        }
-
-        await fs.writeFile(hunk.path, hunk.contents, "utf-8")
+        await Encoding.write(hunk.path, hunk.contents) // kilocode_change - encoding-aware write (mkdirs)
         added.push(hunk.path)
         log.info(`Added file: ${hunk.path}`)
         break
@@ -548,18 +547,13 @@ export async function applyHunksToFiles(hunks: Hunk[]): Promise<AffectedPaths> {
 
         if (hunk.move_path) {
           // Handle file move
-          const moveDir = path.dirname(hunk.move_path)
-          if (moveDir !== "." && moveDir !== "/") {
-            await fs.mkdir(moveDir, { recursive: true })
-          }
-
-          await fs.writeFile(hunk.move_path, fileUpdate.content, "utf-8")
+          await Encoding.write(hunk.move_path, fileUpdate.content, fileUpdate.encoding) // kilocode_change
           await fs.unlink(hunk.path)
           modified.push(hunk.move_path)
           log.info(`Moved file: ${hunk.path} -> ${hunk.move_path}`)
         } else {
           // Regular update
-          await fs.writeFile(hunk.path, fileUpdate.content, "utf-8")
+          await Encoding.write(hunk.path, fileUpdate.content, fileUpdate.encoding) // kilocode_change
           modified.push(hunk.path)
           log.info(`Updated file: ${hunk.path}`)
         }
@@ -624,7 +618,7 @@ export async function maybeParseApplyPatchVerified(
             // For delete, we need to read the current content
             const deletePath = path.resolve(effectiveCwd, hunk.path)
             try {
-              const content = await fs.readFile(deletePath, "utf-8")
+              const content = (await Encoding.read(deletePath)).text // kilocode_change - encoding-aware read
               changes.set(resolvedPath, {
                 type: "delete",
                 content,

packages/opencode/src/tool/apply_patch.ts

@@ -15,6 +15,7 @@ import DESCRIPTION from "./apply_patch.txt"
 import { File } from "../file"
 import { filterDiagnostics } from "./diagnostics" // kilocode_change
 import { ConfigValidation } from "../kilocode/config-validation" // kilocode_change
+import { EncodedIO } from "../kilocode/tool/encoded-io" // kilocode_change
 import { Format } from "../format"
 
 const PatchParams = z.object({
@@ -61,6 +62,7 @@ export const ApplyPatchTool = Tool.define(
         diff: string
         additions: number
         deletions: number
+        encoding: string // kilocode_change - preserved per-file encoding
       }> = []
 
       let totalDiff = ""
@@ -91,6 +93,7 @@ export const ApplyPatchTool = Tool.define(
               diff,
               additions,
               deletions,
+              encoding: "utf-8", // kilocode_change - new files default to utf-8
             })
 
             totalDiff += diff + "\n"
@@ -106,13 +109,15 @@ export const ApplyPatchTool = Tool.define(
               )
             }
 
-            const oldContent = yield* afs.readFileString(filePath)
+            const oldContent = (yield* EncodedIO.read(filePath)).text // kilocode_change - encoding-aware read
             let newContent = oldContent
+            let encoding: string // kilocode_change - filled in by the patch helper below
 
             // Apply the update chunks to get new content
             try {
               const fileUpdate = Patch.deriveNewContentsFromChunks(filePath, hunk.chunks)
               newContent = fileUpdate.content
+              encoding = fileUpdate.encoding // kilocode_change
             } catch (error) {
               return yield* Effect.fail(new Error(`apply_patch verification failed: ${error}`))
             }
@@ -138,6 +143,7 @@ export const ApplyPatchTool = Tool.define(
               diff,
               additions,
               deletions,
+              encoding, // kilocode_change
             })
 
             totalDiff += diff + "\n"
@@ -145,17 +151,18 @@ export const ApplyPatchTool = Tool.define(
           }
 
           case "delete": {
-            const contentToDelete = yield* afs
-              .readFileString(filePath)
-              .pipe(
-                Effect.catch((error) =>
-                  Effect.fail(
-                    new Error(
-                      `apply_patch verification failed: ${error instanceof Error ? error.message : String(error)}`,
-                    ),
+            // kilocode_change start - encoding-aware read
+            const deleteRead = yield* EncodedIO.read(filePath).pipe(
+              Effect.catch((error) =>
+                Effect.fail(
+                  new Error(
+                    `apply_patch verification failed: ${error instanceof Error ? error.message : String(error)}`,
                   ),
                 ),
-              )
+              ),
+            )
+            const contentToDelete = deleteRead.text
+            // kilocode_change end
             const deleteDiff = trimDiff(createTwoFilesPatch(filePath, filePath, contentToDelete, ""))
 
             const deletions = contentToDelete.split("\n").length
@@ -168,6 +175,7 @@ export const ApplyPatchTool = Tool.define(
               diff: deleteDiff,
               additions: 0,
               deletions,
+              encoding: deleteRead.encoding, // kilocode_change
             })
 
             totalDiff += deleteDiff + "\n"
@@ -203,26 +211,25 @@ export const ApplyPatchTool = Tool.define(
       // Apply the changes
       const updates: Array<{ file: string; event: "add" | "change" | "unlink" }> = []
 
+      // kilocode_change start - encoding-aware writes (EncodedIO.write mkdirs recursively)
       for (const change of fileChanges) {
         const edited = change.type === "delete" ? undefined : (change.movePath ?? change.filePath)
         switch (change.type) {
           case "add":
             // Create parent directories (recursive: true is safe on existing/root dirs)
-
-            yield* afs.writeWithDirs(change.filePath, change.newContent)
+            yield* EncodedIO.write(change.filePath, change.newContent, change.encoding)
             updates.push({ file: change.filePath, event: "add" })
             break
 
           case "update":
-            yield* afs.writeWithDirs(change.filePath, change.newContent)
+            yield* EncodedIO.write(change.filePath, change.newContent, change.encoding)
             updates.push({ file: change.filePath, event: "change" })
             break
 
           case "move":
             if (change.movePath) {
               // Create parent directories (recursive: true is safe on existing/root dirs)
-
-              yield* afs.writeWithDirs(change.movePath!, change.newContent)
+              yield* EncodedIO.write(change.movePath!, change.newContent, change.encoding)
               yield* afs.remove(change.filePath)
               updates.push({ file: change.filePath, event: "unlink" })
               updates.push({ file: change.movePath, event: "add" })
@@ -234,6 +241,7 @@ export const ApplyPatchTool = Tool.define(
             updates.push({ file: change.filePath, event: "unlink" })
             break
         }
+        // kilocode_change end
 
         if (edited) {
           yield* format.file(edited)

packages/opencode/src/tool/edit.ts

@@ -20,6 +20,7 @@ import { assertExternalDirectoryEffect } from "./external-directory"
 import { AppFileSystem } from "@opencode-ai/shared/filesystem"
 import { filterDiagnostics } from "./diagnostics" // kilocode_change
 import { ConfigValidation } from "../kilocode/config-validation" // kilocode_change
+import { EncodedIO } from "../kilocode/tool/encoded-io" // kilocode_change
 
 const MAX_DIFF_CONTENT = 500_000 // kilocode_change
 
@@ -96,7 +97,11 @@ export const EditTool = Tool.define(
           yield* Effect.gen(function* () {
             if (params.oldString === "") {
               const existed = yield* afs.existsSafe(filePath)
-              if (existed) contentOld = yield* afs.readFileString(filePath) // kilocode_change
+              // kilocode_change start - preserve file encoding on write
+              const pre = existed ? yield* EncodedIO.read(filePath) : { text: "", encoding: "utf-8" }
+              contentOld = pre.text
+              const encoding = pre.encoding
+              // kilocode_change end
               contentNew = params.newString
               diff = trimDiff(createTwoFilesPatch(filePath, filePath, contentOld, contentNew))
               cachedFilediff = buildFileDiff(filePath, contentOld, contentNew) // kilocode_change
@@ -110,7 +115,7 @@ export const EditTool = Tool.define(
                   filediff: cachedFilediff, // kilocode_change
                 },
               })
-              yield* afs.writeWithDirs(filePath, params.newString)
+              yield* EncodedIO.write(filePath, params.newString, encoding) // kilocode_change
               yield* format.file(filePath)
               yield* bus.publish(File.Event.Edited, { file: filePath })
               yield* bus.publish(FileWatcher.Event.Updated, {
@@ -123,7 +128,11 @@ export const EditTool = Tool.define(
             const info = yield* afs.stat(filePath).pipe(Effect.catch(() => Effect.succeed(undefined)))
             if (!info) throw new Error(`File ${filePath} not found`)
             if (info.type === "Directory") throw new Error(`Path is a directory, not a file: ${filePath}`)
-            contentOld = yield* afs.readFileString(filePath)
+            // kilocode_change start - preserve file encoding
+            const pre = yield* EncodedIO.read(filePath)
+            contentOld = pre.text
+            const encoding = pre.encoding
+            // kilocode_change end
 
             const ending = detectLineEnding(contentOld)
             const old = convertToLineEnding(normalizeLineEndings(params.oldString), ending)
@@ -151,14 +160,14 @@ export const EditTool = Tool.define(
               },
             })
 
-            yield* afs.writeWithDirs(filePath, contentNew)
+            yield* EncodedIO.write(filePath, contentNew, encoding) // kilocode_change
             yield* format.file(filePath)
             yield* bus.publish(File.Event.Edited, { file: filePath })
             yield* bus.publish(FileWatcher.Event.Updated, {
               file: filePath,
               event: "change",
             })
-            contentNew = yield* afs.readFileString(filePath)
+            contentNew = (yield* EncodedIO.read(filePath)).text // kilocode_change
             diff = trimDiff(
               createTwoFilesPatch(
                 filePath,

packages/opencode/src/tool/read.ts

@@ -1,8 +1,8 @@
 import z from "zod"
 import { Effect, Scope } from "effect"
-import { createReadStream } from "fs"
 import { open } from "fs/promises"
 import * as path from "path"
+import { Readable } from "stream" // kilocode_change
 import { createInterface } from "readline"
 import * as Tool from "./tool"
 import { AppFileSystem } from "@opencode-ai/shared/filesystem"
@@ -12,6 +12,7 @@ import { Instance } from "../project/instance"
 import { assertExternalDirectoryEffect } from "./external-directory"
 import { Instruction } from "../session/instruction"
 // kilocode_change start
+import { Encoding } from "../kilocode/encoding"
 import { readDirectoryFiles } from "../kilocode/tool/read-directory"
 // kilocode_change end
 
@@ -234,7 +235,8 @@ export const ReadTool = Tool.define(
 // kilocode_change start
 export async function lines(filepath: string, opts: { limit: number; offset: number }) {
   // kilocode_change end
-  const stream = createReadStream(filepath, { encoding: "utf8" })
+  const encoded = await Encoding.read(filepath) // kilocode_change - decode with detected encoding
+  const stream = Readable.from([encoded.text]) // kilocode_change - replaces createReadStream
   const rl = createInterface({
     input: stream,
     // Note: we use the crlfDelay option to recognize all instances of CR LF
@@ -325,6 +327,10 @@ export async function isBinaryFile(filepath: string, fileSize: number): Promise<
     const result = await fh.read(bytes, 0, sampleSize, 0)
     if (result.bytesRead === 0) return false
 
+    // kilocode_change start - UTF-16 BOM: NUL bytes are legitimate, skip the NUL/control-char heuristic
+    if (Encoding.hasUtf16Bom(bytes, result.bytesRead)) return false
+    // kilocode_change end
+
     let nonPrintableCount = 0
     for (let i = 0; i < result.bytesRead; i++) {
       if (bytes[i] === 0) return true

packages/opencode/src/tool/write.ts

@@ -15,6 +15,7 @@ import { trimDiff, buildFileDiff } from "./edit" // kilocode_change
 import { assertExternalDirectoryEffect } from "./external-directory"
 import { filterDiagnostics } from "./diagnostics" // kilocode_change
 import { ConfigValidation } from "../kilocode/config-validation" // kilocode_change
+import { EncodedIO } from "../kilocode/tool/encoded-io" // kilocode_change
 
 const MAX_PROJECT_DIAGNOSTICS_FILES = 5
 
@@ -40,7 +41,11 @@ export const WriteTool = Tool.define(
           yield* assertExternalDirectoryEffect(ctx, filepath)
 
           const exists = yield* fs.existsSafe(filepath)
-          const contentOld = exists ? yield* fs.readFileString(filepath) : ""
+          // kilocode_change start - preserve file encoding on write
+          const pre = exists ? yield* EncodedIO.read(filepath) : { text: "", encoding: "utf-8" }
+          const contentOld = pre.text
+          const encoding = pre.encoding
+          // kilocode_change end
 
           const diff = trimDiff(createTwoFilesPatch(filepath, filepath, contentOld, params.content))
           const filediff = buildFileDiff(filepath, contentOld, params.content) // kilocode_change
@@ -55,7 +60,7 @@ export const WriteTool = Tool.define(
             },
           })
 
-          yield* fs.writeWithDirs(filepath, params.content)
+          yield* EncodedIO.write(filepath, params.content, encoding) // kilocode_change
           yield* format.file(filepath)
           yield* bus.publish(File.Event.Edited, { file: filepath })
           yield* bus.publish(FileWatcher.Event.Updated, {

packages/opencode/test/kilocode/tool-encoding.test.ts

@@ -0,0 +1,339 @@
+// Integration tests verifying that the agent file tools (read, write, edit,
+// apply_patch) detect and preserve the original encoding of files on disk.
+// Tests exercise the real tool pipeline rather than the Encoding helper
+// directly so we validate end-to-end behaviour.
+
+import { afterEach, describe, expect } from "bun:test"
+import { Effect, Layer } from "effect"
+import path from "path"
+import fs from "fs/promises"
+import iconv from "iconv-lite"
+import { Agent } from "../../src/agent/agent"
+import { AppFileSystem } from "@opencode-ai/shared/filesystem"
+import { ApplyPatchTool } from "../../src/tool/apply_patch"
+import { Bus } from "../../src/bus"
+import * as CrossSpawnSpawner from "../../src/effect/cross-spawn-spawner"
+import { EditTool } from "../../src/tool/edit"
+import { Format } from "../../src/format"
+import { Instance } from "../../src/project/instance"
+import { Instruction } from "../../src/session/instruction"
+import { LSP } from "../../src/lsp"
+import { MessageID, SessionID } from "../../src/session/schema"
+import { ReadTool } from "../../src/tool/read"
+import * as Tool from "../../src/tool/tool"
+import { Truncate } from "../../src/tool"
+import { WriteTool } from "../../src/tool/write"
+import { provideTmpdirInstance } from "../fixture/fixture"
+import { testEffect } from "../lib/effect"
+
+const ctx = {
+  sessionID: SessionID.make("ses_test-encoding"),
+  messageID: MessageID.make(""),
+  callID: "",
+  agent: "build",
+  abort: AbortSignal.any([]),
+  messages: [],
+  metadata: () => Effect.void,
+  ask: () => Effect.void,
+}
+
+afterEach(async () => {
+  await Instance.disposeAll()
+})
+
+const it = testEffect(
+  Layer.mergeAll(
+    Agent.defaultLayer,
+    AppFileSystem.defaultLayer,
+    CrossSpawnSpawner.defaultLayer,
+    Instruction.defaultLayer,
+    LSP.defaultLayer,
+    Bus.layer,
+    Format.defaultLayer,
+    Truncate.defaultLayer,
+  ),
+)
+
+const runRead = (args: Tool.InferParameters<typeof ReadTool>) =>
+  Effect.gen(function* () {
+    const info = yield* ReadTool
+    const tool = yield* info.init()
+    return yield* tool.execute(args, ctx)
+  })
+
+const runWrite = (args: Tool.InferParameters<typeof WriteTool>) =>
+  Effect.gen(function* () {
+    const info = yield* WriteTool
+    const tool = yield* info.init()
+    return yield* tool.execute(args, ctx)
+  })
+
+const runEdit = (args: Tool.InferParameters<typeof EditTool>) =>
+  Effect.gen(function* () {
+    const info = yield* EditTool
+    const tool = yield* info.init()
+    return yield* tool.execute(args, ctx)
+  })
+
+const runPatch = (args: Tool.InferParameters<typeof ApplyPatchTool>) =>
+  Effect.gen(function* () {
+    const info = yield* ApplyPatchTool
+    const tool = yield* info.init()
+    return yield* tool.execute(args, ctx)
+  })
+
+// FileTime was removed upstream; edit/write no longer require a prior read.
+const markRead = (_filepath: string) => Effect.void
+
+// iconv-lite's UTF codecs don't emit BOMs, but this codebase supports
+// "UTF-X with BOM" as a distinct variant. Prepend one here for fixture files
+// that are meant to have one.
+const UTF8_BOM = "utf-8-bom"
+const encodeBytes = (text: string, encoding: string): Buffer => {
+  if (encoding === UTF8_BOM) return Buffer.concat([Buffer.from([0xef, 0xbb, 0xbf]), iconv.encode(text, "utf-8")])
+  const lower = encoding.toLowerCase()
+  if (lower === "utf-16le") return Buffer.concat([Buffer.from([0xff, 0xfe]), iconv.encode(text, encoding)])
+  if (lower === "utf-16be") return Buffer.concat([Buffer.from([0xfe, 0xff]), iconv.encode(text, encoding)])
+  return iconv.encode(text, encoding)
+}
+
+// Create a file with the given encoding by writing raw bytes.
+const putEncoded = (filepath: string, text: string, encoding: string) =>
+  Effect.promise(async () => {
+    await fs.mkdir(path.dirname(filepath), { recursive: true })
+    await fs.writeFile(filepath, encodeBytes(text, encoding))
+  })
+
+const loadDecoded = (filepath: string, encoding: string) =>
+  Effect.promise(async () => {
+    const bytes = await fs.readFile(filepath)
+    if (encoding === UTF8_BOM) {
+      const stripped = bytes.length >= 3 && bytes[0] === 0xef && bytes[1] === 0xbb && bytes[2] === 0xbf
+      return iconv.decode(stripped ? bytes.subarray(3) : bytes, "utf-8")
+    }
+    return iconv.decode(bytes, encoding)
+  })
+
+const loadBytes = (filepath: string) => Effect.promise(() => fs.readFile(filepath))
+
+// Sample phrases chosen to exercise each encoding's characteristic byte patterns.
+const samples = {
+  utf8: "Hello, world! — £100",
+  shiftJis: "こんにちは、世界！日本語のテストです。",
+  eucJp: "日本語のEUC-JPテスト文字列です。",
+  gb2312: "你好，世界！这是简体中文测试。",
+  big5: "你好，世界！這是繁體中文測試。",
+  eucKr: "안녕하세요, 세계! 한국어 테스트입니다.",
+  windows1251: "Привет, мир! Это тест кириллицы.",
+  koi8r: "Привет, мир! КОИ-8 Р тест.",
+}
+
+describe("tool encoding preservation", () => {
+  describe("ReadTool decodes files with non-UTF-8 encodings", () => {
+    const cases: Array<[string, string, string]> = [
+      ["UTF-8", "utf-8", samples.utf8],
+      ["UTF-8 with BOM", UTF8_BOM, samples.utf8],
+      ["UTF-16 LE with BOM", "utf-16le", samples.utf8],
+      ["UTF-16 BE with BOM", "utf-16be", samples.utf8],
+      ["Shift_JIS", "Shift_JIS", samples.shiftJis],
+      ["EUC-JP", "euc-jp", samples.eucJp],
+      ["GB2312", "gb2312", samples.gb2312],
+      ["Big5", "big5", samples.big5],
+      ["EUC-KR", "euc-kr", samples.eucKr],
+      ["Windows-1251", "windows-1251", samples.windows1251],
+      ["KOI8-R", "koi8-r", samples.koi8r],
+    ]
+
+    for (const [label, encoding, text] of cases) {
+      it.live(`decodes ${label} content for the model`, () =>
+        provideEncoded(encoding, text, (filepath) =>
+          Effect.gen(function* () {
+            const result = yield* runRead({ filePath: filepath })
+            expect(result.output).toContain(text)
+          }),
+        ),
+      )
+    }
+  })
+
+  describe("ReadTool does not flag non-Latin text files as binary", () => {
+    it.live("accepts Shift_JIS", () =>
+      provideEncoded("Shift_JIS", samples.shiftJis, (filepath) =>
+        Effect.gen(function* () {
+          const result = yield* runRead({ filePath: filepath })
+          expect(result.output).toContain(samples.shiftJis)
+        }),
+      ),
+    )
+
+    it.live("accepts UTF-16 LE with BOM (contains NUL bytes)", () =>
+      provideEncoded("utf-16le", samples.utf8, (filepath) =>
+        Effect.gen(function* () {
+          const result = yield* runRead({ filePath: filepath })
+          expect(result.output).toContain(samples.utf8)
+        }),
+      ),
+    )
+  })
+
+  describe("WriteTool preserves existing file encoding when overwriting", () => {
+    const cases: Array<[string, string, string]> = [
+      ["UTF-8 with BOM", UTF8_BOM, samples.utf8],
+      ["Shift_JIS", "Shift_JIS", samples.shiftJis],
+      ["GB2312", "gb2312", samples.gb2312],
+      ["Windows-1251", "windows-1251", samples.windows1251],
+      ["UTF-16 LE", "utf-16le", samples.utf8],
+    ]
+
+    for (const [label, encoding, original] of cases) {
+      it.live(`preserves ${label} encoding on overwrite`, () =>
+        provideTmpdirInstance((dir) =>
+          Effect.gen(function* () {
+            const filepath = path.join(dir, "file.txt")
+            yield* putEncoded(filepath, original, encoding)
+            yield* markRead(filepath)
+
+            const replacement = original + " updated"
+            yield* runWrite({ filePath: filepath, content: replacement })
+
+            const decoded = yield* loadDecoded(filepath, encoding)
+            expect(decoded).toBe(replacement)
+
+            // Bytes should still match the original encoding (and differ from UTF-8).
+            const bytes = yield* loadBytes(filepath)
+            expect(bytes.equals(encodeBytes(replacement, encoding))).toBe(true)
+          }),
+        ),
+      )
+    }
+
+    it.live("defaults new files to UTF-8", () =>
+      provideTmpdirInstance((dir) =>
+        Effect.gen(function* () {
+          const filepath = path.join(dir, "new.txt")
+          yield* runWrite({ filePath: filepath, content: samples.utf8 })
+
+          const bytes = yield* loadBytes(filepath)
+          expect(bytes.equals(Buffer.from(samples.utf8, "utf-8"))).toBe(true)
+        }),
+      ),
+    )
+
+    // Guard against double-BOM regressions: if the model ever hands back content
+    // that already starts with U+FEFF (e.g. by round-tripping literal bytes),
+    // writing it to a BOM-encoded file must still produce exactly one BOM.
+    const bomCases: Array<[string, string, Buffer]> = [
+      ["UTF-8 with BOM", UTF8_BOM, Buffer.from([0xef, 0xbb, 0xbf])],
+      ["UTF-16 LE", "utf-16le", Buffer.from([0xff, 0xfe])],
+      ["UTF-16 BE", "utf-16be", Buffer.from([0xfe, 0xff])],
+    ]
+    for (const [label, encoding, bom] of bomCases) {
+      it.live(`does not emit a double BOM for ${label} when content starts with U+FEFF`, () =>
+        provideTmpdirInstance((dir) =>
+          Effect.gen(function* () {
+            const filepath = path.join(dir, "file.txt")
+            yield* putEncoded(filepath, "hello", encoding)
+            yield* markRead(filepath)
+
+            yield* runWrite({ filePath: filepath, content: "\uFEFFgoodbye" })
+
+            const bytes = yield* loadBytes(filepath)
+            // Exactly one BOM prefix, immediately followed by encoded payload.
+            expect(bytes.subarray(0, bom.length).equals(bom)).toBe(true)
+            expect(bytes.subarray(bom.length, bom.length * 2).equals(bom)).toBe(false)
+            const decoded = yield* loadDecoded(filepath, encoding)
+            expect(decoded).toBe("goodbye")
+          }),
+        ),
+      )
+    }
+  })
+
+  describe("EditTool preserves existing file encoding across edits", () => {
+    const cases: Array<[string, string, string, string, string]> = [
+      ["UTF-8 with BOM", UTF8_BOM, samples.utf8 + "\n second line", "world", "earth"],
+      ["Shift_JIS", "Shift_JIS", samples.shiftJis, "日本語", "ニホンゴ"],
+      ["GB2312", "gb2312", samples.gb2312, "简体中文", "中文简体"],
+      ["Windows-1251", "windows-1251", samples.windows1251, "мир", "планета"],
+      ["UTF-16 LE", "utf-16le", samples.utf8 + "\n second line", "world", "earth"],
+    ]
+
+    for (const [label, encoding, original, oldString, newString] of cases) {
+      it.live(`preserves ${label} through edit`, () =>
+        provideTmpdirInstance((dir) =>
+          Effect.gen(function* () {
+            const filepath = path.join(dir, "doc.txt")
+            yield* putEncoded(filepath, original, encoding)
+            yield* markRead(filepath)
+
+            yield* runEdit({ filePath: filepath, oldString, newString })
+
+            const decoded = yield* loadDecoded(filepath, encoding)
+            const expected = original.replace(oldString, newString)
+            expect(decoded).toBe(expected)
+
+            const bytes = yield* loadBytes(filepath)
+            expect(bytes.equals(encodeBytes(expected, encoding))).toBe(true)
+          }),
+        ),
+      )
+    }
+  })
+
+  describe("ApplyPatchTool preserves encoding", () => {
+    it.live("preserves Shift_JIS through an update hunk", () =>
+      provideTmpdirInstance((dir) =>
+        Effect.gen(function* () {
+          const filepath = path.join(dir, "doc.txt")
+          const replacement = "日本語"
+          const original = "line1\n" + samples.shiftJis + "\nline3\n"
+          const expected = original.replace(samples.shiftJis, replacement)
+          yield* putEncoded(filepath, original, "Shift_JIS")
+
+          const patch = [
+            "*** Begin Patch",
+            "*** Update File: doc.txt",
+            "@@",
+            " line1",
+            "-" + samples.shiftJis,
+            "+" + replacement,
+            " line3",
+            "*** End Patch",
+          ].join("\n")
+
+          yield* runPatch({ patchText: patch })
+
+          const decoded = yield* loadDecoded(filepath, "Shift_JIS")
+          expect(decoded).toBe(expected)
+
+          // Bytes must still be Shift_JIS, not silently promoted to UTF-8.
+          const bytes = yield* loadBytes(filepath)
+          expect(bytes.equals(encodeBytes(expected, "Shift_JIS"))).toBe(true)
+        }),
+      ),
+    )
+
+    it.live("new files added via apply_patch are UTF-8", () =>
+      provideTmpdirInstance((dir) =>
+        Effect.gen(function* () {
+          const patch = ["*** Begin Patch", "*** Add File: new.txt", "+hello world", "*** End Patch"].join("\n")
+          yield* runPatch({ patchText: patch })
+          const bytes = yield* loadBytes(path.join(dir, "new.txt"))
+          expect(bytes.equals(Buffer.from("hello world\n", "utf-8"))).toBe(true)
+        }),
+      ),
+    )
+  })
+})
+
+// Shared helper to set up a temp instance with an encoded file at `file.txt`.
+function provideEncoded<A, E, R>(encoding: string, text: string, body: (filepath: string) => Effect.Effect<A, E, R>) {
+  return provideTmpdirInstance((dir) =>
+    Effect.gen(function* () {
+      const filepath = path.join(dir, "file.txt")
+      yield* putEncoded(filepath, text, encoding)
+      yield* markRead(filepath)
+      return yield* body(filepath)
+    }),
+  )
+}