diff --git a/script/upstream/README.md b/script/upstream/README.md index 97944397c2..bf120764a3 100644 --- a/script/upstream/README.md +++ b/script/upstream/README.md @@ -34,6 +34,7 @@ bun run merge.ts --version v1.1.50 --base-branch catrielmuller/kilo-opencode-v1. | `merge.ts` | Main orchestration script for upstream merges | | `list-versions.ts` | List available upstream versions | | `analyze.ts` | Analyze changes without merging | +| `fix-kilocode-markers.ts` | Rebuild `kilocode_change` markers for one file against the last merged upstream | ### Transform Scripts @@ -230,6 +231,18 @@ Options: --output Output file for report ``` +### fix-kilocode-markers.ts + +``` +Usage: + bun run script/upstream/fix-kilocode-markers.ts [--dry-run] + +Options: + --dry-run Show what would change without writing the file +``` + +The command finds the newest upstream tag already merged into `HEAD`, reads that upstream version of the file, applies the same branding transforms used by upstream merge automation, strips existing `kilocode_change` markers from the current file, and adds fresh markers around the remaining lines that differ from upstream. + ## Using Custom Base Branches By default, upstream merges start from the `main` branch. However, you can use `--base-branch` to start from a different branch. This is useful for: diff --git a/script/upstream/fix-kilocode-markers.ts b/script/upstream/fix-kilocode-markers.ts new file mode 100644 index 0000000000..0011583939 --- /dev/null +++ b/script/upstream/fix-kilocode-markers.ts @@ -0,0 +1,617 @@ +#!/usr/bin/env bun +/** + * Rebuild kilocode_change markers for one file by comparing it with the last + * merged upstream version. + * + * Usage: + * bun run script/upstream/fix-kilocode-markers.ts packages/opencode/src/file.ts + * bun run script/upstream/fix-kilocode-markers.ts packages/opencode/src/file.ts --dry-run + */ + +import { $ } from "bun" +import { mkdtemp, rm } from "node:fs/promises" +import { tmpdir } from "node:os" +import path from "node:path" +import { compareVersions, parseVersion, type VersionInfo } from "./utils/version" +import { isAncestor } from "./utils/git" +import { error, header, info, success, warn } from "./utils/logger" +import { transformI18nContent } from "./transforms/transform-i18n" +import { applyBrandingTransforms } from "./transforms/transform-take-theirs" +import { applyScriptTransforms } from "./transforms/transform-scripts" +import { applyTauriTransforms } from "./transforms/transform-tauri" +import { applyExtensionTransforms } from "./transforms/transform-extensions" +import { applyWebTransforms } from "./transforms/transform-web" +import { applyPackageNameTransforms } from "./transforms/package-names" + +interface Args { + file?: string + dryRun: boolean + help: boolean +} + +interface Text { + lines: string[] + eol: string + final: boolean +} + +interface Clean { + text: Text + marks: Marks +} + +interface Diff { + lines: Set + deleted: number +} + +interface Range { + start: number + end: number +} + +interface Block extends Range { + before: string + after: string +} + +interface Marks { + inline: Map + starts: Map + ends: Map + blocks: Block[] + file?: string +} + +type Style = "slash" | "hash" | "jsx" | "block" + +const standalone = [ + /^\s*\/\/\s*kilocode_change\b.*$/, + /^\s*#\s*kilocode_change\b.*$/, + /^\s*\{?\s*\/\*\s*kilocode_change\b.*\*\/\}?\s*$/, +] +const start = /\bkilocode_change\s+start\b/ +const end = /\bkilocode_change\s+end\b/ +const freshmark = /\bkilocode_change\s*-\s*new\s*file\b/ +const unsupported = new Set([".json", ".jsonc", ".lock", ".png", ".jpg", ".jpeg", ".gif", ".webp", ".ico"]) +const styles = new Map([ + [".ts", "slash"], + [".tsx", "slash"], + [".js", "slash"], + [".jsx", "slash"], + [".css", "block"], + [".yml", "hash"], + [".yaml", "hash"], + [".toml", "hash"], + [".sh", "hash"], + [".bash", "hash"], + [".zsh", "hash"], +]) +const workflows = [".github/workflows/publish.yml", ".github/workflows/beta.yml"] +const url = "https://github.com/anomalyco/opencode.git" +const exempt = ["script/upstream/"] + +function usage() { + console.log(`Usage: bun run script/upstream/fix-kilocode-markers.ts [--dry-run] + +Rebuilds kilocode_change markers by: + 1. Finding the newest upstream tag whose commit is already merged into HEAD. + 2. Applying upstream merge branding transforms to that upstream file. + 3. Comparing the transformed upstream file with the current working tree file. + 4. Removing existing kilocode_change markers and adding fresh markers around remaining changed lines. + +Options: + --dry-run Show what would change without writing the file. + --help Show this help message.`) +} + +function args(): Args { + const raw = process.argv.slice(2) + return { + file: raw.find((arg) => !arg.startsWith("--")), + dryRun: raw.includes("--dry-run"), + help: raw.includes("--help") || raw.includes("-h"), + } +} + +async function root() { + return (await $`git rev-parse --show-toplevel`.text()).trim() +} + +function normalize(root: string, file: string) { + if (path.isAbsolute(file)) throw new Error("File must be relative to the repo root") + if (file.includes("\0")) throw new Error("File path contains a null byte") + + const abs = path.resolve(root, file) + const rel = path.relative(root, abs).replaceAll(path.sep, "/") + + if (!rel || rel.startsWith("..") || path.isAbsolute(rel)) throw new Error("File must stay inside the repo") + return rel +} + +function ext(file: string) { + return path.extname(file).toLowerCase() +} + +function supported(file: string, text: string) { + const kind = ext(file) + if (unsupported.has(kind)) return false + if (styles.has(kind)) return true + return !kind && text.startsWith("#!") +} + +function annotates(file: string) { + return !exempt.some((scope) => file.startsWith(scope)) +} + +async function translate(file: string, text: string) { + const names = applyPackageNameTransforms(text).result + const script = applyScriptTransforms(names).result + const branded = applyBrandingTransforms(script).result + const i18n = transformI18nContent(branded).result + const tauri = applyTauriTransforms(i18n, file).result + const ext = applyExtensionTransforms(tauri, file).result + const web = applyWebTransforms(ext).result + + return workflow(file, web) +} + +function workflow(file: string, text: string) { + if (!workflows.includes(file)) return text + return text + .replace(/github\.repository == 'anomalyco\/opencode'/g, "github.repository == 'Kilo-Org/kilocode'") + .replace(/github\.repository == "anomalyco\/opencode"/g, 'github.repository == "Kilo-Org/kilocode"') + .replace(/\bopencode-ai\b/g, "@kilocode/cli") + .replace( + /GH_REPO:\s*\$\{\{ \(github\.ref_name == 'beta' && 'anomalyco\/opencode-beta'\) \|\| github\.repository \}\}/g, + "GH_REPO: ${{ github.repository }}", + ) +} + +function split(text: string): Text { + const eol = text.includes("\r\n") ? "\r\n" : "\n" + const final = text.endsWith("\n") + const body = final ? text.slice(0, text.endsWith("\r\n") ? -2 : -1) : text + return { lines: body ? body.split(/\r?\n/) : [], eol, final } +} + +function join(text: Text) { + return text.lines.join(text.eol) + (text.final ? text.eol : "") +} + +function strip(file: string, line: string): { line: string | null; mark?: string } { + if (standalone.some((item) => item.test(line))) return { line: null } + if (style(file) === "hash") return comment(line, [/^#\s*kilocode_change\b/]) + return comment(line, [/^\{\/\*\s*kilocode_change\b/, /^\/\*\s*kilocode_change\b/, /^\/\/\s*kilocode_change\b/]) +} + +function comment(line: string, tokens: RegExp[]) { + let quote = "" + let escape = false + + for (let i = 0; i < line.length; i++) { + const char = line[i] + if (!char) continue + + if (quote) { + if (escape) { + escape = false + continue + } + if (char === "\\") { + escape = true + continue + } + if (char === quote) quote = "" + continue + } + + if (char === '"' || char === "'" || char === "`") { + quote = char + continue + } + + const rest = line.slice(i) + if (tokens.some((item) => item.test(rest))) { + const next = line.slice(0, i).trimEnd() + return { line: next, mark: line.slice(next.length) } + } + } + + return { line } +} + +function clean(file: string, text: string): Clean { + const parsed = split(text) + const marks: Marks = { inline: new Map(), starts: new Map(), ends: new Map(), blocks: [] } + const lines: string[] = [] + const opens: { before: string; start?: number }[] = [] + + for (const line of parsed.lines) { + if (standalone.some((item) => item.test(line))) { + if (freshmark.test(line)) marks.file = line + if (start.test(line)) { + opens.push({ before: line }) + continue + } + if (end.test(line)) { + const open = opens.pop() + const last = lines.length - 1 + if (open?.start !== undefined && last >= open.start) { + marks.ends.set(last, line) + marks.blocks.push({ start: open.start, end: last, before: open.before, after: line }) + } + if (!open && last >= 0) marks.ends.set(last, line) + continue + } + continue + } + + const next = strip(file, line) + if (next.line === null) continue + + const index = lines.length + lines.push(next.line) + + for (const open of opens) { + if (open.start !== undefined) continue + open.start = index + marks.starts.set(index, open.before) + } + + if (next.mark) marks.inline.set(index, next.mark) + } + + return { text: { ...parsed, lines }, marks } +} + +async function last(): Promise { + const source = await remote() + + info(`Fetching upstream tags from ${source}...`) + const fetch = await $`git fetch ${source} --tags --force`.quiet().nothrow() + if (fetch.exitCode !== 0) throw new Error(`Failed to fetch upstream: ${fetch.stderr.toString()}`) + + const versions = await list(source) + for (const version of versions) { + if (await isAncestor(version.commit, "HEAD")) return version + } + + throw new Error("Could not find a merged upstream tag in HEAD") +} + +async function remote() { + const result = await $`git remote get-url upstream`.quiet().nothrow() + if (result.exitCode === 0) return "upstream" + + warn(`No 'upstream' remote found; using ${url}`) + return url +} + +async function list(source: string): Promise { + const result = await $`git ls-remote --tags ${source}`.quiet().nothrow() + if (result.exitCode !== 0) throw new Error(`Failed to list upstream tags: ${result.stderr.toString()}`) + + const found = new Map() + for (const line of result.stdout.toString().trim().split("\n")) { + const match = line.match(/^([a-f0-9]+)\s+refs\/tags\/([^^]+)(\^\{\})?$/) + if (!match) continue + + const commit = match[1] + const tag = match[2] + const peeled = Boolean(match[3]) + if (commit && tag && (peeled || !found.has(tag))) found.set(tag, commit) + } + + return [...found] + .flatMap(([tag, commit]) => { + const version = parseVersion(tag) + return version ? [{ version, tag, commit }] : [] + }) + .sort((a, b) => compareVersions(b.version, a.version)) +} + +async function upstream(ref: string, file: string) { + const spec = `${ref}:${file}` + const result = await $`git show ${spec}`.quiet().nothrow() + if (result.exitCode === 0) return result.stdout.toString() + + const stderr = result.stderr.toString() + if (stderr.includes("exists on disk") || stderr.includes("does not exist") || stderr.includes("Path")) return null + throw new Error(`Failed to read ${file} from ${ref}: ${stderr}`) +} + +function style(file: string): Style { + const kind = ext(file) + return styles.get(kind) ?? "hash" +} + +function context(file: string, text: Text, range: Range): Style { + const base = style(file) + if (![".tsx", ".jsx"].includes(ext(file))) return base + + if (tag(text.lines, range.start)) return "block" + if (child(text.lines, range.start)) return "jsx" + return base +} + +function nearby(lines: string[], start: number, step: number) { + for (let i = start; i >= 0 && i < lines.length; i += step) { + const line = lines[i]?.trim() + if (line) return line + } + return "" +} + +function tag(lines: string[], start: number) { + const current = lines[start]?.trim() ?? "" + if (!current) return false + if (/^[A-Za-z_$][\w$.:/-]*(=|\s*=)/.test(current)) return true + + for (let i = start - 1; i >= Math.max(0, start - 20); i--) { + const line = lines[i]?.trim() ?? "" + if (!line) continue + if (line.includes(">")) return false + if (/^<\/?[A-Za-z]/.test(line)) return true + } + + return false +} + +function child(lines: string[], start: number) { + const current = lines[start]?.trim() ?? "" + const prev = nearby(lines, start - 1, -1) + const next = nearby(lines, start + 1, 1) + + if (prev.endsWith(">") && !prev.endsWith("=>")) return true + if (next.startsWith(" a.start - b.start) + .reduce((acc, item) => { + const prev = acc.at(-1) + if (prev && item.start <= prev.end + 1) { + prev.end = Math.max(prev.end, item.end) + return acc + } + acc.push({ ...item }) + return acc + }, []) +} + +function ranges(nums: Set): Range[] { + const sorted = [...nums].sort((a, b) => a - b) + return merge( + sorted.reduce((acc, num) => { + const prev = acc.at(-1) + if (prev && num === prev.end + 1) { + prev.end = num + return acc + } + acc.push({ start: num, end: num }) + return acc + }, []), + ) +} + +function expand(found: Range[], marks: Marks) { + return merge( + found.map((range) => { + const next = { ...range } + for (const block of marks.blocks) { + if (next.end < block.start || next.start > block.end) continue + next.start = Math.min(next.start, block.start) + next.end = Math.max(next.end, block.end) + } + return next + }), + ) +} + +function boundary(line: string | undefined, kind: RegExp) { + if (!line) return false + return standalone.some((item) => item.test(line)) && kind.test(line) +} + +function gap(lines: string[], index: number) { + const next = lines.slice(index).findIndex((line) => line.trim() !== "") + return next === -1 ? -1 : index + next +} + +function collapse(lines: string[]): string[] { + const index = lines.findIndex((line, pos) => { + if (!boundary(line, end)) return false + const next = gap(lines, pos + 1) + return next !== -1 && boundary(lines[next], start) + }) + if (index === -1) return lines + + const next = gap(lines, index + 1) + return collapse(lines.filter((_, pos) => pos !== index && pos !== next)) +} + +function saved(marks: Marks, range: Range) { + return marks.blocks.find((block) => block.start === range.start && block.end === range.end) +} + +function annotate(file: string, clean: Clean, found: Range[]) { + const text = clean.text + const marks = clean.marks + const lines = [...text.lines] + + for (const range of expand(found, marks).reverse()) { + const mode = context(file, text, range) + const prior = saved(marks, range) + const before = prior?.before ?? marks.starts.get(range.start) + const after = prior?.after ?? marks.ends.get(range.end) + + if (!before && !after && range.start === range.end && inline(file, text.lines, range, mode)) { + lines[range.start] = `${lines[range.start]}${marks.inline.get(range.start) ?? note(mode)}` + continue + } + + const pad = indent(text.lines[range.start] ?? "") + const fallback = block(mode, pad) + const pair = { + start: before ?? fallback.start, + end: after ?? fallback.end, + } + lines.splice(range.end + 1, 0, pair.end) + lines.splice(range.start, 0, pair.start) + } + + return join({ ...text, lines: collapse(lines) }) +} + +function fresh(file: string, clean: Clean) { + const lines = [...clean.text.lines] + const mode = style(file) + const line = clean.marks.file ?? (mode === "hash" ? "# kilocode_change - new file" : "// kilocode_change - new file") + const at = lines[0]?.startsWith("#!") ? 1 : 0 + lines.splice(at, 0, line) + return join({ ...clean.text, lines }) +} + +function patch(out: string): Diff { + const lines = new Set() + const state = { next: 0, deleted: 0, added: 0, removed: 0 } + const flush = () => { + if (state.removed > 0 && state.added === 0) state.deleted += state.removed + state.added = 0 + state.removed = 0 + } + + for (const line of out.split("\n")) { + const hunk = line.match(/^@@ -\d+(?:,\d+)? \+(\d+)(?:,(\d+))? @@/) + if (hunk) { + flush() + state.next = Number(hunk[1]) - 1 + continue + } + + if (line.startsWith("+++") || line.startsWith("---")) continue + if (line.startsWith("+")) { + if (line.slice(1).trim()) lines.add(state.next) + state.added++ + state.next++ + continue + } + if (line.startsWith("-")) { + state.removed++ + continue + } + if (line.startsWith(" ")) state.next++ + } + + flush() + return { lines, deleted: state.deleted } +} + +async function changed(base: Text, head: Text): Promise { + const dir = await mkdtemp(path.join(tmpdir(), "kilo-markers-")) + const left = path.join(dir, "upstream") + const right = path.join(dir, "current") + + try { + await Bun.write(left, join({ ...base, eol: "\n" })) + await Bun.write(right, join({ ...head, eol: "\n" })) + + const result = await $`git diff --no-index --no-ext-diff --unified=0 -- ${left} ${right}`.quiet().nothrow() + if (result.exitCode === 0) return { lines: new Set(), deleted: 0 } + if (result.exitCode === 1) return patch(result.stdout.toString()) + throw new Error(result.stderr.toString()) + } finally { + await rm(dir, { recursive: true, force: true }) + } +} + +async function main() { + const opts = args() + if (opts.help) { + usage() + return + } + if (!opts.file) { + usage() + process.exit(1) + } + + const top = await root() + process.chdir(top) + + const file = normalize(top, opts.file) + const abs = path.join(top, file) + const current = await Bun.file(abs).text() + if (!supported(file, current)) throw new Error(`Cannot safely add comment markers to ${file}`) + if (current.includes("\0")) throw new Error(`${file} appears to be binary`) + + header("Fix kilocode_change markers") + + const version = await last() + success(`Last merged upstream: ${version.tag} (${version.commit.slice(0, 8)})`) + + const base = await upstream(version.commit, file) + const head = clean(file, current) + const baseText = base === null ? null : await translate(file, base) + const diff = baseText === null ? null : await changed(clean(file, baseText).text, head.text) + const found = ranges(diff?.lines ?? new Set()) + const next = base === null ? fresh(file, head) : annotate(file, head, found) + + if (base === null && annotates(file)) warn(`${file} does not exist upstream; marked as a new Kilo file`) + if (base === null && !annotates(file)) warn(`${file} does not exist upstream`) + if (diff && diff.deleted > 0) + warn(`${diff.deleted} upstream-only deleted line(s) cannot be annotated in the current file`) + if (!annotates(file)) warn(`${file} is exempt from annotation checks; this command still reports differences`) + if (!annotates(file)) { + success(`${file} differs from ${version.tag} in ${found.length} range(s)`) + return + } + + if (next === current) { + success(`${file} already has normalized kilocode_change markers`) + return + } + + if (opts.dryRun) { + info(`[DRY-RUN] Would update ${file}`) + return + } + + await Bun.write(abs, next) + success(`Updated ${file}`) +} + +main().catch((err) => { + error(err instanceof Error ? err.message : String(err)) + process.exit(1) +}) diff --git a/script/upstream/package.json b/script/upstream/package.json index eb95db9908..268c04fdc5 100644 --- a/script/upstream/package.json +++ b/script/upstream/package.json @@ -13,6 +13,7 @@ "transform:strings": "bun run codemods/transform-strings.ts", "transform:all": "bun run transforms/package-names.ts && bun run codemods/transform-imports.ts && bun run codemods/transform-strings.ts", "versions": "bun run transforms/preserve-versions.ts", + "fix:markers": "bun run fix-kilocode-markers.ts", "keep-ours": "bun run transforms/keep-ours.ts" }, "dependencies": { diff --git a/script/upstream/transforms/package-names.ts b/script/upstream/transforms/package-names.ts index 84a9115d18..6614249ffb 100644 --- a/script/upstream/transforms/package-names.ts +++ b/script/upstream/transforms/package-names.ts @@ -14,7 +14,7 @@ */ import { Glob } from "bun" -import { info, success, warn, debug } from "../utils/logger" +import { info, success } from "../utils/logger" import { defaultConfig } from "../utils/config" export interface TransformResult { @@ -91,27 +91,34 @@ const PACKAGE_PATTERNS = [ { pattern: /opencode\.db/g, replacement: "kilo.db" }, ] +/** + * Apply package name and branding transforms to content. + */ +export function applyPackageNameTransforms(input: string): { result: string; changes: number } { + return PACKAGE_PATTERNS.reduce( + (state, { pattern, replacement }) => { + const regex = typeof pattern === "string" ? new RegExp(pattern, "g") : pattern + regex.lastIndex = 0 + const count = (state.result.match(regex) || []).length + regex.lastIndex = 0 + const result = state.result.replace(regex, replacement) + if (result === state.result) return state + return { result, changes: state.changes + count } + }, + { result: input, changes: 0 }, + ) +} + /** * Transform package names in a single file */ export async function transformFile(filePath: string, options: TransformOptions = {}): Promise { const file = Bun.file(filePath) - let content = await file.text() - const original = content - let changes = 0 - - for (const { pattern, replacement } of PACKAGE_PATTERNS) { - const regex = typeof pattern === "string" ? new RegExp(pattern, "g") : pattern - const newContent = content.replace(regex, replacement) - if (newContent !== content) { - const count = (content.match(regex) || []).length - changes += count - content = newContent - } - } + const input = await file.text() + const { result, changes } = applyPackageNameTransforms(input) if (changes > 0 && !options.dryRun) { - await Bun.write(filePath, content) + await Bun.write(filePath, result) } return {