diff --git a/.changeset/indexing-file-extensions.md b/.changeset/indexing-file-extensions.md new file mode 100644 index 0000000000..c464e7a19a --- /dev/null +++ b/.changeset/indexing-file-extensions.md @@ -0,0 +1,7 @@ +--- +"@kilocode/cli": minor +"@kilocode/kilo-indexing": minor +"kilo-code": minor +--- + +Configure a custom file extension allowlist for codebase indexing to limit scans to relevant project files and support additional text formats. diff --git a/bun.lock b/bun.lock index c9d80961ef..44a2d20472 100644 --- a/bun.lock +++ b/bun.lock @@ -166,6 +166,7 @@ "name": "@kilocode/kilo-console", "version": "7.4.11", "dependencies": { + "@kilocode/kilo-indexing": "workspace:*", "@kilocode/kilo-web-ui": "workspace:*", "@kilocode/sdk": "workspace:*", "@lottiefiles/dotlottie-web": "0.74.0", diff --git a/packages/kilo-console/package.json b/packages/kilo-console/package.json index 3b75d93151..9553caff22 100755 --- a/packages/kilo-console/package.json +++ b/packages/kilo-console/package.json @@ -11,6 +11,7 @@ "typecheck": "tsgo --noEmit" }, "dependencies": { + "@kilocode/kilo-indexing": "workspace:*", "@kilocode/kilo-web-ui": "workspace:*", "@kilocode/sdk": "workspace:*", "@lottiefiles/dotlottie-web": "0.74.0", diff --git a/packages/kilo-console/src/routes/config/IndexingRoute.tsx b/packages/kilo-console/src/routes/config/IndexingRoute.tsx index 2b5463e939..109ceb01e9 100644 --- a/packages/kilo-console/src/routes/config/IndexingRoute.tsx +++ b/packages/kilo-console/src/routes/config/IndexingRoute.tsx @@ -6,7 +6,16 @@ import { CustomSelect, type SelectOption } from "../../components/CustomSelect" import { loadEmbeddingModels } from "../../client" import { useConfig } from "../../context/config" import { ConfigPage, ConfigTag as Tag, SourceBadge } from "./ConfigPage" -import { clean, clone, merge, providerPatch, removed, shouldSync, validate } from "./state/indexing" +import { + clean, + clone, + merge, + parseFileExtensions, + providerPatch, + removed, + shouldSync, + validate, +} from "./state/indexing" type Provider = NonNullable type ProviderValue = Provider | "" @@ -111,6 +120,7 @@ export function IndexingRoute() { const [draft, setDraft] = createSignal({}) const [source, setSource] = createSignal("") const [dirty, setDirty] = createSignal(false) + const [extensionText, setExtensionText] = createSignal("") const scope = () => ctx.query()?.scope ?? "global" const [selected, setSelected] = createSignal(scope()) const project = () => scope() === "project" @@ -149,6 +159,8 @@ export function IndexingRoute() { setSelected(current) setSource(key) setDraft(clone(next)) + const extensions = project() ? merge(global(), next).fileExtensions : next.fileExtensions + setExtensionText(extensions?.join(", ") ?? "") setDirty(false) }) @@ -478,6 +490,28 @@ export function IndexingRoute() {
+ + } + > + { + const value = event.currentTarget.value + setExtensionText(value) + update({ fileExtensions: parseFileExtensions(value) }) + }} + /> + { test("merges project settings over nested global settings", () => { @@ -59,4 +59,13 @@ describe("indexing config state", () => { "Embedding batch size must be a positive integer.", ]) }) + + test("parses and cleans file extension allowlists", () => { + expect(parseFileExtensions(" PHP, , .JS, js, C++, c++ ")).toEqual([".c++", ".js", ".php"]) + expect(parseFileExtensions(" , , ")).toBeUndefined() + expect(clean({ fileExtensions: [] })).toEqual({}) + expect(validate({ fileExtensions: ["*.js", ".d.ts", "src/php"] })).toEqual([ + "File extensions must contain only a name with an optional leading dot.", + ]) + }) }) diff --git a/packages/kilo-console/src/routes/config/state/indexing.ts b/packages/kilo-console/src/routes/config/state/indexing.ts index 1882b16df1..28fb10aa02 100644 --- a/packages/kilo-console/src/routes/config/state/indexing.ts +++ b/packages/kilo-console/src/routes/config/state/indexing.ts @@ -1,4 +1,7 @@ import type { IndexingConfig } from "@kilocode/sdk/v2/client" +import { isFileExtension } from "@kilocode/kilo-indexing/file-extensions" + +export { parseFileExtensions } from "@kilocode/kilo-indexing/file-extensions" function record(input: unknown): input is Record { return typeof input === "object" && input !== null && !Array.isArray(input) @@ -26,6 +29,7 @@ export function merge(base: IndexingConfig | undefined, patch: IndexingConfig | function prune(input: unknown): unknown { if (typeof input === "string") return input.trim() || undefined + if (Array.isArray(input)) return input.length > 0 ? input : undefined if (!record(input)) return input ?? undefined const entries = Object.entries(input).flatMap(([key, value]) => { const next = prune(value) @@ -64,6 +68,9 @@ export function removed(before: IndexingConfig, after: IndexingConfig): string[] export function validate(input: IndexingConfig): string[] { const errors: string[] = [] + if (input.fileExtensions?.some((item) => !isFileExtension(item))) { + errors.push("File extensions must contain only a name with an optional leading dot.") + } if (input.dimension !== undefined && input.dimension !== null) { if (!Number.isInteger(input.dimension) || input.dimension <= 0) errors.push("Vector dimension must be a positive integer.") diff --git a/packages/kilo-docs/pages/customize/context/codebase-indexing.md b/packages/kilo-docs/pages/customize/context/codebase-indexing.md index 40a8327211..1e995fc2ef 100644 --- a/packages/kilo-docs/pages/customize/context/codebase-indexing.md +++ b/packages/kilo-docs/pages/customize/context/codebase-indexing.md @@ -51,6 +51,7 @@ You can also edit the `indexing` section in `kilo.jsonc` directly: "enabled": true, "provider": "openai", "model": "text-embedding-3-small", + "fileExtensions": [".php", ".js", ".css"], "vectorStore": "lancedb", "openai": { "apiKey": "sk-..." }, "lancedb": {} @@ -129,6 +130,7 @@ You can also edit the `indexing` section directly. This is the full shape of the "apiKey": "pa-..." }, "lancedb": {}, + "fileExtensions": [".php", ".js", ".css"], "searchMinScore": 0.4, "searchMaxResults": 50, "embeddingBatchSize": 60, @@ -227,6 +229,18 @@ The interface shows real-time status: ### Automatic File Filtering +Set `indexing.fileExtensions` to a non-empty array to index only the listed file extensions. Values are case-insensitive and may be written with or without a leading dot. When this setting is omitted, Kilo uses its built-in language list. Configured text formats without a Tree-sitter parser use line-based fallback chunking. + +```json +{ + "indexing": { + "fileExtensions": [".php", ".js", ".css"] + } +} +``` + +The configured list replaces the built-in defaults rather than adding to them. Clear the field in the settings UI to restore the defaults, or to inherit the global list from project scope. + The indexer automatically excludes: - Binary files and images diff --git a/packages/kilo-indexing/package.json b/packages/kilo-indexing/package.json index 223623968d..b95a9cbbe1 100644 --- a/packages/kilo-indexing/package.json +++ b/packages/kilo-indexing/package.json @@ -19,6 +19,7 @@ "./detect": "./src/detect.ts", "./embedding-models": "./src/kilo-embedding-models.ts", "./engine": "./src/indexing/index.ts", + "./file-extensions": "./src/file-extensions.ts", "./server": "./src/server/routes.ts", "./status": "./src/status.ts" }, diff --git a/packages/kilo-indexing/src/config.ts b/packages/kilo-indexing/src/config.ts index b68097af85..7dfaa3b53b 100644 --- a/packages/kilo-indexing/src/config.ts +++ b/packages/kilo-indexing/src/config.ts @@ -3,8 +3,10 @@ import z from "zod" import type { IndexingConfigInput } from "./indexing/config-manager" import { DEFAULT_VECTOR_STORE } from "./indexing/constants" import type { EmbedderProvider } from "./indexing/interfaces/manager" +import { FILE_EXTENSION_PATTERN, normalizeFileExtensions } from "./file-extensions" export { DEFAULT_VECTOR_STORE } from "./indexing/constants" +export { isFileExtension, normalizeFileExtensions, parseFileExtensions } from "./file-extensions" const providers = [ "kilo", @@ -128,6 +130,11 @@ export const IndexingConfig = z .positive() .optional() .describe("Maximum retry attempts for failed embedding batches (default: 3)"), + fileExtensions: z + .array(z.string().trim().regex(FILE_EXTENSION_PATTERN)) + .min(1) + .optional() + .describe("File extension allowlist for codebase indexing (uses built-in defaults if omitted)"), }) .strict() .meta({ ref: "IndexingConfig" }) @@ -229,6 +236,15 @@ export const IndexingSchema = Schema.Struct({ scannerMaxBatchRetries: Schema.optional(PositiveInt).annotate({ description: "Maximum retry attempts for failed embedding batches (default: 3)", }), + fileExtensions: Schema.optional( + Schema.mutable( + Schema.Array(Schema.String.check(Schema.isPattern(/^\s*\.?[A-Za-z0-9][A-Za-z0-9_+-]*\s*$/))).check( + Schema.isMinLength(1), + ), + ), + ).annotate({ + description: "File extension allowlist for codebase indexing (uses built-in defaults if omitted)", + }), }).annotate({ identifier: "IndexingConfig", description: "Codebase indexing configuration", @@ -250,6 +266,7 @@ export function toIndexingConfigInput(cfg: IndexingConfig | undefined): Indexing searchMaxResults: cfg?.searchMaxResults, embeddingBatchSize: cfg?.embeddingBatchSize, scannerMaxBatchRetries: cfg?.scannerMaxBatchRetries, + fileExtensions: normalizeFileExtensions(cfg?.fileExtensions), kiloApiKey: cfg?.kilo?.apiKey, kiloBaseUrl: cfg?.kilo?.baseUrl, kiloOrganizationId: cfg?.kilo?.organizationId, diff --git a/packages/kilo-indexing/src/file-extensions.ts b/packages/kilo-indexing/src/file-extensions.ts new file mode 100644 index 0000000000..eed990407b --- /dev/null +++ b/packages/kilo-indexing/src/file-extensions.ts @@ -0,0 +1,24 @@ +export const FILE_EXTENSION_PATTERN = /^\.?[A-Za-z0-9][A-Za-z0-9_+-]*$/ + +export function isFileExtension(input: string): boolean { + return FILE_EXTENSION_PATTERN.test(input.trim()) +} + +export function normalizeFileExtensions(input: readonly string[] | undefined): string[] | undefined { + if (!input) return undefined + const values = new Set() + for (const raw of input) { + const item = raw.trim().toLowerCase() + if (!item) continue + values.add(item.startsWith(".") ? item : `.${item}`) + } + return values.size > 0 ? [...values].sort() : undefined +} + +export function parseFileExtensions(input: string): string[] | undefined { + const values = input + .split(",") + .map((item) => item.trim()) + .filter(Boolean) + return values.length > 0 ? normalizeFileExtensions(values) : undefined +} diff --git a/packages/kilo-indexing/src/indexing/config-manager.ts b/packages/kilo-indexing/src/indexing/config-manager.ts index 574ea64420..4f5da3410d 100644 --- a/packages/kilo-indexing/src/indexing/config-manager.ts +++ b/packages/kilo-indexing/src/indexing/config-manager.ts @@ -3,6 +3,7 @@ import type { CodeIndexConfig, PreviousConfigSnapshot } from "./interfaces/confi import { DEFAULT_SEARCH_MIN_SCORE, DEFAULT_MAX_SEARCH_RESULTS, DEFAULT_VECTOR_STORE } from "./constants" import { getDefaultModelId, getModelDimension, getModelScoreThreshold } from "./model-registry" import { isEmbeddingProfileEqual, resolveEmbeddingProfile } from "./embedding-profile" +import { resolveFileExtensions } from "./shared/supported-extensions" /** * Raw input fed to CodeIndexConfigManager from the host environment. @@ -22,6 +23,7 @@ export interface IndexingConfigInput { searchMaxResults?: number embeddingBatchSize?: number scannerMaxBatchRetries?: number + fileExtensions?: string[] kiloApiKey?: string kiloBaseUrl?: string kiloOrganizationId?: string @@ -70,6 +72,7 @@ export class CodeIndexConfigManager { private searchMaxResults?: number private embeddingBatchSize?: number private scannerMaxBatchRetries?: number + private fileExtensions: string[] = resolveFileExtensions(undefined) constructor(input: IndexingConfigInput) { this.applyInput(input) @@ -96,6 +99,7 @@ export class CodeIndexConfigManager { this.searchMaxResults = input.searchMaxResults this.embeddingBatchSize = input.embeddingBatchSize this.scannerMaxBatchRetries = input.scannerMaxBatchRetries + this.fileExtensions = resolveFileExtensions(input.fileExtensions) this.modelId = input.modelId // Validate and set model dimension @@ -153,6 +157,7 @@ export class CodeIndexConfigManager { voyageApiKey: this.voyageOptions?.apiKey ?? "", qdrantUrl: this.qdrantUrl ?? "", qdrantApiKey: this.qdrantApiKey ?? "", + fileExtensions: [...this.fileExtensions], } } @@ -230,6 +235,8 @@ export class CodeIndexConfigManager { if ((prev.qdrantUrl ?? "") !== (this.qdrantUrl ?? "") || (prev.qdrantApiKey ?? "") !== (this.qdrantApiKey ?? "")) return true + if (prev.fileExtensions.join("\0") !== this.fileExtensions.join("\0")) return true + if (this.hasEmbeddingProfileChanged(prevProvider, prev.modelId, prev.modelDimension)) return true return false @@ -276,6 +283,7 @@ export class CodeIndexConfigManager { searchMaxResults: this.currentSearchMaxResults, embeddingBatchSize: this.currentEmbeddingBatchSize, scannerMaxBatchRetries: this.currentScannerMaxBatchRetries, + fileExtensions: [...this.fileExtensions], } } diff --git a/packages/kilo-indexing/src/indexing/interfaces/config.ts b/packages/kilo-indexing/src/indexing/interfaces/config.ts index a0eee87973..45b087d641 100644 --- a/packages/kilo-indexing/src/indexing/interfaces/config.ts +++ b/packages/kilo-indexing/src/indexing/interfaces/config.ts @@ -30,6 +30,7 @@ export interface CodeIndexConfig { searchMaxResults?: number embeddingBatchSize?: number scannerMaxBatchRetries?: number + fileExtensions: string[] } export type PreviousConfigSnapshot = { @@ -57,4 +58,5 @@ export type PreviousConfigSnapshot = { voyageApiKey?: string qdrantUrl?: string qdrantApiKey?: string + fileExtensions: string[] } diff --git a/packages/kilo-indexing/src/indexing/processors/file-watcher.ts b/packages/kilo-indexing/src/indexing/processors/file-watcher.ts index 4a0d925e47..14fd793d73 100644 --- a/packages/kilo-indexing/src/indexing/processors/file-watcher.ts +++ b/packages/kilo-indexing/src/indexing/processors/file-watcher.ts @@ -14,6 +14,7 @@ import { import { scannerExtensions } from "../shared/supported-extensions" import { type IFileWatcher, + type ICodeParser, type FileProcessingResult, type IEmbedder, type IVectorStore, @@ -33,6 +34,7 @@ import { Log } from "../../util/log" import type { WorktreeOverlay } from "../worktree-overlay" import { sanitizeErrorMessage } from "../shared/validation-helpers" import type { IgnoreMatcher } from "../shared/load-ignore" +import { isBinary } from "../shared/is-binary" const log = Log.create({ service: "file-watcher" }) @@ -56,6 +58,7 @@ export class FileWatcher implements IFileWatcher { private drainTask?: Promise private ready?: Promise private overlay?: WorktreeOverlay + private readonly extensions: ReadonlySet public readonly onDidStartBatchProcessing = new Emitter() public readonly onBatchProgressUpdate = new Emitter<{ @@ -75,12 +78,15 @@ export class FileWatcher implements IFileWatcher { maxBatchRetries?: number, private readonly onTelemetry?: IndexingTelemetryReporter, private readonly telemetryMeta?: IndexingTelemetryMeta, + extensions: readonly string[] = scannerExtensions, + private readonly parser: ICodeParser = codeParser, ) { if (ignoreInstance) { this.ignoreInstance = ignoreInstance } this.batchSegmentThreshold = batchSegmentThreshold ?? BATCH_SEGMENT_THRESHOLD this.maxBatchRetries = maxBatchRetries ?? MAX_BATCH_RETRIES + this.extensions = new Set(extensions) } private emitRetry(attempt: number, batchSize: number, err: unknown): void { @@ -271,7 +277,7 @@ export class FileWatcher implements IFileWatcher { const ext = path.extname(filePath).toLowerCase() if (FileIgnore.match(relativeFilePath)) return false if (this.ignoreInstance?.ignores(relativeFilePath)) return false - return scannerExtensions.includes(ext) || !path.extname(filePath) + return this.extensions.has(ext) } /** @@ -657,6 +663,14 @@ export class FileWatcher implements IFileWatcher { */ async processFile(filePath: string): Promise { try { + if (!this.extensions.has(path.extname(filePath).toLowerCase())) { + return { + path: filePath, + status: "skipped" as const, + reason: "File extension is not configured for indexing", + } + } + // Check if file is in an ignored directory const relativeFilePath = generateRelativeIgnorePath(filePath, this.workspacePath) if (!relativeFilePath) { @@ -695,7 +709,16 @@ export class FileWatcher implements IFileWatcher { } // Read file content - const content = await readFile(filePath, "utf-8") + const bytes = await readFile(filePath) + if (isBinary(bytes)) { + this.cacheManager.deleteHash(filePath) + return { + path: filePath, + status: "skipped" as const, + reason: "File is binary", + } + } + const content = bytes.toString("utf-8") // Calculate hash const newHash = createHash("sha256").update(content).digest("hex") @@ -710,7 +733,7 @@ export class FileWatcher implements IFileWatcher { } // Parse file - const blocks = await codeParser.parseFile(filePath, { content, fileHash: newHash }) + const blocks = await this.parser.parseFile(filePath, { content, fileHash: newHash }) // Prepare points for batch processing let pointsToUpsert: PointStruct[] = [] diff --git a/packages/kilo-indexing/src/indexing/processors/parser.ts b/packages/kilo-indexing/src/indexing/processors/parser.ts index 50a3ea2f9c..6185a31d24 100644 --- a/packages/kilo-indexing/src/indexing/processors/parser.ts +++ b/packages/kilo-indexing/src/indexing/processors/parser.ts @@ -20,6 +20,11 @@ export class CodeParser implements ICodeParser { private pendingLoads: Map> = new Map() private failedParsers: Set = new Set() private parserFallbackNotified: Set = new Set() + private readonly extensions: ReadonlySet + + constructor(extensions: readonly string[] = scannerExtensions) { + this.extensions = new Set(extensions) + } // Markdown files are now supported using the custom markdown parser // which extracts headers and sections for semantic indexing @@ -93,7 +98,7 @@ export class CodeParser implements ICodeParser { * @returns Boolean indicating if the language is supported */ private isSupportedLanguage(extension: string): boolean { - return scannerExtensions.includes(extension) + return this.extensions.has(extension) } /** diff --git a/packages/kilo-indexing/src/indexing/processors/scanner.ts b/packages/kilo-indexing/src/indexing/processors/scanner.ts index a746b5d456..c95e7ec608 100644 --- a/packages/kilo-indexing/src/indexing/processors/scanner.ts +++ b/packages/kilo-indexing/src/indexing/processors/scanner.ts @@ -28,6 +28,7 @@ import { Log } from "../../util/log" import { sanitizeErrorMessage } from "../shared/validation-helpers" import type { IndexingTelemetryMeta, IndexingTelemetryMode, IndexingTelemetryReporter } from "../interfaces/telemetry" import type { IgnoreMatcher } from "../shared/load-ignore" +import { isBinary } from "../shared/is-binary" const log = Log.create({ service: "indexing-scanner" }) @@ -35,6 +36,7 @@ export class DirectoryScanner implements IDirectoryScanner { private _cancelled = false private batchSegmentThreshold: number private maxBatchRetries: number + private readonly extensions: ReadonlySet constructor( private readonly embedder: IEmbedder, @@ -46,9 +48,11 @@ export class DirectoryScanner implements IDirectoryScanner { maxBatchRetries?: number, private readonly onTelemetry?: IndexingTelemetryReporter, private readonly telemetryMeta?: IndexingTelemetryMeta, + extensions: readonly string[] = scannerExtensions, ) { this.batchSegmentThreshold = batchSegmentThreshold ?? BATCH_SEGMENT_THRESHOLD this.maxBatchRetries = maxBatchRetries ?? MAX_BATCH_RETRIES + this.extensions = new Set(extensions) } private emitFileCount(mode: IndexingTelemetryMode, discovered: number, candidate: number): void { @@ -166,7 +170,7 @@ export class DirectoryScanner implements IDirectoryScanner { return false } - return scannerExtensions.includes(ext) && !this.ignoreInstance.ignores(relativeFilePath) + return this.extensions.has(ext) && !this.ignoreInstance.ignores(relativeFilePath) }) log.info("discovered candidate files for indexing", { workspacePath: scanWorkspace, @@ -265,7 +269,12 @@ export class DirectoryScanner implements IDirectoryScanner { } // Read file content using fs/promises - const content = await readFile(filePath, "utf-8") + const bytes = await readFile(filePath) + if (isBinary(bytes)) { + skippedCount++ + return + } + const content = bytes.toString("utf-8") if (this._cancelled) { return diff --git a/packages/kilo-indexing/src/indexing/search-service.ts b/packages/kilo-indexing/src/indexing/search-service.ts index 139c1a41a8..2fded9bfb5 100644 --- a/packages/kilo-indexing/src/indexing/search-service.ts +++ b/packages/kilo-indexing/src/indexing/search-service.ts @@ -23,6 +23,12 @@ export class CodeIndexSearchService { private readonly baseline?: BaselineSearch, ) {} + private allowed(result: VectorStoreSearchResult, extensions: ReadonlySet): boolean { + const file = result.payload?.filePath + if (typeof file !== "string") return false + return extensions.has(path.extname(file).toLowerCase()) + } + public async searchIndex(query: string, directoryPrefix?: string): Promise { if (!this.configManager.isFeatureEnabled || !this.configManager.isFeatureConfigured) { throw new Error("Code index feature is disabled or not configured.") @@ -44,7 +50,11 @@ export class CodeIndexSearchService { } const normalizedPrefix = directoryPrefix ? path.normalize(directoryPrefix) : undefined - if (!this.baseline) return await this.vectorStore.search(vector, normalizedPrefix, minScore, maxResults) + const extensions = new Set(this.configManager.getConfig().fileExtensions) + if (!this.baseline) { + const results = await this.vectorStore.search(vector, normalizedPrefix, minScore, maxResults) + return results.filter((result) => this.allowed(result, extensions)) + } if (!this.baseline.overlay.ready) throw new Error("Worktree index reconciliation is not complete.") const ceiling = Math.max(maxResults, Math.min(maxResults * 16, 1000)) @@ -77,9 +87,12 @@ export class CodeIndexSearchService { "\0", ) - for (const result of baseline) merged.set(key(result), result) + for (const result of baseline) { + if (this.allowed(result, extensions)) merged.set(key(result), result) + } for (const result of current) { - if (this.baseline.overlay.deltaResult(result)) merged.set(key(result), result) + if (this.allowed(result, extensions) && this.baseline.overlay.deltaResult(result)) + merged.set(key(result), result) } return [...merged.values()].sort((left, right) => right.score - left.score).slice(0, maxResults) } catch (err) { diff --git a/packages/kilo-indexing/src/indexing/service-factory.ts b/packages/kilo-indexing/src/indexing/service-factory.ts index 837578edb9..736f2137e6 100644 --- a/packages/kilo-indexing/src/indexing/service-factory.ts +++ b/packages/kilo-indexing/src/indexing/service-factory.ts @@ -15,7 +15,7 @@ import { OpenRouterEmbedder } from "./embedders/openrouter" import { VoyageEmbedder } from "./embedders/voyage" import { QdrantVectorStore } from "./vector-store/qdrant-client" import { LanceDBVectorStore } from "./vector-store/lancedb-vector-store" -import { codeParser, DirectoryScanner, FileWatcher } from "./processors" +import { CodeParser, DirectoryScanner, FileWatcher } from "./processors" import type { AvailableEmbedders, ICodeParser, IEmbedder, IFileWatcher, IVectorStore } from "./interfaces" import type { CodeIndexConfigManager } from "./config-manager" import type { CacheManager } from "./cache-manager" @@ -240,6 +240,7 @@ export class CodeIndexServiceFactory { config.scannerMaxBatchRetries, this.onTelemetry, meta, + config.fileExtensions, ) } @@ -248,6 +249,7 @@ export class CodeIndexServiceFactory { vectorStore: IVectorStore, cacheManager: CacheManager, ignoreInstance: IgnoreMatcher, + parser: ICodeParser, ): IFileWatcher { const config = this.configManager.getConfig() const meta = this.getTelemetryMeta() @@ -261,6 +263,8 @@ export class CodeIndexServiceFactory { config.scannerMaxBatchRetries, this.onTelemetry, meta, + config.fileExtensions, + parser, ) } @@ -289,9 +293,9 @@ export class CodeIndexServiceFactory { const embedder = this.createEmbedder() const vectorStore = this.createVectorStore() - const parser = codeParser + const parser = new CodeParser(config.fileExtensions) const scanner = this.createDirectoryScanner(embedder, vectorStore, parser, ignoreInstance) - const fileWatcher = this.createFileWatcher(embedder, vectorStore, cacheManager, ignoreInstance) + const fileWatcher = this.createFileWatcher(embedder, vectorStore, cacheManager, ignoreInstance, parser) log.info("indexing services created", { workspacePath: this.workspacePath, diff --git a/packages/kilo-indexing/src/indexing/shared/is-binary.ts b/packages/kilo-indexing/src/indexing/shared/is-binary.ts new file mode 100644 index 0000000000..18be88a077 --- /dev/null +++ b/packages/kilo-indexing/src/indexing/shared/is-binary.ts @@ -0,0 +1,14 @@ +const SAMPLE_BYTES = 4096 + +export function isBinary(input: Uint8Array): boolean { + const length = Math.min(input.length, SAMPLE_BYTES) + if (length === 0) return false + + let control = 0 + for (let index = 0; index < length; index++) { + const byte = input[index] + if (byte === 0) return true + if (byte < 9 || (byte > 13 && byte < 32)) control++ + } + return control / length > 0.3 +} diff --git a/packages/kilo-indexing/src/indexing/shared/supported-extensions.ts b/packages/kilo-indexing/src/indexing/shared/supported-extensions.ts index 1a85092e7a..e7a443fc6c 100644 --- a/packages/kilo-indexing/src/indexing/shared/supported-extensions.ts +++ b/packages/kilo-indexing/src/indexing/shared/supported-extensions.ts @@ -1,8 +1,13 @@ import { extensions as allExtensions } from "../../tree-sitter" +import { normalizeFileExtensions } from "../../file-extensions" // Include all extensions including markdown for the scanner export const scannerExtensions = allExtensions +export function resolveFileExtensions(input: readonly string[] | undefined): string[] { + return normalizeFileExtensions(input) ?? [...scannerExtensions] +} + /** * Extensions that should always use fallback chunking instead of tree-sitter parsing. * diff --git a/packages/kilo-indexing/test/kilocode/indexing/config-manager.test.ts b/packages/kilo-indexing/test/kilocode/indexing/config-manager.test.ts index 091ff3bbd9..68f4d6feb3 100644 --- a/packages/kilo-indexing/test/kilocode/indexing/config-manager.test.ts +++ b/packages/kilo-indexing/test/kilocode/indexing/config-manager.test.ts @@ -1,5 +1,10 @@ import { describe, expect, test } from "bun:test" -import { toIndexingConfigInput } from "../../../src/config" +import { + IndexingConfig, + normalizeFileExtensions, + parseFileExtensions, + toIndexingConfigInput, +} from "../../../src/config" import { CodeIndexConfigManager, type IndexingConfigInput } from "../../../src/indexing/config-manager" function createInput(input: Partial = {}): IndexingConfigInput { @@ -72,6 +77,25 @@ describe("CodeIndexConfigManager", () => { expect(cfg.getConfig().vectorStoreProvider).toBe("qdrant") }) + test("normalizes configured file extensions", () => { + expect(normalizeFileExtensions([" PHP ", ".JS", "js", "css"])).toEqual([".css", ".js", ".php"]) + expect(parseFileExtensions(" PHP, .JS, js, css ")).toEqual([".css", ".js", ".php"]) + expect(parseFileExtensions(" ")).toBeUndefined() + expect(toIndexingConfigInput({ fileExtensions: ["PHP", ".JS"] }).fileExtensions).toEqual([".js", ".php"]) + expect(normalizeFileExtensions(["", " "])).toBeUndefined() + expect( + normalizeFileExtensions(Array.from({ length: 10_000 }, (_, index) => (index % 2 ? " PHP " : ".JS"))), + ).toEqual([".js", ".php"]) + }) + + test("validates file extension tokens", () => { + expect(IndexingConfig.safeParse({ fileExtensions: ["php", " .JS "] }).success).toBe(true) + expect(IndexingConfig.safeParse({ fileExtensions: [] }).success).toBe(false) + expect(IndexingConfig.safeParse({ fileExtensions: ["*.js"] }).success).toBe(false) + expect(IndexingConfig.safeParse({ fileExtensions: ["src/php"] }).success).toBe(false) + expect(IndexingConfig.safeParse({ fileExtensions: [".d.ts"] }).success).toBe(false) + }) + test("configures Kilo with hosted auth options and explicit model metadata", () => { const cfg = new CodeIndexConfigManager( createInput({ @@ -206,5 +230,13 @@ describe("CodeIndexConfigManager", () => { expect(result.requiresRestart).toBe(true) }) + + test("restarts only when the normalized file extension allowlist changes", () => { + const cfg = new CodeIndexConfigManager(createInput({ fileExtensions: ["php", ".JS"] })) + + expect(cfg.getConfig().fileExtensions).toEqual([".js", ".php"]) + expect(cfg.loadConfiguration(createInput({ fileExtensions: [".js", ".PHP", "php"] })).requiresRestart).toBe(false) + expect(cfg.loadConfiguration(createInput({ fileExtensions: [".css"] })).requiresRestart).toBe(true) + }) }) }) diff --git a/packages/kilo-indexing/test/kilocode/indexing/processors/file-watcher.test.ts b/packages/kilo-indexing/test/kilocode/indexing/processors/file-watcher.test.ts index 80c8ace10f..628cddc233 100644 --- a/packages/kilo-indexing/test/kilocode/indexing/processors/file-watcher.test.ts +++ b/packages/kilo-indexing/test/kilocode/indexing/processors/file-watcher.test.ts @@ -14,6 +14,7 @@ import type { VectorStoreSearchResult, } from "../../../../src/indexing/interfaces" import { FileWatcher } from "../../../../src/indexing/processors/file-watcher" +import { CodeParser } from "../../../../src/indexing/processors/parser" import { loadIgnore } from "../../../../src/indexing/shared/load-ignore" import { WorktreeOverlay } from "../../../../src/indexing/worktree-overlay" @@ -318,6 +319,46 @@ describe("FileWatcher", () => { expect(result.reason).toBe("File is ignored by .gitignore or .kilocodeignore") }) + test("processFile uses the configured extension allowlist", async () => { + const root = await mkdtemp(path.join(tmpdir(), "file-watcher-test-")) + const cacheDir = path.join(root, ".cache") + const custom = path.join(root, "source.custom") + const excluded = path.join(root, "source.ts") + const content = "custom source content ".repeat(20) + await mkdir(cacheDir, { recursive: true }) + await writeFile(custom, content) + await writeFile(excluded, content) + + const cache = new CacheManager(cacheDir, root) + await cache.initialize() + const watcher = new FileWatcher( + root, + cache, + createEmbedder(), + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + [".custom"], + new CodeParser([".custom"]), + ) + + const first = await watcher.processFile(custom) + expect(first.status).toBe("processed_for_batching") + if (first.status === "processed_for_batching" && first.newHash) cache.updateHash(custom, first.newHash) + expect(await watcher.processFile(excluded)).toMatchObject({ + status: "skipped", + reason: "File extension is not configured for indexing", + }) + await writeFile(custom, new Uint8Array([0, 1, 2, 3])) + expect(await watcher.processFile(custom)).toMatchObject({ status: "skipped", reason: "File is binary" }) + expect(cache.getHash(custom)).toBeUndefined() + await writeFile(custom, content) + expect((await watcher.processFile(custom)).status).toBe("processed_for_batching") + }) + test("processFile skips files matched by nested .gitignore during incremental updates", async () => { const root = await mkdtemp(path.join(tmpdir(), "file-watcher-test-")) try { diff --git a/packages/kilo-indexing/test/kilocode/indexing/processors/parser.test.ts b/packages/kilo-indexing/test/kilocode/indexing/processors/parser.test.ts index 2ba9b716be..cd07c54b48 100644 --- a/packages/kilo-indexing/test/kilocode/indexing/processors/parser.test.ts +++ b/packages/kilo-indexing/test/kilocode/indexing/processors/parser.test.ts @@ -71,6 +71,21 @@ describe("CodeParser", () => { expect(result).toEqual([]) }) + test("uses fallback chunking for configured custom extensions", async () => { + const custom = new CodeParser([".custom"]) + const result = await custom.parseFile("test.custom", { content: "custom source line ".repeat(20) }) + + expect(result.length).toBeGreaterThan(0) + expect(result[0]?.type).toBe("fallback_chunk") + }) + + test("excludes built-in extensions outside a configured allowlist", async () => { + const custom = new CodeParser([".php"]) + const result = await custom.parseFile("test.js", { content: "const value = 1;".repeat(20) }) + + expect(result).toEqual([]) + }) + test("should use provided content instead of reading file when options.content is provided", async () => { const content = `/* This is a long test content string that exceeds 100 characters to properly test the parser's behavior with large inputs. It includes multiple lines and various JavaScript constructs to simulate real-world code. diff --git a/packages/kilo-indexing/test/kilocode/indexing/processors/scanner.test.ts b/packages/kilo-indexing/test/kilocode/indexing/processors/scanner.test.ts index 6ea5bfff02..ea2e75a275 100644 --- a/packages/kilo-indexing/test/kilocode/indexing/processors/scanner.test.ts +++ b/packages/kilo-indexing/test/kilocode/indexing/processors/scanner.test.ts @@ -309,6 +309,63 @@ describe("DirectoryScanner", () => { expect(cache.getHash(open)).toBeDefined() }) + test("uses a configured extension allowlist and removes excluded cached files", async () => { + const root = await mkdtemp(join(tmpdir(), "scanner-test-")) + const cacheDir = await mkdtemp(join(tmpdir(), "scanner-cache-")) + const custom = join(root, "main.custom") + const excluded = join(root, "main.ts") + await Bun.write(custom, "custom source content\n") + await Bun.write(excluded, "export const excluded = 1\n") + + const cache = new CacheManager(cacheDir, root) + await cache.initialize() + cache.updateHash(excluded, "old-hash") + const scan = new DirectoryScanner( + new Emb(), + new Store(), + new Parser(), + cache, + ignore(), + 1, + 1, + undefined, + undefined, + [".custom"], + ) + + const result = await scan.scanDirectory(root) + + expect(result.stats.processed).toBe(1) + expect(cache.getHash(custom)).toBeDefined() + expect(cache.getHash(excluded)).toBeUndefined() + }) + + test("skips binary files admitted by a custom extension", async () => { + const root = await mkdtemp(join(tmpdir(), "scanner-test-")) + const cacheDir = await mkdtemp(join(tmpdir(), "scanner-cache-")) + const file = join(root, "data.custom") + await Bun.write(file, new Uint8Array([0, 1, 2, 3])) + const cache = new CacheManager(cacheDir, root) + await cache.initialize() + const scan = new DirectoryScanner( + new Emb(), + new Store(), + new Parser(), + cache, + ignore(), + 1, + 1, + undefined, + undefined, + [".custom"], + ) + + const result = await scan.scanDirectory(root) + + expect(result.stats).toEqual({ processed: 0, skipped: 1 }) + expect(cache.getHash(file)).toBeUndefined() + }) + test("skips files matched by nested .kilocodeignore during full scans", async () => { const root = await mkdtemp(join(tmpdir(), "scanner-test-")) const cacheDir = await mkdtemp(join(tmpdir(), "scanner-cache-")) diff --git a/packages/kilo-indexing/test/kilocode/indexing/search-service.test.ts b/packages/kilo-indexing/test/kilocode/indexing/search-service.test.ts index 9447aa688f..c5a39795ba 100644 --- a/packages/kilo-indexing/test/kilocode/indexing/search-service.test.ts +++ b/packages/kilo-indexing/test/kilocode/indexing/search-service.test.ts @@ -36,16 +36,34 @@ const embedder = (calls: string[][]): IEmbedder => ({ }, }) -const config = () => +const config = (fileExtensions?: string[]) => new CodeIndexConfigManager({ enabled: true, embedderProvider: "openai", openAiKey: "test", vectorStoreProvider: "lancedb", searchMaxResults: 2, + fileExtensions, }) describe("CodeIndexSearchService worktree search", () => { + test("filters stale results using one vector query", async () => { + const limits: number[] = [] + const state = new CodeIndexStateManager() + state.setSystemState("Indexed") + const service = new CodeIndexSearchService( + config([".php"]), + state, + embedder([]), + store([result("src/old.ts", 0.99), result("src/first.php", 0.9), result("src/second.php", 0.8)], limits), + ) + + const results = await service.searchIndex("query") + + expect(limits).toEqual([2]) + expect(results.map((item) => item.payload?.filePath)).toEqual(["src/first.php"]) + }) + test("embeds once, hides baseline paths, and merges the current delta", async () => { const root = await mkdtemp(path.join(tmpdir(), "search-worktree-")) const main = await mkdtemp(path.join(tmpdir(), "search-main-")) diff --git a/packages/kilo-vscode/tests/unit/indexing-tab-state.test.ts b/packages/kilo-vscode/tests/unit/indexing-tab-state.test.ts index 35f0193cd3..8bad228590 100644 --- a/packages/kilo-vscode/tests/unit/indexing-tab-state.test.ts +++ b/packages/kilo-vscode/tests/unit/indexing-tab-state.test.ts @@ -126,4 +126,13 @@ describe("indexing tab scope state", () => { qdrant: { url: "http://project", apiKey: "global-secret" }, }) }) + + it("replaces inherited file extensions with a project allowlist", () => { + const global = { fileExtensions: [".ts", ".tsx"] } + + expect(indexingConfig("project", global, {}).fileExtensions).toEqual([".ts", ".tsx"]) + expect(indexingConfig("project", global, { fileExtensions: [".php"] }).fileExtensions).toEqual([".php"]) + expect(indexingInheritance("project", global, {}, [["fileExtensions"]])).toBe("inherited") + expect(indexingInheritance("project", global, { fileExtensions: [".php"] }, [["fileExtensions"]])).toBe("none") + }) }) diff --git a/packages/kilo-vscode/webview-ui/src/components/settings/IndexingTab.tsx b/packages/kilo-vscode/webview-ui/src/components/settings/IndexingTab.tsx index ed6aa59799..07864b04ff 100644 --- a/packages/kilo-vscode/webview-ui/src/components/settings/IndexingTab.tsx +++ b/packages/kilo-vscode/webview-ui/src/components/settings/IndexingTab.tsx @@ -1,7 +1,7 @@ import { Component, For, Show, createMemo, createSignal } from "solid-js" import { Button } from "@kilocode/kilo-ui/button" import { Card } from "@kilocode/kilo-ui/card" -import { DEFAULT_VECTOR_STORE } from "@kilocode/kilo-indexing/config" +import { DEFAULT_VECTOR_STORE, isFileExtension, parseFileExtensions } from "@kilocode/kilo-indexing/config" import { formatKiloEmbeddingModelLabel, getKiloEmbeddingModel } from "@kilocode/kilo-indexing/embedding-models" import { Select } from "@kilocode/kilo-ui/select" import { Switch } from "@kilocode/kilo-ui/switch" @@ -102,6 +102,8 @@ const IndexingTab: Component = () => { const [providerDrafts, setProviderDrafts] = createSignal>({}) const [storeDrafts, setStoreDrafts] = createSignal>({}) const [tuningDrafts, setTuningDrafts] = createSignal>({}) + const [extensionDrafts, setExtensionDrafts] = createSignal>({}) + const [extensionErrors, setExtensionErrors] = createSignal>({}) const [scope, setScope] = createSignal("global") const globalCfg = createMemo(() => globalConfig().indexing ?? {}) @@ -247,6 +249,30 @@ const IndexingTab: Component = () => { return value === undefined ? "" : String(value) } + const extensionValue = () => { + const draft = extensionDrafts()[scope()] + if (draft !== undefined) return draft + return cfg().fileExtensions?.join(", ") ?? "" + } + + const saveExtensions = (value: string) => { + const values = value + .split(",") + .map((item) => item.trim()) + .filter(Boolean) + const invalid = values.find((item) => !isFileExtension(item)) + if (invalid) { + setExtensionErrors((prev) => ({ + ...prev, + [scope()]: language.t("settings.indexing.fileExtensions.invalid", { extension: invalid }), + })) + return + } + updateIndexing({ fileExtensions: parseFileExtensions(value) }) + setExtensionDrafts((prev) => Object.fromEntries(Object.entries(prev).filter(([entry]) => entry !== scope()))) + setExtensionErrors((prev) => Object.fromEntries(Object.entries(prev).filter(([entry]) => entry !== scope()))) + } + const content = (_scope: IndexingScope) => (
@@ -517,6 +543,26 @@ const IndexingTab: Component = () => { + tag(scope(), [["fileExtensions"]])} + > + { + const target = e.currentTarget as HTMLInputElement + setExtensionDrafts((prev) => ({ ...prev, [scope()]: target.value })) + }} + onBlur={(e: FocusEvent) => { + const target = e.currentTarget as HTMLInputElement + saveExtensions(target.value) + }} + /> + {(item, index) => ( )) break + case "fileExtensions": { + const result = await DialogPrompt.show(dialog, "File Extensions", { + value: indexing.fileExtensions?.join(", ") ?? "", + placeholder: ".php, .js, .css (empty uses built-in defaults)", + }) + if (result !== null) { + const values = result + .split(",") + .map((item) => item.trim()) + .filter(Boolean) + const invalid = values.find((item) => !isFileExtension(item)) + if (invalid) { + toast.show({ message: `Invalid file extension: "${invalid}"`, variant: "error" }) + } else { + await saveScopedIndexing( + sdk, + sync, + scope(), + raw, + { ...raw, fileExtensions: parseFileExtensions(result) }, + toast, + ) + } + } + dialog.replace(() => ) + break + } case "tuning": dialog.replace(() => ( diff --git a/packages/opencode/src/kilocode/config/overlay.ts b/packages/opencode/src/kilocode/config/overlay.ts index 5b02ee9c45..2d576c3bce 100644 --- a/packages/opencode/src/kilocode/config/overlay.ts +++ b/packages/opencode/src/kilocode/config/overlay.ts @@ -115,6 +115,7 @@ export namespace KilocodeConfigOverlay { ["indexing", "searchMaxResults"], ["indexing", "embeddingBatchSize"], ["indexing", "scannerMaxBatchRetries"], + ["indexing", "fileExtensions"], ] as const const collectionPaths = ["provider", "mcp", "permission", "agent", "formatter", "lsp"] as const diff --git a/packages/opencode/test/kilocode/indexing-dialog-state.test.ts b/packages/opencode/test/kilocode/indexing-dialog-state.test.ts index 9aa6d3e8a3..290bf542e8 100644 --- a/packages/opencode/test/kilocode/indexing-dialog-state.test.ts +++ b/packages/opencode/test/kilocode/indexing-dialog-state.test.ts @@ -107,6 +107,15 @@ describe("indexing dialog state", () => { expect(indexingInheritance("project", global, project, [["searchMinScore"]])).toBe("inherited") }) + test("replaces inherited file extensions with a project allowlist", () => { + const global: IndexingConfig = { fileExtensions: [".ts", ".tsx"] } + const project: IndexingConfig = { fileExtensions: [".php"] } + + expect(mergeIndexingConfig(global, project).fileExtensions).toEqual([".php"]) + expect(indexingInheritance("project", global, {}, [["fileExtensions"]])).toBe("inherited") + expect(indexingInheritance("project", global, project, [["fileExtensions"]])).toBe("none") + }) + test("isolates global auth config from project indexing values", () => { const project: IndexingConfig = { kilo: { apiKey: "project-key", baseUrl: "https://project.test" } } const inherited: IndexingConfig = { enabled: true } diff --git a/packages/opencode/test/kilocode/server/config-overlay.test.ts b/packages/opencode/test/kilocode/server/config-overlay.test.ts index 89fa6da213..d3351b0d50 100644 --- a/packages/opencode/test/kilocode/server/config-overlay.test.ts +++ b/packages/opencode/test/kilocode/server/config-overlay.test.ts @@ -208,6 +208,7 @@ describe("config overlay routes", () => { indexing: { enabled: true, provider: "ollama", + fileExtensions: [".php", ".js"], ollama: { baseUrl: "http://localhost:11434" }, }, }) @@ -216,6 +217,11 @@ describe("config overlay routes", () => { expect(body.fields["indexing.enabled"]).toMatchObject({ source: "global", inherited: true, value: true }) expect(body.fields["indexing.provider"]).toMatchObject({ source: "global", inherited: true, value: "ollama" }) + expect(body.fields["indexing.fileExtensions"]).toMatchObject({ + source: "global", + inherited: true, + value: [".php", ".js"], + }) expect(body.fields["indexing.ollama.baseUrl"]).toMatchObject({ source: "global", inherited: true, diff --git a/packages/sdk/js/src/v2/gen/types.gen.ts b/packages/sdk/js/src/v2/gen/types.gen.ts index 41f9fe95f6..9bdf89a86c 100644 --- a/packages/sdk/js/src/v2/gen/types.gen.ts +++ b/packages/sdk/js/src/v2/gen/types.gen.ts @@ -1267,6 +1267,7 @@ export type IndexingConfig = { searchMaxResults?: number embeddingBatchSize?: number scannerMaxBatchRetries?: number + fileExtensions?: Array } export type PermissionActionConfig = "ask" | "allow" | "deny" diff --git a/packages/sdk/openapi.json b/packages/sdk/openapi.json index 67a124897e..1c1f594374 100644 --- a/packages/sdk/openapi.json +++ b/packages/sdk/openapi.json @@ -25844,6 +25844,14 @@ "scannerMaxBatchRetries": { "type": "integer", "exclusiveMinimum": 0 + }, + "fileExtensions": { + "type": "array", + "items": { + "type": "string", + "pattern": "^\\s*\\.?[A-Za-z0-9][A-Za-z0-9_+-]*\\s*$" + }, + "minItems": 1 } }, "additionalProperties": false