Merge branch 'ocasta181/clineignore' of github.com:cline/cline into ocasta181/clineignore

src/services/glob/list-files.ts

@@ -1,7 +1,18 @@
-import { globby, Options } from "globby"
 import os from "os"
 import * as path from "path"
 import { arePathsEqual } from "../../utils/path"
+import { ignoreParser } from "./parse-ignore"
+
+// Define Options type inline to avoid ESM import issues
+interface Options {
+	cwd?: string
+	dot?: boolean
+	absolute?: boolean
+	markDirectories?: boolean
+	gitignore?: boolean
+	ignore?: string[]
+	onlyFiles?: boolean
+}
 
 export async function listFiles(dirPath: string, recursive: boolean, limit: number): Promise<[string[], boolean]> {
 	const absolutePath = path.resolve(dirPath)
@@ -36,18 +47,60 @@ export async function listFiles(dirPath: string, recursive: boolean, limit: numb
 		".*", // '!**/.*' excludes hidden directories, while '!**/.*/**' excludes only their contents. This way we are at least aware of the existence of hidden directories.
 	].map((dir) => `**/${dir}/**`)
 
-	const options = {
+	// Load .clineignore patterns if they exist
+	await ignoreParser.loadIgnoreFile(dirPath)
+	const clineignorePatterns = ignoreParser.getIgnorePatterns()
+	ignoreParser.clear() // Clear patterns after use to prevent interference with future calls
+
+	const options: Options = {
 		cwd: dirPath,
 		dot: true, // do not ignore hidden files/directories
 		absolute: true,
 		markDirectories: true, // Append a / on any directories matched (/ is used on windows as well, so dont use path.sep)
 		gitignore: recursive, // globby ignores any files that are gitignored
-		ignore: recursive ? dirsToIgnore : undefined, // just in case there is no gitignore, we ignore sensible defaults
 		onlyFiles: false, // true by default, false means it will list directories on their own too
 	}
+
 	// * globs all files in one dir, ** globs files in nested directories
-	const files = recursive ? await globbyLevelByLevel(limit, options) : (await globby("*", options)).slice(0, limit)
-	return [files, files.length >= limit]
+	const { globby } = await import("globby")
+
+	if (!recursive) {
+		return [(await globby("*", options)).slice(0, limit), false]
+	}
+
+	// For recursive listing, handle ignore patterns
+	const baseOptions = { ...options, ignore: undefined }
+
+	// Get all files first
+	const allFiles = await globby("**", baseOptions)
+
+	// Split patterns into ignore and negated patterns
+	const ignorePatterns = [...dirsToIgnore]
+	const negatedPatterns: string[] = []
+
+	if (clineignorePatterns.length > 0) {
+		clineignorePatterns.forEach((pattern) => {
+			if (pattern.startsWith("!")) {
+				negatedPatterns.push(pattern.slice(1)) // Remove the ! prefix
+			} else {
+				ignorePatterns.push(pattern)
+			}
+		})
+	}
+
+	// Get files that match ignore patterns
+	const ignoreOptions = { ...baseOptions }
+	const ignoredFiles = new Set(await globby(ignorePatterns, ignoreOptions))
+
+	// Get files that match negated patterns (these override ignores)
+	const includedFiles = negatedPatterns.length > 0 ? new Set(await globby(negatedPatterns, baseOptions)) : new Set()
+
+	// Filter files:
+	// - Keep if it doesn't match any ignore pattern
+	// - Or if it matches a negated pattern
+	const files = allFiles.filter((file) => !ignoredFiles.has(file) || includedFiles.has(file))
+
+	return [files.slice(0, limit), files.length >= limit]
 }
 
 /*
@@ -62,7 +115,7 @@ Breadth-first traversal of directory structure level by level up to a limit:
    - Potential for loops if symbolic links reference back to parent (we could use followSymlinks: false but that may not be ideal for some projects and it's pointless if they're not using symlinks wrong)
    - Timeout mechanism prevents infinite loops
 */
-async function globbyLevelByLevel(limit: number, options?: Options) {
+async function globbyLevelByLevel(limit: number, options: Options, globby: any) {
 	let results: Set<string> = new Set()
 	let queue: string[] = ["*"]
 

src/services/glob/parse-ignore.ts

@@ -0,0 +1,106 @@
+import { promises as fs } from "fs"
+import * as path from "path"
+
+interface IgnorePattern {
+	pattern: string
+	isNegated: boolean
+}
+
+export class IgnoreParser {
+	private patterns: IgnorePattern[] = []
+
+	/**
+	 * Load and parse a .clineignore file
+	 * @param dirPath Directory path to look for .clineignore file
+	 */
+	async loadIgnoreFile(dirPath: string): Promise<void> {
+		const ignorePath = path.join(dirPath, ".clineignore")
+		try {
+			const content = await fs.readFile(ignorePath, "utf8")
+			this.parsePatterns(content)
+		} catch (error) {
+			if ((error as NodeJS.ErrnoException).code !== "ENOENT") {
+				throw error
+			}
+			// File doesn't exist - that's okay, just use default patterns
+		}
+	}
+
+	/**
+	 * Parse ignore patterns from file content
+	 * @param content Raw content of .clineignore file
+	 */
+	private parsePatterns(content: string): void {
+		const lines = content.split("\n")
+
+		for (const line of lines) {
+			const trimmed = line.trim()
+
+			// Skip empty lines and comments
+			if (!trimmed || trimmed.startsWith("#")) {
+				continue
+			}
+
+			this.patterns.push({
+				pattern: trimmed,
+				isNegated: trimmed.startsWith("!"),
+			})
+		}
+	}
+
+	/**
+	 * Convert .clineignore pattern to globby-compatible pattern
+	 */
+	private normalizePattern(pattern: string): string {
+		// Remove leading and trailing slashes
+		let normalized = pattern.replace(/^\/+|\/+$/g, "")
+
+		// Handle patterns that should match files in any directory
+		if (!normalized.startsWith("**/") && !normalized.startsWith("/")) {
+			normalized = `**/${normalized}`
+		}
+
+		// Detect if this is a directory pattern
+		const isDirectoryPattern =
+			normalized.endsWith("/") || // Explicit directory pattern
+			(!normalized.includes(".") && !normalized.includes("*")) || // No extension or wildcards
+			/^[\w-]+$/.test(normalized.split("/").pop() || "") // Simple name without special chars
+
+		// Handle directory patterns
+		if (isDirectoryPattern) {
+			normalized = normalized.replace(/\/?$/, "/**")
+		}
+
+		// Handle file patterns
+		const isFilePattern = normalized.includes(".") || normalized.includes("*")
+		if (isFilePattern && !isDirectoryPattern) {
+			// Don't add /** to file patterns
+			normalized = normalized.replace(/\/\*\*$/, "")
+		}
+
+		return normalized
+	}
+
+	/**
+	 * Get all ignore patterns including negations
+	 */
+	getIgnorePatterns(): string[] {
+		return this.patterns.map(({ pattern, isNegated }) => {
+			if (isNegated) {
+				const cleanPattern = pattern.slice(1)
+				return `!${this.normalizePattern(cleanPattern)}`
+			}
+			return this.normalizePattern(pattern)
+		})
+	}
+
+	/**
+	 * Clear all loaded patterns
+	 */
+	clear(): void {
+		this.patterns = []
+	}
+}
+
+// Singleton instance for reuse
+export const ignoreParser = new IgnoreParser()

src/test/suite/parse-ignore.test.ts

@@ -0,0 +1,113 @@
+import { promises as fs } from "fs"
+import * as path from "path"
+import { ignoreParser } from "../../services/glob/parse-ignore"
+import { listFiles } from "../../services/glob/list-files"
+import os from "os"
+import { describe, it, beforeEach, afterEach } from "mocha"
+import "should"
+
+describe("IgnoreParser", function () {
+	// Increase timeout for async operations
+	this.timeout(10000)
+	const tmpDir = path.join(os.tmpdir(), "cline-test-" + Math.random().toString(36).slice(2))
+
+	beforeEach(async function () {
+		await fs.mkdir(tmpDir, { recursive: true })
+		ignoreParser.clear()
+	})
+
+	afterEach(async function () {
+		await fs.rm(tmpDir, { recursive: true, force: true })
+	})
+
+	it("should parse basic ignore patterns", async function () {
+		const ignoreContent = `
+			# Comment
+			node_modules/
+			*.log
+			/dist
+			!important.log
+		`
+		await fs.writeFile(path.join(tmpDir, ".clineignore"), ignoreContent)
+		await ignoreParser.loadIgnoreFile(tmpDir)
+
+		const patterns = ignoreParser.getIgnorePatterns()
+		patterns.should.containEql("**/node_modules/**")
+		patterns.should.containEql("**/*.log")
+		patterns.should.containEql("**/dist/**")
+		patterns.should.containEql("!**/important.log")
+	})
+
+	it("should handle empty or non-existent ignore file", async function () {
+		await ignoreParser.loadIgnoreFile(tmpDir)
+		ignoreParser.getIgnorePatterns().should.have.length(0)
+
+		await fs.writeFile(path.join(tmpDir, ".clineignore"), "")
+		await ignoreParser.loadIgnoreFile(tmpDir)
+		ignoreParser.getIgnorePatterns().should.have.length(0)
+	})
+
+	it("should integrate with listFiles", async function () {
+		this.timeout(15000) // Increase timeout for dynamic import
+
+		try {
+			// Create test files
+			await fs.writeFile(path.join(tmpDir, "test.txt"), "test")
+			await fs.writeFile(path.join(tmpDir, "test.log"), "log")
+			await fs.mkdir(path.join(tmpDir, "node_modules"), { recursive: true })
+			await fs.writeFile(path.join(tmpDir, "node_modules/package.json"), "{}")
+			await fs.writeFile(path.join(tmpDir, "important.log"), "important")
+
+			// Create .clineignore
+			const ignoreContent = `
+				*.log
+				node_modules/
+				!important.log
+			`
+			await fs.writeFile(path.join(tmpDir, ".clineignore"), ignoreContent)
+
+			const [files] = await listFiles(tmpDir, true, 100)
+			const relativePaths = files.map((filePath: string) => path.relative(tmpDir, filePath))
+
+			// Should include
+			relativePaths.should.containEql("test.txt")
+			relativePaths.should.containEql("important.log")
+
+			// Should exclude
+			relativePaths.should.not.containEql("test.log")
+			relativePaths.should.not.containEql("node_modules/package.json")
+		} catch (error) {
+			if (error instanceof Error && error.message.includes("ERR_REQUIRE_ESM")) {
+				this.skip() // Skip this test if we hit ESM issues
+			} else {
+				throw error
+			}
+		}
+	})
+
+	it("should handle complex patterns", async function () {
+		const ignoreContent = `
+			# Ignore all .txt files
+			**/*.txt
+			
+			# But not in docs
+			!docs/**/*.txt
+			
+			# Ignore build directories
+			**/build/
+			
+			# Ignore temp files but not temp directory
+			*.tmp
+			!temp/
+		`
+		await fs.writeFile(path.join(tmpDir, ".clineignore"), ignoreContent)
+		await ignoreParser.loadIgnoreFile(tmpDir)
+
+		const patterns = ignoreParser.getIgnorePatterns()
+		patterns.should.containEql("**/*.txt")
+		patterns.should.containEql("!**/docs/**/*.txt")
+		patterns.should.containEql("**/build/**")
+		patterns.should.containEql("**/*.tmp")
+		patterns.should.containEql("!**/temp/**")
+	})
+})

tsconfig.test.json

@@ -6,12 +6,14 @@
 	// compiled to make them compatible with VS Code's test runner.
 	"extends": "./tsconfig.json",
 	"compilerOptions": {
-		"module": "commonjs",
-		"moduleResolution": "node",
+		"module": "Node16",
+		"moduleResolution": "node16",
 		"types": ["node", "mocha", "should", "vscode", "chai"],
 		"typeRoots": ["./node_modules/@types", "./src/test/types"],
 		"outDir": "out",
-		"rootDir": "src"
+		"rootDir": "src",
+		"allowJs": true,
+		"esModuleInterop": true
 	},
 	"include": ["src/**/*.test.ts"],
 	"exclude": ["src/test/**/*.js"]