From 68e4155fedcc3e51a0a0cca7eced85afc1a2a28c Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Sat, 28 Feb 2026 23:37:07 -0500 Subject: [PATCH] feat(agent/filefinder): add plocate-lite file finder package (#22453) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds an in-memory trigram-indexed file finder package at `agent/filefinder`, designed to power a future `FindFiles` HTTP handler on the WorkspaceAgent. ## What it does Fast fuzzy file search with VS Code-quality matching across millions of files. Sub-millisecond search latency at 100K files. ## Architecture - **Index**: append-only docs slice with trigram + prefix posting lists - **Snapshot**: lock-free reader view via frozen slice headers + shallow-copied deleted set - **Search pipeline**: trigram intersection → fuzzy fallback (prefix bucket + subsequence) → brute-force scan (capped at 5K docs) - **Scoring**: subsequence match, basename prefix, boundary hits, contiguous runs, depth/length penalties - **Engine**: multi-root with fsnotify watcher (50ms batch coalescing), atomic snapshot publishing ## Benchmarks (10K files) | Query Type | Latency | |---|---| | exact_basename (`handler.go`) | ~43µs | | short_query (`ha`) | ~7µs | | fuzzy_basename (`hndlr`) | ~50µs | | path_structured (`internal/handler`) | ~29µs | | multi_token (`api handler`) | ~15µs | ## File inventory (11 files, 3273 lines) | File | Lines | Purpose | |---|---|---| | `text.go` | 264 | Normalization, trigram extraction, scoring | | `delta.go` | 128 | Index, Snapshot, CRUD operations | | `query.go` | 272 | Query planning, search strategies, top-K merge | | `engine.go` | 323 | Multi-root engine, watcher integration | | `watcher_fs.go` | 201 | fsnotify wrapper with batch coalescing | | `*_test.go` | 2085 | Unit tests, integration tests, benchmarks | --------- Co-authored-by: Coder --- agent/filefinder/bench_test.go | 316 ++++++++++++++++++++++++++ agent/filefinder/delta.go | 125 ++++++++++ agent/filefinder/delta_test.go | 120 ++++++++++ agent/filefinder/engine.go | 364 ++++++++++++++++++++++++++++++ agent/filefinder/engine_test.go | 233 +++++++++++++++++++ agent/filefinder/export_test.go | 85 +++++++ agent/filefinder/query.go | 299 ++++++++++++++++++++++++ agent/filefinder/query_test.go | 343 ++++++++++++++++++++++++++++ agent/filefinder/text.go | 288 ++++++++++++++++++++++++ agent/filefinder/text_test.go | 388 ++++++++++++++++++++++++++++++++ agent/filefinder/watcher_fs.go | 210 +++++++++++++++++ 11 files changed, 2771 insertions(+) create mode 100644 agent/filefinder/bench_test.go create mode 100644 agent/filefinder/delta.go create mode 100644 agent/filefinder/delta_test.go create mode 100644 agent/filefinder/engine.go create mode 100644 agent/filefinder/engine_test.go create mode 100644 agent/filefinder/export_test.go create mode 100644 agent/filefinder/query.go create mode 100644 agent/filefinder/query_test.go create mode 100644 agent/filefinder/text.go create mode 100644 agent/filefinder/text_test.go create mode 100644 agent/filefinder/watcher_fs.go diff --git a/agent/filefinder/bench_test.go b/agent/filefinder/bench_test.go new file mode 100644 index 0000000000..fd36be5612 --- /dev/null +++ b/agent/filefinder/bench_test.go @@ -0,0 +1,316 @@ +package filefinder_test + +import ( + "context" + "fmt" + "math/rand" + "os" + "path/filepath" + "runtime" + "sync" + "testing" + + "github.com/stretchr/testify/require" + + "cdr.dev/slog/v3" + "cdr.dev/slog/v3/sloggers/slogtest" + "github.com/coder/coder/v2/agent/filefinder" +) + +var ( + dirNames = []string{ + "cmd", "internal", "pkg", "api", "auth", "database", "server", "client", "middleware", + "handler", "config", "utils", "models", "service", "worker", "scheduler", "notification", + "provisioner", "template", "workspace", "agent", "proxy", "crypto", "telemetry", "billing", + } + fileExts = []string{ + ".go", ".ts", ".tsx", ".js", ".py", ".sql", ".yaml", ".json", ".md", ".proto", ".sh", + } + fileStems = []string{ + "main", "handler", "middleware", "service", "model", "query", "config", "utils", "helpers", + "types", "interface", "test", "mock", "factory", "builder", "adapter", "observer", "provider", + "resolver", "schema", "migration", "fixture", "snapshot", "checkpoint", + } +) + +// generateFileTree creates n files under root in a realistic nested directory structure. +func generateFileTree(t testing.TB, root string, n int, seed int64) { + t.Helper() + rng := rand.New(rand.NewSource(seed)) //nolint:gosec // deterministic benchmarks + + numDirs := n / 5 + if numDirs < 10 { + numDirs = 10 + } + dirs := make([]string, 0, numDirs) + for i := 0; i < numDirs; i++ { + depth := rng.Intn(6) + 1 + parts := make([]string, depth) + for d := 0; d < depth; d++ { + parts[d] = dirNames[rng.Intn(len(dirNames))] + } + dirs = append(dirs, filepath.Join(parts...)) + } + + created := make(map[string]struct{}) + for _, d := range dirs { + full := filepath.Join(root, d) + if _, ok := created[full]; ok { + continue + } + require.NoError(t, os.MkdirAll(full, 0o755)) + created[full] = struct{}{} + } + + for i := 0; i < n; i++ { + dir := dirs[rng.Intn(len(dirs))] + stem := fileStems[rng.Intn(len(fileStems))] + ext := fileExts[rng.Intn(len(fileExts))] + name := fmt.Sprintf("%s_%d%s", stem, i, ext) + full := filepath.Join(root, dir, name) + f, err := os.Create(full) + require.NoError(t, err) + _ = f.Close() + } +} + +// buildIndex walks root and returns a populated Index, the same +// way Engine.AddRoot does but without starting a watcher. +func buildIndex(t testing.TB, root string) *filefinder.Index { + t.Helper() + absRoot, err := filepath.Abs(root) + require.NoError(t, err) + idx, err := filefinder.BuildTestIndex(absRoot) + require.NoError(t, err) + return idx +} + +func BenchmarkBuildIndex(b *testing.B) { + scales := []struct { + name string + n int + }{ + {"1K", 1_000}, + {"10K", 10_000}, + {"100K", 100_000}, + } + + for _, sc := range scales { + b.Run(sc.name, func(b *testing.B) { + if sc.n >= 100_000 && testing.Short() { + b.Skip("skipping large-scale benchmark") + } + dir := b.TempDir() + generateFileTree(b, dir, sc.n, 42) + + b.ResetTimer() + for i := 0; i < b.N; i++ { + idx := buildIndex(b, dir) + if idx.Len() == 0 { + b.Fatal("expected non-empty index") + } + } + b.StopTimer() + + idx := buildIndex(b, dir) + b.ReportMetric(float64(idx.Len())/b.Elapsed().Seconds(), "files/sec") + }) + } +} + +func BenchmarkSearch_ByScale(b *testing.B) { + queries := []struct { + name string + query string + }{ + {"exact_basename", "handler.go"}, + {"short_query", "ha"}, + {"fuzzy_basename", "hndlr"}, + {"path_structured", "internal/handler"}, + {"multi_token", "api handler"}, + } + scales := []struct { + name string + n int + }{ + {"1K", 1_000}, + {"10K", 10_000}, + {"100K", 100_000}, + } + + for _, sc := range scales { + b.Run(sc.name, func(b *testing.B) { + if sc.n >= 100_000 && testing.Short() { + b.Skip("skipping large-scale benchmark") + } + dir := b.TempDir() + generateFileTree(b, dir, sc.n, 42) + idx := buildIndex(b, dir) + snap := idx.Snapshot() + opts := filefinder.DefaultSearchOptions() + + for _, q := range queries { + b.Run(q.name, func(b *testing.B) { + p := filefinder.NewQueryPlanForTest(q.query) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _ = filefinder.SearchSnapshotForTest(p, snap, opts.MaxCandidates) + } + }) + } + }) + } +} + +func BenchmarkSearch_ConcurrentReads(b *testing.B) { + dir := b.TempDir() + generateFileTree(b, dir, 10_000, 42) + + logger := slogtest.Make(b, &slogtest.Options{IgnoreErrors: true}).Leveled(slog.LevelError) + ctx := context.Background() + eng := filefinder.NewEngine(logger) + require.NoError(b, eng.AddRoot(ctx, dir)) + b.Cleanup(func() { _ = eng.Close() }) + + opts := filefinder.DefaultSearchOptions() + goroutines := []int{1, 4, 16, 64} + + for _, g := range goroutines { + b.Run(fmt.Sprintf("goroutines_%d", g), func(b *testing.B) { + b.SetParallelism(g) + b.ResetTimer() + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + results, err := eng.Search(ctx, "handler", opts) + if err != nil { + b.Fatal(err) + } + _ = results + } + }) + }) + } +} + +func BenchmarkDeltaUpdate(b *testing.B) { + dir := b.TempDir() + generateFileTree(b, dir, 10_000, 42) + + addCounts := []int{1, 10, 100} + + for _, count := range addCounts { + b.Run(fmt.Sprintf("add_%d_files", count), func(b *testing.B) { + paths := make([]string, count) + for i := range paths { + paths[i] = fmt.Sprintf("injected/dir_%d/newfile_%d.go", i%10, i) + } + b.ResetTimer() + for i := 0; i < b.N; i++ { + b.StopTimer() + idx := buildIndex(b, dir) + b.StartTimer() + for _, p := range paths { + idx.Add(p, 0) + } + } + b.ReportMetric(float64(count), "files_added/op") + }) + } + + b.Run("search_after_100_additions", func(b *testing.B) { + idx := buildIndex(b, dir) + for i := 0; i < 100; i++ { + idx.Add(fmt.Sprintf("injected/extra/file_%d.go", i), 0) + } + snap := idx.Snapshot() + plan := filefinder.NewQueryPlanForTest("handler") + opts := filefinder.DefaultSearchOptions() + + b.ResetTimer() + for i := 0; i < b.N; i++ { + _ = filefinder.SearchSnapshotForTest(plan, snap, opts.MaxCandidates) + } + }) +} + +func BenchmarkMemoryProfile(b *testing.B) { + scales := []struct { + name string + n int + }{ + {"10K", 10_000}, + {"100K", 100_000}, + } + + for _, sc := range scales { + b.Run(sc.name, func(b *testing.B) { + if sc.n >= 100_000 && testing.Short() { + b.Skip("skipping large-scale memory profile") + } + dir := b.TempDir() + generateFileTree(b, dir, sc.n, 42) + + b.ResetTimer() + for i := 0; i < b.N; i++ { + idx := buildIndex(b, dir) + _ = idx.Snapshot() + } + b.StopTimer() + + // Report memory stats on the last iteration. + runtime.GC() + var before runtime.MemStats + runtime.ReadMemStats(&before) + idx := buildIndex(b, dir) + var after runtime.MemStats + runtime.ReadMemStats(&after) + + allocDelta := after.TotalAlloc - before.TotalAlloc + b.ReportMetric(float64(allocDelta)/float64(idx.Len()), "bytes/file") + + runtime.GC() + runtime.ReadMemStats(&before) + snap := idx.Snapshot() + _ = snap + runtime.GC() + runtime.ReadMemStats(&after) + + snapAlloc := after.TotalAlloc - before.TotalAlloc + b.ReportMetric(float64(snapAlloc)/float64(idx.Len()), "snap-bytes/file") + }) + } +} + +func BenchmarkSearch_ConcurrentReads_Throughput(b *testing.B) { + dir := b.TempDir() + generateFileTree(b, dir, 10_000, 42) + idx := buildIndex(b, dir) + snap := idx.Snapshot() + + goroutines := []int{1, 4, 16, 64} + plan := filefinder.NewQueryPlanForTest("handler.go") + maxCands := filefinder.DefaultSearchOptions().MaxCandidates + + for _, g := range goroutines { + b.Run(fmt.Sprintf("goroutines_%d", g), func(b *testing.B) { + b.ResetTimer() + var wg sync.WaitGroup + perGoroutine := b.N / g + if perGoroutine < 1 { + perGoroutine = 1 + } + for gi := 0; gi < g; gi++ { + wg.Add(1) + go func() { + defer wg.Done() + for j := 0; j < perGoroutine; j++ { + _ = filefinder.SearchSnapshotForTest(plan, snap, maxCands) + } + }() + } + wg.Wait() + totalOps := float64(g * perGoroutine) + b.ReportMetric(totalOps/b.Elapsed().Seconds(), "searches/sec") + }) + } +} diff --git a/agent/filefinder/delta.go b/agent/filefinder/delta.go new file mode 100644 index 0000000000..f0090f61bc --- /dev/null +++ b/agent/filefinder/delta.go @@ -0,0 +1,125 @@ +package filefinder + +import "strings" + +// FileFlag represents the type of filesystem entry. +type FileFlag uint16 + +const ( + FlagFile FileFlag = 0 + FlagDir FileFlag = 1 + FlagSymlink FileFlag = 2 +) + +type doc struct { + path string + baseOff int + baseLen int + depth int + flags uint16 +} + +// Index is an append-only in-memory file index with snapshot support. +type Index struct { + docs []doc + byGram map[uint32][]uint32 + byPrefix1 [256][]uint32 + byPrefix2 map[uint16][]uint32 + byPath map[string]uint32 + deleted map[uint32]bool +} + +// Snapshot is a frozen, read-only view of the index at a point in time. +type Snapshot struct { + docs []doc + deleted map[uint32]bool + byGram map[uint32][]uint32 + byPrefix1 [256][]uint32 + byPrefix2 map[uint16][]uint32 +} + +// NewIndex creates an empty Index. +func NewIndex() *Index { + return &Index{ + byGram: make(map[uint32][]uint32), + byPrefix2: make(map[uint16][]uint32), + byPath: make(map[string]uint32), + deleted: make(map[uint32]bool), + } +} + +// Add inserts a path into the index, tombstoning any previous entry. +func (idx *Index) Add(path string, flags uint16) uint32 { + norm := string(normalizePathBytes([]byte(path))) + if oldID, ok := idx.byPath[norm]; ok { + idx.deleted[oldID] = true + } + id := uint32(len(idx.docs)) //nolint:gosec // Index will never exceed 2^32 docs. + baseOff, baseLen := extractBasename([]byte(norm)) + idx.docs = append(idx.docs, doc{ + path: norm, baseOff: baseOff, baseLen: baseLen, + depth: strings.Count(norm, "/"), flags: flags, + }) + idx.byPath[norm] = id + for _, g := range extractTrigrams([]byte(norm)) { + idx.byGram[g] = append(idx.byGram[g], id) + } + if baseLen > 0 { + basename := []byte(norm[baseOff : baseOff+baseLen]) + p1 := prefix1(basename) + idx.byPrefix1[p1] = append(idx.byPrefix1[p1], id) + p2 := prefix2(basename) + idx.byPrefix2[p2] = append(idx.byPrefix2[p2], id) + } + return id +} + +// Remove marks the entry for path as deleted. +func (idx *Index) Remove(path string) bool { + norm := string(normalizePathBytes([]byte(path))) + id, ok := idx.byPath[norm] + if !ok { + return false + } + idx.deleted[id] = true + delete(idx.byPath, norm) + return true +} + +// Has reports whether path exists (not deleted) in the index. +func (idx *Index) Has(path string) bool { + _, ok := idx.byPath[string(normalizePathBytes([]byte(path)))] + return ok +} + +// Len returns the number of live (non-deleted) documents. +func (idx *Index) Len() int { return len(idx.byPath) } + +func copyPostings[K comparable](m map[K][]uint32) map[K][]uint32 { + cp := make(map[K][]uint32, len(m)) + for k, v := range m { + cp[k] = v[:len(v):len(v)] + } + return cp +} + +// Snapshot returns a frozen read-only view of the index. +func (idx *Index) Snapshot() *Snapshot { + del := make(map[uint32]bool, len(idx.deleted)) + for id := range idx.deleted { + del[id] = true + } + var p1Copy [256][]uint32 + for i, ids := range idx.byPrefix1 { + if len(ids) > 0 { + p1Copy[i] = ids[:len(ids):len(ids)] + } + } + return &Snapshot{ + docs: idx.docs[:len(idx.docs):len(idx.docs)], + deleted: del, + byGram: copyPostings(idx.byGram), + byPrefix1: p1Copy, + byPrefix2: copyPostings(idx.byPrefix2), + } +} diff --git a/agent/filefinder/delta_test.go b/agent/filefinder/delta_test.go new file mode 100644 index 0000000000..f2bbceb015 --- /dev/null +++ b/agent/filefinder/delta_test.go @@ -0,0 +1,120 @@ +package filefinder_test + +import ( + "testing" + + "github.com/coder/coder/v2/agent/filefinder" +) + +func TestIndex_AddAndLen(t *testing.T) { + t.Parallel() + idx := filefinder.NewIndex() + idx.Add("foo/bar.go", 0) + idx.Add("foo/baz.go", 0) + if idx.Len() != 2 { + t.Fatalf("expected 2, got %d", idx.Len()) + } +} + +func TestIndex_Has(t *testing.T) { + t.Parallel() + idx := filefinder.NewIndex() + idx.Add("foo/bar.go", 0) + if !idx.Has("foo/bar.go") { + t.Fatal("expected Has to return true") + } + if idx.Has("foo/missing.go") { + t.Fatal("expected Has to return false for missing path") + } +} + +func TestIndex_Remove(t *testing.T) { + t.Parallel() + idx := filefinder.NewIndex() + idx.Add("foo/bar.go", 0) + if !idx.Remove("foo/bar.go") { + t.Fatal("expected Remove to return true") + } + if idx.Has("foo/bar.go") { + t.Fatal("expected Has to return false after Remove") + } + if idx.Len() != 0 { + t.Fatalf("expected Len 0 after Remove, got %d", idx.Len()) + } +} + +func TestIndex_AddOverwrite(t *testing.T) { + t.Parallel() + idx := filefinder.NewIndex() + idx.Add("foo/bar.go", uint16(filefinder.FlagFile)) + idx.Add("foo/bar.go", uint16(filefinder.FlagDir)) // overwrite + if idx.Len() != 1 { + t.Fatalf("expected 1 after overwrite, got %d", idx.Len()) + } + // The old entry should be tombstoned. + if !filefinder.IndexIsDeleted(idx, 0) { + t.Fatal("expected old entry to be deleted") + } + if filefinder.IndexIsDeleted(idx, 1) { + t.Fatal("expected new entry to be live") + } +} + +func TestIndex_Snapshot(t *testing.T) { + t.Parallel() + idx := filefinder.NewIndex() + idx.Add("foo/bar.go", 0) + idx.Add("foo/baz.go", 0) + + snap := idx.Snapshot() + if filefinder.SnapshotCount(snap) != 2 { + t.Fatalf("expected snapshot count 2, got %d", filefinder.SnapshotCount(snap)) + } + + // Adding more docs after snapshot doesn't affect it. + idx.Add("foo/qux.go", 0) + if filefinder.SnapshotCount(snap) != 2 { + t.Fatal("snapshot count should not change after new adds") + } +} + +func TestIndex_TrigramIndex(t *testing.T) { + t.Parallel() + idx := filefinder.NewIndex() + idx.Add("handler.go", 0) + + // "handler.go" should produce trigrams for "handler.go". + // Check that at least one trigram exists. + if filefinder.IndexByGramLen(idx) == 0 { + t.Fatal("expected non-empty trigram index") + } +} + +func TestIndex_PrefixIndex(t *testing.T) { + t.Parallel() + idx := filefinder.NewIndex() + idx.Add("handler.go", 0) + + // basename is "handler.go", first byte is 'h' + if filefinder.IndexByPrefix1Len(idx, 'h') == 0 { + t.Fatal("expected prefix1['h'] to be non-empty") + } +} + +func TestIndex_RemoveNonexistent(t *testing.T) { + t.Parallel() + idx := filefinder.NewIndex() + if idx.Remove("nonexistent.go") { + t.Fatal("expected Remove to return false for missing path") + } +} + +func TestIndex_PathNormalization(t *testing.T) { + t.Parallel() + idx := filefinder.NewIndex() + idx.Add("Foo/Bar.go", 0) + // Should be findable with lowercase. + if !idx.Has("foo/bar.go") { + t.Fatal("expected case-insensitive Has") + } +} diff --git a/agent/filefinder/engine.go b/agent/filefinder/engine.go new file mode 100644 index 0000000000..b7aae2dc90 --- /dev/null +++ b/agent/filefinder/engine.go @@ -0,0 +1,364 @@ +// Package filefinder provides an in-memory file index with trigram +// matching, fuzzy search, and filesystem watching. It is designed +// to power file-finding features on workspace agents. +package filefinder + +import ( + "context" + "os" + "path/filepath" + "slices" + "strings" + "sync" + "sync/atomic" + + "golang.org/x/xerrors" + + "cdr.dev/slog/v3" +) + +// SearchOptions controls search behavior. +type SearchOptions struct { + Limit int + MaxCandidates int +} + +// DefaultSearchOptions returns sensible default search options. +func DefaultSearchOptions() SearchOptions { + return SearchOptions{Limit: 100, MaxCandidates: 10000} +} + +type rootSnapshot struct { + root string + snap *Snapshot +} + +// Engine is the main file finder. Safe for concurrent use. +type Engine struct { + snap atomic.Pointer[[]*rootSnapshot] + logger slog.Logger + mu sync.Mutex + roots map[string]*rootState + eventCh chan rootEvent + closeCh chan struct{} + closed atomic.Bool + wg sync.WaitGroup +} +type rootState struct { + root string + index *Index + watcher *fsWatcher + cancel context.CancelFunc +} +type rootEvent struct { + root string + events []FSEvent +} + +// walkRoot performs a full filesystem walk of absRoot and returns +// a populated Index containing all discovered files and directories. +func walkRoot(absRoot string) (*Index, error) { + idx := NewIndex() + err := filepath.Walk(absRoot, func(path string, info os.FileInfo, walkErr error) error { + if walkErr != nil { + return nil //nolint:nilerr + } + base := filepath.Base(path) + if _, skip := skipDirs[base]; skip && info.IsDir() { + return filepath.SkipDir + } + if path == absRoot { + return nil + } + relPath, relErr := filepath.Rel(absRoot, path) + if relErr != nil { + return nil //nolint:nilerr + } + relPath = filepath.ToSlash(relPath) + var flags uint16 + if info.IsDir() { + flags = uint16(FlagDir) + } else if info.Mode()&os.ModeSymlink != 0 { + flags = uint16(FlagSymlink) + } + idx.Add(relPath, flags) + return nil + }) + return idx, err +} + +// NewEngine creates a new Engine. +func NewEngine(logger slog.Logger) *Engine { + e := &Engine{ + logger: logger, + roots: make(map[string]*rootState), + eventCh: make(chan rootEvent, 256), + closeCh: make(chan struct{}), + } + empty := make([]*rootSnapshot, 0) + e.snap.Store(&empty) + e.wg.Add(1) + go e.start() + return e +} + +// ErrClosed is returned when operations are attempted on a +// closed engine. +var ErrClosed = xerrors.New("engine is closed") + +// AddRoot adds a directory root to the engine. +func (e *Engine) AddRoot(ctx context.Context, root string) error { + absRoot, err := filepath.Abs(root) + if err != nil { + return xerrors.Errorf("resolve root: %w", err) + } + e.mu.Lock() + if e.closed.Load() { + e.mu.Unlock() + return ErrClosed + } + if _, exists := e.roots[absRoot]; exists { + e.mu.Unlock() + return nil + } + e.mu.Unlock() + + // Walk and create the watcher outside the lock to avoid + // blocking the event pipeline on filesystem I/O. + idx, walkErr := walkRoot(absRoot) + if walkErr != nil { + return xerrors.Errorf("walk root: %w", walkErr) + } + wCtx, wCancel := context.WithCancel(context.Background()) + w, wErr := newFSWatcher(absRoot, e.logger) + if wErr != nil { + wCancel() + return xerrors.Errorf("create watcher: %w", wErr) + } + + e.mu.Lock() + // Re-check after re-acquiring the lock: another goroutine + // may have added this root or closed the engine while we + // were walking. + if e.closed.Load() { + e.mu.Unlock() + wCancel() + _ = w.Close() + return ErrClosed + } + if _, exists := e.roots[absRoot]; exists { + e.mu.Unlock() + wCancel() + _ = w.Close() + return nil + } + rs := &rootState{root: absRoot, index: idx, watcher: w, cancel: wCancel} + e.roots[absRoot] = rs + w.Start(wCtx) + e.wg.Add(1) + go e.forwardEvents(wCtx, absRoot, w) + e.publishSnapshot() + fileCount := idx.Len() + e.mu.Unlock() + e.logger.Info(ctx, "added root to engine", + slog.F("root", absRoot), + slog.F("files", fileCount), + ) + return nil +} + +// RemoveRoot stops watching a root and removes it. +func (e *Engine) RemoveRoot(root string) error { + absRoot, err := filepath.Abs(root) + if err != nil { + return xerrors.Errorf("resolve root: %w", err) + } + e.mu.Lock() + defer e.mu.Unlock() + rs, exists := e.roots[absRoot] + if !exists { + return xerrors.Errorf("root %q not found", absRoot) + } + rs.cancel() + _ = rs.watcher.Close() + delete(e.roots, absRoot) + e.publishSnapshot() + return nil +} + +// Search performs a fuzzy file search across all roots. +func (e *Engine) Search(_ context.Context, query string, opts SearchOptions) ([]Result, error) { + if e.closed.Load() { + return nil, ErrClosed + } + snapPtr := e.snap.Load() + if snapPtr == nil || len(*snapPtr) == 0 { + return nil, nil + } + roots := *snapPtr + plan := newQueryPlan(query) + if len(plan.Normalized) == 0 { + return nil, nil + } + if opts.Limit <= 0 { + opts.Limit = 100 + } + if opts.MaxCandidates <= 0 { + opts.MaxCandidates = 10000 + } + params := defaultScoreParams() + var allCands []candidate + for _, rs := range roots { + allCands = append(allCands, searchSnapshot(plan, rs.snap, opts.MaxCandidates)...) + } + results := mergeAndScore(allCands, plan, params, opts.Limit) + return results, nil +} + +// Close shuts down the engine. +func (e *Engine) Close() error { + if e.closed.Swap(true) { + return nil + } + close(e.closeCh) + e.mu.Lock() + for _, rs := range e.roots { + rs.cancel() + _ = rs.watcher.Close() + } + e.roots = make(map[string]*rootState) + e.mu.Unlock() + e.wg.Wait() + return nil +} + +// Rebuild forces a complete re-walk and re-index of a root. +func (e *Engine) Rebuild(ctx context.Context, root string) error { + absRoot, err := filepath.Abs(root) + if err != nil { + return xerrors.Errorf("resolve root: %w", err) + } + + // Walk outside the lock to avoid blocking the event + // pipeline on potentially slow filesystem I/O. + idx, walkErr := walkRoot(absRoot) + if walkErr != nil { + return xerrors.Errorf("rebuild walk: %w", walkErr) + } + + e.mu.Lock() + rs, exists := e.roots[absRoot] + if !exists { + e.mu.Unlock() + return xerrors.Errorf("root %q not found", absRoot) + } + rs.index = idx + e.publishSnapshot() + fileCount := idx.Len() + e.mu.Unlock() + e.logger.Info(ctx, "rebuilt root in engine", + slog.F("root", absRoot), + slog.F("files", fileCount), + ) + return nil +} + +func (e *Engine) start() { + defer e.wg.Done() + for { + select { + case <-e.closeCh: + return + case re, ok := <-e.eventCh: + if !ok { + return + } + e.applyEvents(re) + } + } +} + +func (e *Engine) forwardEvents(ctx context.Context, root string, w *fsWatcher) { + defer e.wg.Done() + for { + select { + case <-ctx.Done(): + return + case <-e.closeCh: + return + case evts, ok := <-w.Events(): + if !ok { + return + } + select { + case e.eventCh <- rootEvent{root: root, events: evts}: + case <-ctx.Done(): + return + case <-e.closeCh: + return + } + } + } +} + +func (e *Engine) applyEvents(re rootEvent) { + e.mu.Lock() + defer e.mu.Unlock() + rs, exists := e.roots[re.root] + if !exists { + return + } + changed := false + for _, ev := range re.events { + relPath, err := filepath.Rel(rs.root, ev.Path) + if err != nil { + continue + } + relPath = filepath.ToSlash(relPath) + switch ev.Op { + case OpCreate: + if rs.index.Has(relPath) { + continue + } + var flags uint16 + if ev.IsDir { + flags = uint16(FlagDir) + } + rs.index.Add(relPath, flags) + changed = true + case OpRemove, OpRename: + if rs.index.Remove(relPath) { + changed = true + } + if ev.IsDir || ev.Op == OpRename { + prefix := strings.ToLower(filepath.ToSlash(relPath)) + "/" + for path := range rs.index.byPath { + if strings.HasPrefix(path, prefix) { + rs.index.Remove(path) + changed = true + } + } + } + case OpModify: + } + } + if changed { + e.publishSnapshot() + } +} + +// publishSnapshot builds and atomically publishes a new snapshot. +// Must be called with e.mu held. +func (e *Engine) publishSnapshot() { + roots := make([]*rootSnapshot, 0, len(e.roots)) + for _, rs := range e.roots { + roots = append(roots, &rootSnapshot{ + root: rs.root, + snap: rs.index.Snapshot(), + }) + } + slices.SortFunc(roots, func(a, b *rootSnapshot) int { + return strings.Compare(a.root, b.root) + }) + e.snap.Store(&roots) +} diff --git a/agent/filefinder/engine_test.go b/agent/filefinder/engine_test.go new file mode 100644 index 0000000000..17ba761915 --- /dev/null +++ b/agent/filefinder/engine_test.go @@ -0,0 +1,233 @@ +package filefinder_test + +import ( + "context" + "os" + "path/filepath" + "sort" + "testing" + + "github.com/stretchr/testify/require" + + "cdr.dev/slog/v3" + "cdr.dev/slog/v3/sloggers/slogtest" + "github.com/coder/coder/v2/agent/filefinder" + "github.com/coder/coder/v2/testutil" +) + +func newTestEngine(t *testing.T) (*filefinder.Engine, context.Context) { + t.Helper() + logger := slogtest.Make(t, &slogtest.Options{IgnoreErrors: true}).Leveled(slog.LevelDebug) + eng := filefinder.NewEngine(logger) + t.Cleanup(func() { _ = eng.Close() }) + return eng, context.Background() +} + +func requireResultHasPath(t *testing.T, results []filefinder.Result, path string) { + t.Helper() + for _, r := range results { + if r.Path == path { + return + } + } + t.Errorf("expected %q in results, got %v", path, resultPaths(results)) +} + +func TestEngine_SearchFindsKnownFile(t *testing.T) { + t.Parallel() + dir := t.TempDir() + createFile(t, dir, "src/main.go", "package main") + createFile(t, dir, "src/handler.go", "package main") + createFile(t, dir, "README.md", "# hello") + + eng, ctx := newTestEngine(t) + require.NoError(t, eng.AddRoot(ctx, dir)) + + results, err := eng.Search(ctx, "main.go", filefinder.DefaultSearchOptions()) + require.NoError(t, err) + require.NotEmpty(t, results, "expected to find main.go") + requireResultHasPath(t, results, "src/main.go") +} + +func TestEngine_SearchFuzzyMatch(t *testing.T) { + t.Parallel() + dir := t.TempDir() + createFile(t, dir, "src/controllers/user_handler.go", "package controllers") + createFile(t, dir, "src/models/user.go", "package models") + createFile(t, dir, "docs/api.md", "# API") + + eng, ctx := newTestEngine(t) + require.NoError(t, eng.AddRoot(ctx, dir)) + + // "handler" should match "user_handler.go". + results, err := eng.Search(ctx, "handler", filefinder.DefaultSearchOptions()) + require.NoError(t, err) + // The query is a subsequence of "user_handler.go" so it + // should appear somewhere in the results. + requireResultHasPath(t, results, "src/controllers/user_handler.go") +} + +func TestEngine_IndexPicksUpNewFile(t *testing.T) { + t.Parallel() + dir := t.TempDir() + createFile(t, dir, "existing.txt", "hello") + + eng, ctx := newTestEngine(t) + require.NoError(t, eng.AddRoot(ctx, dir)) + createFile(t, dir, "newfile_unique.txt", "world") + + require.Eventually(t, func() bool { + results, sErr := eng.Search(ctx, "newfile_unique", filefinder.DefaultSearchOptions()) + if sErr != nil { + return false + } + for _, r := range results { + if r.Path == "newfile_unique.txt" { + return true + } + } + return false + }, testutil.WaitShort, testutil.IntervalFast, "expected newfile_unique.txt to appear via watcher") +} + +func TestEngine_IndexRemovesDeletedFile(t *testing.T) { + t.Parallel() + dir := t.TempDir() + createFile(t, dir, "deleteme_unique.txt", "goodbye") + createFile(t, dir, "keeper.txt", "stay") + + eng, ctx := newTestEngine(t) + require.NoError(t, eng.AddRoot(ctx, dir)) + + results, err := eng.Search(ctx, "deleteme_unique", filefinder.DefaultSearchOptions()) + require.NoError(t, err) + require.NotEmpty(t, results, "expected to find deleteme_unique.txt initially") + + require.NoError(t, os.Remove(filepath.Join(dir, "deleteme_unique.txt"))) + + require.Eventually(t, func() bool { + results, sErr := eng.Search(ctx, "deleteme_unique", filefinder.DefaultSearchOptions()) + if sErr != nil { + return false + } + for _, r := range results { + if r.Path == "deleteme_unique.txt" { + return false // still found + } + } + return true + }, testutil.WaitShort, testutil.IntervalFast, "expected deleteme_unique.txt to disappear after removal") +} + +func TestEngine_MultipleRoots(t *testing.T) { + t.Parallel() + dir1 := t.TempDir() + dir2 := t.TempDir() + createFile(t, dir1, "alpha_unique.go", "package alpha") + createFile(t, dir2, "beta_unique.go", "package beta") + + eng, ctx := newTestEngine(t) + require.NoError(t, eng.AddRoot(ctx, dir1)) + require.NoError(t, eng.AddRoot(ctx, dir2)) + + results, err := eng.Search(ctx, "alpha_unique", filefinder.DefaultSearchOptions()) + require.NoError(t, err) + requireResultHasPath(t, results, "alpha_unique.go") + + results, err = eng.Search(ctx, "beta_unique", filefinder.DefaultSearchOptions()) + require.NoError(t, err) + requireResultHasPath(t, results, "beta_unique.go") +} + +func TestEngine_EmptyQueryReturnsEmpty(t *testing.T) { + t.Parallel() + dir := t.TempDir() + createFile(t, dir, "something.txt", "data") + + eng, ctx := newTestEngine(t) + require.NoError(t, eng.AddRoot(ctx, dir)) + + results, err := eng.Search(ctx, "", filefinder.DefaultSearchOptions()) + require.NoError(t, err) + require.Empty(t, results, "empty query should return no results") +} + +func TestEngine_CloseIsClean(t *testing.T) { + t.Parallel() + dir := t.TempDir() + createFile(t, dir, "file.txt", "data") + + logger := slogtest.Make(t, &slogtest.Options{IgnoreErrors: true}).Leveled(slog.LevelDebug) + ctx := context.Background() + eng := filefinder.NewEngine(logger) + require.NoError(t, eng.AddRoot(ctx, dir)) + require.NoError(t, eng.Close()) + + _, err := eng.Search(ctx, "file", filefinder.DefaultSearchOptions()) + require.Error(t, err) +} + +func TestEngine_AddRootIdempotent(t *testing.T) { + t.Parallel() + dir := t.TempDir() + createFile(t, dir, "file.txt", "data") + + eng, ctx := newTestEngine(t) + require.NoError(t, eng.AddRoot(ctx, dir)) + require.NoError(t, eng.AddRoot(ctx, dir)) + + snapLen := filefinder.EngineSnapLen(eng) + require.Equal(t, 1, snapLen, "expected exactly one root after duplicate add") +} + +func TestEngine_RemoveRoot(t *testing.T) { + t.Parallel() + dir := t.TempDir() + createFile(t, dir, "file.txt", "data") + + eng, ctx := newTestEngine(t) + require.NoError(t, eng.AddRoot(ctx, dir)) + + results, err := eng.Search(ctx, "file", filefinder.DefaultSearchOptions()) + require.NoError(t, err) + require.NotEmpty(t, results) + + require.NoError(t, eng.RemoveRoot(dir)) + + results, err = eng.Search(ctx, "file", filefinder.DefaultSearchOptions()) + require.NoError(t, err) + require.Empty(t, results) +} + +func TestEngine_Rebuild(t *testing.T) { + t.Parallel() + dir := t.TempDir() + createFile(t, dir, "original.txt", "data") + + eng, ctx := newTestEngine(t) + require.NoError(t, eng.AddRoot(ctx, dir)) + + createFile(t, dir, "sneaky_rebuild.txt", "hidden") + require.NoError(t, eng.Rebuild(ctx, dir)) + + results, err := eng.Search(ctx, "sneaky_rebuild", filefinder.DefaultSearchOptions()) + require.NoError(t, err) + requireResultHasPath(t, results, "sneaky_rebuild.txt") +} + +// createFile creates a file (and parent dirs) at relPath under dir. +func createFile(t *testing.T, dir, relPath, content string) { + t.Helper() + full := filepath.Join(dir, relPath) + require.NoError(t, os.MkdirAll(filepath.Dir(full), 0o755)) + require.NoError(t, os.WriteFile(full, []byte(content), 0o600)) +} + +func resultPaths(results []filefinder.Result) []string { + paths := make([]string, len(results)) + for i, r := range results { + paths[i] = r.Path + } + sort.Strings(paths) + return paths +} diff --git a/agent/filefinder/export_test.go b/agent/filefinder/export_test.go new file mode 100644 index 0000000000..74db437978 --- /dev/null +++ b/agent/filefinder/export_test.go @@ -0,0 +1,85 @@ +package filefinder + +// Test helpers that need internal access. + +// MakeTestSnapshot builds a Snapshot from a list of paths. Useful for +// query-level tests that don't need a real filesystem. +func MakeTestSnapshot(paths []string) *Snapshot { + idx := NewIndex() + for _, p := range paths { + idx.Add(p, 0) + } + return idx.Snapshot() +} + +// BuildTestIndex walks root and returns a populated Index, the same +// way Engine.AddRoot does but without starting a watcher. +func BuildTestIndex(root string) (*Index, error) { + return walkRoot(root) +} + +// IndexIsDeleted reports whether the document at id is tombstoned. +func IndexIsDeleted(idx *Index, id uint32) bool { + return idx.deleted[id] +} + +// IndexByGramLen returns the number of entries in the trigram index. +func IndexByGramLen(idx *Index) int { + return len(idx.byGram) +} + +// IndexByPrefix1Len returns the number of posting-list entries for +// the given single-byte prefix. +func IndexByPrefix1Len(idx *Index, b byte) int { + return len(idx.byPrefix1[b]) +} + +// SnapshotCount returns the number of documents in a Snapshot. +func SnapshotCount(snap *Snapshot) int { + return len(snap.docs) +} + +// EngineSnapLen returns the number of root snapshots currently held +// by the engine, or -1 if the pointer is nil. +func EngineSnapLen(eng *Engine) int { + p := eng.snap.Load() + if p == nil { + return -1 + } + return len(*p) +} + +// DefaultScoreParamsForTest exposes defaultScoreParams for tests. +var DefaultScoreParamsForTest = defaultScoreParams + +// ScoreParamsForTest is a type alias for scoreParams. +type ScoreParamsForTest = scoreParams + +// Exported aliases for internal functions used in tests. +var ( + NewQueryPlanForTest = newQueryPlan + SearchSnapshotForTest = searchSnapshot + IntersectSortedForTest = intersectSorted + IntersectAllForTest = intersectAll + MergeAndScoreForTest = mergeAndScore + NormalizeQueryForTest = normalizeQuery + NormalizePathBytesForTest = normalizePathBytes + ExtractTrigramsForTest = extractTrigrams + ExtractBasenameForTest = extractBasename + ExtractSegmentsForTest = extractSegments + Prefix1ForTest = prefix1 + Prefix2ForTest = prefix2 + IsSubsequenceForTest = isSubsequence + LongestContiguousMatchForTest = longestContiguousMatch + IsBoundaryForTest = isBoundary + CountBoundaryHitsForTest = countBoundaryHits + EqualFoldASCIIForTest = equalFoldASCII + ScorePathForTest = scorePath + PackTrigramForTest = packTrigram +) + +// Type aliases for internal types used in tests. +type ( + CandidateForTest = candidate + QueryPlanForTest = queryPlan +) diff --git a/agent/filefinder/query.go b/agent/filefinder/query.go new file mode 100644 index 0000000000..15c13dd1f3 --- /dev/null +++ b/agent/filefinder/query.go @@ -0,0 +1,299 @@ +package filefinder + +import ( + "container/heap" + "slices" + "strings" +) + +type candidate struct { + DocID uint32 + Path string + BaseOff int + BaseLen int + Depth int + Flags uint16 +} + +// Result is a scored search result returned to callers. +type Result struct { + Path string + Score float32 + IsDir bool +} + +type queryPlan struct { + Original string + Normalized string + Tokens [][]byte + Trigrams []uint32 + IsShort bool + HasSlash bool + BasenameQ []byte + DirTokens [][]byte +} + +func newQueryPlan(q string) *queryPlan { + norm := normalizeQuery(q) + p := &queryPlan{Original: q, Normalized: norm} + if len(norm) == 0 { + p.IsShort = true + return p + } + raw := strings.ReplaceAll(norm, "/", " ") + parts := strings.Fields(raw) + p.HasSlash = strings.ContainsRune(norm, '/') + for _, part := range parts { + p.Tokens = append(p.Tokens, []byte(part)) + } + if len(p.Tokens) > 0 { + p.BasenameQ = p.Tokens[len(p.Tokens)-1] + if len(p.Tokens) > 1 { + p.DirTokens = p.Tokens[:len(p.Tokens)-1] + } + } + p.IsShort = true + for _, tok := range p.Tokens { + if len(tok) >= 3 { + p.IsShort = false + break + } + } + if !p.IsShort { + p.Trigrams = extractQueryTrigrams(p.Tokens) + } + return p +} + +func extractQueryTrigrams(tokens [][]byte) []uint32 { + seen := make(map[uint32]struct{}) + for _, tok := range tokens { + if len(tok) < 3 { + continue + } + for i := 0; i <= len(tok)-3; i++ { + seen[packTrigram(tok[i], tok[i+1], tok[i+2])] = struct{}{} + } + } + if len(seen) == 0 { + return nil + } + result := make([]uint32, 0, len(seen)) + for g := range seen { + result = append(result, g) + } + return result +} + +func packTrigram(a, b, c byte) uint32 { + return uint32(toLowerASCII(a))<<16 | uint32(toLowerASCII(b))<<8 | uint32(toLowerASCII(c)) +} + +// searchSnapshot runs the full search pipeline against a single +// root snapshot: it selects a strategy (prefix, trigram, or +// fuzzy fallback) based on query length, retrieves candidate +// doc IDs, and converts them into candidate structs. +func searchSnapshot(plan *queryPlan, snap *Snapshot, limit int) []candidate { + if snap == nil || len(snap.docs) == 0 || len(plan.Normalized) == 0 { + return nil + } + var ids []uint32 + if plan.IsShort { + ids = searchShort(plan, snap) + } else { + ids = searchTrigrams(plan, snap) + if len(ids) == 0 && len(plan.BasenameQ) > 0 { + ids = searchFuzzyFallback(plan, snap) + } + } + if len(ids) == 0 { + return nil + } + cands := make([]candidate, 0, min(len(ids), limit)) + for _, id := range ids { + if snap.deleted[id] || int(id) >= len(snap.docs) { + continue + } + d := snap.docs[id] + cands = append(cands, candidate{ + DocID: id, Path: d.path, BaseOff: d.baseOff, + BaseLen: d.baseLen, Depth: d.depth, Flags: d.flags, + }) + if len(cands) >= limit { + break + } + } + return cands +} + +func searchShort(plan *queryPlan, snap *Snapshot) []uint32 { + if len(plan.BasenameQ) == 0 { + return nil + } + if len(plan.BasenameQ) >= 2 { + if ids := snap.byPrefix2[prefix2(plan.BasenameQ)]; len(ids) > 0 { + return ids + } + } + return snap.byPrefix1[prefix1(plan.BasenameQ)] +} + +func searchTrigrams(plan *queryPlan, snap *Snapshot) []uint32 { + if len(plan.Trigrams) == 0 { + return nil + } + lists := make([][]uint32, 0, len(plan.Trigrams)) + for _, g := range plan.Trigrams { + ids, ok := snap.byGram[g] + if !ok || len(ids) == 0 { + return nil + } + lists = append(lists, ids) + } + return intersectAll(lists) +} + +func searchFuzzyFallback(plan *queryPlan, snap *Snapshot) []uint32 { + if len(plan.BasenameQ) == 0 { + return nil + } + bucket := snap.byPrefix1[prefix1(plan.BasenameQ)] + if len(bucket) == 0 { + return searchSubsequenceScan(plan, snap, 5000) + } + var ids []uint32 + for _, id := range bucket { + if snap.deleted[id] || int(id) >= len(snap.docs) { + continue + } + if isSubsequence([]byte(snap.docs[id].path), plan.BasenameQ) { + ids = append(ids, id) + } + } + if len(ids) == 0 { + return searchSubsequenceScan(plan, snap, 5000) + } + return ids +} + +func searchSubsequenceScan(plan *queryPlan, snap *Snapshot, maxCheck int) []uint32 { + if len(plan.BasenameQ) == 0 { + return nil + } + var ids []uint32 + checked := 0 + for id := 0; id < len(snap.docs) && checked < maxCheck; id++ { + uid := uint32(id) //nolint:gosec // Snapshot count is bounded well below 2^32. + if snap.deleted[uid] { + continue + } + checked++ + if isSubsequence([]byte(snap.docs[id].path), plan.BasenameQ) { + ids = append(ids, uid) + } + } + return ids +} + +func intersectSorted(a, b []uint32) []uint32 { + if len(a) == 0 || len(b) == 0 { + return nil + } + var result []uint32 + ai, bi := 0, 0 + for ai < len(a) && bi < len(b) { + switch { + case a[ai] < b[bi]: + ai++ + case a[ai] > b[bi]: + bi++ + default: + result = append(result, a[ai]) + ai++ + bi++ + } + } + return result +} + +func intersectAll(lists [][]uint32) []uint32 { + if len(lists) == 0 { + return nil + } + if len(lists) == 1 { + return lists[0] + } + slices.SortFunc(lists, func(a, b []uint32) int { return len(a) - len(b) }) + result := lists[0] + for i := 1; i < len(lists) && len(result) > 0; i++ { + result = intersectSorted(result, lists[i]) + } + return result +} + +func mergeAndScore(cands []candidate, plan *queryPlan, params scoreParams, topK int) []Result { + if topK <= 0 || len(cands) == 0 { + return nil + } + query := []byte(plan.Normalized) + h := &resultHeap{} + heap.Init(h) + for i := range cands { + c := &cands[i] + s := scorePath([]byte(c.Path), c.BaseOff, c.BaseLen, c.Depth, query, plan.Tokens, params) + if s <= 0 { + continue + } + // DirTokenHit is applied here rather than in scorePath because + // it depends on the query plan's directory tokens, which are + // split from the full query during planning. scorePath operates + // on raw query bytes without knowledge of token boundaries. + if len(plan.DirTokens) > 0 { + segments := extractSegments([]byte(c.Path)) + for _, dt := range plan.DirTokens { + for _, seg := range segments { + if equalFoldASCII(seg, dt) { + s += params.DirTokenHit + break + } + } + } + } + r := Result{Path: c.Path, Score: s, IsDir: c.Flags == uint16(FlagDir)} + if h.Len() < topK { + heap.Push(h, r) + } else if s > (*h)[0].Score { + (*h)[0] = r + heap.Fix(h, 0) + } + } + n := h.Len() + results := make([]Result, n) + for i := n - 1; i >= 0; i-- { + v := heap.Pop(h) + if r, ok := v.(Result); ok { + results[i] = r + } + } + return results +} + +type resultHeap []Result + +func (h resultHeap) Len() int { return len(h) } +func (h resultHeap) Less(i, j int) bool { return h[i].Score < h[j].Score } +func (h resultHeap) Swap(i, j int) { h[i], h[j] = h[j], h[i] } +func (h *resultHeap) Push(x interface{}) { + r, ok := x.(Result) + if ok { + *h = append(*h, r) + } +} + +func (h *resultHeap) Pop() interface{} { + old := *h + n := len(old) + x := old[n-1] + *h = old[:n-1] + return x +} diff --git a/agent/filefinder/query_test.go b/agent/filefinder/query_test.go new file mode 100644 index 0000000000..23883033cb --- /dev/null +++ b/agent/filefinder/query_test.go @@ -0,0 +1,343 @@ +package filefinder_test + +import ( + "slices" + "testing" + + "github.com/coder/coder/v2/agent/filefinder" +) + +func TestNewQueryPlan(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + query string + wantNorm string + wantShort bool + wantSlash bool + wantBase string + wantTokens []string + wantDirTok []string + wantTriCnt int // -1 to skip check + }{ + {"Simple", "foo", "foo", false, false, "foo", []string{"foo"}, nil, 1}, + {"MultiToken", "foo bar", "foo bar", false, false, "bar", []string{"foo", "bar"}, []string{"foo"}, -1}, + {"Slash", "internal/foo", "internal/foo", false, true, "foo", []string{"internal", "foo"}, []string{"internal"}, -1}, + {"SingleChar", "a", "a", true, false, "a", []string{"a"}, nil, 0}, + {"TwoChars", "ab", "ab", true, false, "ab", []string{"ab"}, nil, -1}, + {"ThreeChars", "abc", "abc", false, false, "abc", []string{"abc"}, nil, 1}, + {"DotPrefix", ".go", ".go", false, false, ".go", []string{".go"}, nil, -1}, + {"UpperCase", "FOO", "foo", false, false, "foo", []string{"foo"}, nil, -1}, + {"Empty", "", "", true, false, "", nil, nil, -1}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + plan := filefinder.NewQueryPlanForTest(tt.query) + if plan.Normalized != tt.wantNorm { + t.Errorf("normalized = %q, want %q", plan.Normalized, tt.wantNorm) + } + if plan.IsShort != tt.wantShort { + t.Errorf("isShort = %v, want %v", plan.IsShort, tt.wantShort) + } + if plan.HasSlash != tt.wantSlash { + t.Errorf("hasSlash = %v, want %v", plan.HasSlash, tt.wantSlash) + } + if string(plan.BasenameQ) != tt.wantBase { + t.Errorf("basenameQ = %q, want %q", plan.BasenameQ, tt.wantBase) + } + if tt.wantTokens == nil { + if len(plan.Tokens) != 0 { + t.Errorf("expected 0 tokens, got %d", len(plan.Tokens)) + } + } else { + if len(plan.Tokens) != len(tt.wantTokens) { + t.Fatalf("tokens len = %d, want %d", len(plan.Tokens), len(tt.wantTokens)) + } + for i, tok := range plan.Tokens { + if string(tok) != tt.wantTokens[i] { + t.Errorf("tokens[%d] = %q, want %q", i, tok, tt.wantTokens[i]) + } + } + } + if tt.wantDirTok != nil { + if len(plan.DirTokens) != len(tt.wantDirTok) { + t.Fatalf("dirTokens len = %d, want %d", len(plan.DirTokens), len(tt.wantDirTok)) + } + for i, tok := range plan.DirTokens { + if string(tok) != tt.wantDirTok[i] { + t.Errorf("dirTokens[%d] = %q, want %q", i, tok, tt.wantDirTok[i]) + } + } + } + if tt.wantTriCnt >= 0 && len(plan.Trigrams) != tt.wantTriCnt { + t.Errorf("trigram count = %d, want %d", len(plan.Trigrams), tt.wantTriCnt) + } + }) + } + + // ThreeChars: verify the actual trigram value. + plan := filefinder.NewQueryPlanForTest("abc") + if want := filefinder.PackTrigramForTest('a', 'b', 'c'); plan.Trigrams[0] != want { + t.Errorf("trigram = %x, want %x", plan.Trigrams[0], want) + } + + // ShortMultiToken: both tokens < 3 chars so isShort should be true. + plan = filefinder.NewQueryPlanForTest("ab cd") + if !plan.IsShort { + t.Error("expected isShort=true when all tokens < 3 chars") + } + // One token >= 3 chars, so isShort should be false. + plan = filefinder.NewQueryPlanForTest("ab cde") + if plan.IsShort { + t.Error("expected isShort=false when any token >= 3 chars") + } +} + +func requireCandHasPath(t *testing.T, cands []filefinder.CandidateForTest, path string) { + t.Helper() + for _, c := range cands { + if c.Path == path { + return + } + } + t.Errorf("expected to find %q in candidates", path) +} + +func TestSearchSnapshot_TrigramMatch(t *testing.T) { + t.Parallel() + snap := filefinder.MakeTestSnapshot([]string{"src/handler.go", "src/router.go", "lib/utils.go"}) + cands := filefinder.SearchSnapshotForTest(filefinder.NewQueryPlanForTest("handler"), snap, 100) + if len(cands) == 0 { + t.Fatal("expected at least 1 candidate for 'handler'") + } + requireCandHasPath(t, cands, "src/handler.go") +} + +func TestSearchSnapshot_ShortQuery(t *testing.T) { + t.Parallel() + snap := filefinder.MakeTestSnapshot([]string{"foo.go", "bar.go", "fab.go"}) + cands := filefinder.SearchSnapshotForTest(filefinder.NewQueryPlanForTest("fo"), snap, 100) + if len(cands) == 0 { + t.Fatal("expected at least 1 candidate for 'fo'") + } + requireCandHasPath(t, cands, "foo.go") +} + +func TestSearchSnapshot_FuzzyFallback(t *testing.T) { + t.Parallel() + snap := filefinder.MakeTestSnapshot([]string{"src/handler.go", "src/router.go", "lib/utils.go"}) + cands := filefinder.SearchSnapshotForTest(filefinder.NewQueryPlanForTest("hndlr"), snap, 100) + if len(cands) == 0 { + t.Fatal("expected fuzzy fallback to find 'handler.go' for query 'hndlr'") + } + requireCandHasPath(t, cands, "src/handler.go") +} + +func TestSearchSnapshot_FuzzyFallbackNoFirstCharMatch(t *testing.T) { + t.Parallel() + snap := filefinder.MakeTestSnapshot([]string{"src/xylophone.go", "lib/extra.go"}) + cands := filefinder.SearchSnapshotForTest(filefinder.NewQueryPlanForTest("xylo"), snap, 100) + if len(cands) == 0 { + t.Fatal("expected at least 1 candidate for 'xylo'") + } + requireCandHasPath(t, cands, "src/xylophone.go") +} + +func TestSearchSnapshot_NilSnapshot(t *testing.T) { + t.Parallel() + cands := filefinder.SearchSnapshotForTest(filefinder.NewQueryPlanForTest("foo"), nil, 100) + if cands != nil { + t.Errorf("expected nil for nil snapshot, got %v", cands) + } +} + +func TestSearchSnapshot_EmptyQuery(t *testing.T) { + t.Parallel() + snap := filefinder.MakeTestSnapshot([]string{"foo.go"}) + cands := filefinder.SearchSnapshotForTest(filefinder.NewQueryPlanForTest(""), snap, 100) + if cands != nil { + t.Errorf("expected nil for empty query, got %v", cands) + } +} + +func TestSearchSnapshot_DeletedDocsExcluded(t *testing.T) { + t.Parallel() + idx := filefinder.NewIndex() + idx.Add("handler.go", 0) + idx.Remove("handler.go") + snap := idx.Snapshot() + cands := filefinder.SearchSnapshotForTest(filefinder.NewQueryPlanForTest("handler"), snap, 100) + for _, c := range cands { + if c.Path == "handler.go" { + t.Error("deleted doc should not appear in results") + } + } +} + +func TestSearchSnapshot_Limit(t *testing.T) { + t.Parallel() + paths := make([]string, 50) + for i := range paths { + paths[i] = "handler" + string(rune('a'+i%26)) + ".go" + } + snap := filefinder.MakeTestSnapshot(paths) + cands := filefinder.SearchSnapshotForTest(filefinder.NewQueryPlanForTest("handler"), snap, 3) + if len(cands) > 3 { + t.Errorf("expected at most 3 candidates, got %d", len(cands)) + } +} + +func TestIntersectSorted(t *testing.T) { + t.Parallel() + tests := []struct { + name string + a, b []uint32 + want []uint32 + }{ + {"both empty", nil, nil, nil}, + {"a empty", nil, []uint32{1, 2}, nil}, + {"b empty", []uint32{1, 2}, nil, nil}, + {"no overlap", []uint32{1, 3, 5}, []uint32{2, 4, 6}, nil}, + {"full overlap", []uint32{1, 2, 3}, []uint32{1, 2, 3}, []uint32{1, 2, 3}}, + {"partial overlap", []uint32{1, 2, 3, 5}, []uint32{2, 4, 5}, []uint32{2, 5}}, + {"single match", []uint32{1, 2, 3}, []uint32{2}, []uint32{2}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + got := filefinder.IntersectSortedForTest(tt.a, tt.b) + if len(tt.want) == 0 { + if len(got) != 0 { + t.Errorf("got %v, want empty/nil", got) + } + return + } + if !slices.Equal(got, tt.want) { + t.Errorf("got %v, want %v", got, tt.want) + } + }) + } +} + +func TestIntersectAll(t *testing.T) { + t.Parallel() + t.Run("empty", func(t *testing.T) { + t.Parallel() + if got := filefinder.IntersectAllForTest(nil); got != nil { + t.Errorf("got %v, want nil", got) + } + }) + t.Run("single", func(t *testing.T) { + t.Parallel() + if got := filefinder.IntersectAllForTest([][]uint32{{1, 2, 3}}); len(got) != 3 { + t.Fatalf("len = %d, want 3", len(got)) + } + }) + t.Run("multiple", func(t *testing.T) { + t.Parallel() + got := filefinder.IntersectAllForTest([][]uint32{{1, 2, 3, 4, 5}, {2, 3, 5}, {3, 5, 7}}) + if !slices.Equal(got, []uint32{3, 5}) { + t.Errorf("got %v, want [3 5]", got) + } + }) + t.Run("no overlap", func(t *testing.T) { + t.Parallel() + if got := filefinder.IntersectAllForTest([][]uint32{{1, 2}, {3, 4}}); got != nil { + t.Errorf("got %v, want nil", got) + } + }) +} + +func TestMergeAndScore_SortedDescending(t *testing.T) { + t.Parallel() + plan := filefinder.NewQueryPlanForTest("foo") + params := filefinder.DefaultScoreParamsForTest() + cands := []filefinder.CandidateForTest{ + {DocID: 0, Path: "a/b/c/d/e/foo", BaseOff: 10, BaseLen: 3, Depth: 5}, + {DocID: 1, Path: "src/foo", BaseOff: 4, BaseLen: 3, Depth: 1}, + {DocID: 2, Path: "foo", BaseOff: 0, BaseLen: 3, Depth: 0}, + } + results := filefinder.MergeAndScoreForTest(cands, plan, params, 10) + if len(results) == 0 { + t.Fatal("expected non-empty results") + } + for i := 1; i < len(results); i++ { + if results[i].Score > results[i-1].Score { + t.Errorf("results not sorted: [%d].Score=%f > [%d].Score=%f", + i, results[i].Score, i-1, results[i-1].Score) + } + } +} + +func TestMergeAndScore_TopKLimit(t *testing.T) { + t.Parallel() + plan := filefinder.NewQueryPlanForTest("f") + params := filefinder.DefaultScoreParamsForTest() + var cands []filefinder.CandidateForTest + for i := range 20 { + p := "f" + string(rune('a'+i)) + cands = append(cands, filefinder.CandidateForTest{DocID: uint32(i), Path: p, BaseOff: 0, BaseLen: len(p), Depth: 0}) //nolint:gosec // test index is tiny + } + if results := filefinder.MergeAndScoreForTest(cands, plan, params, 5); len(results) != 5 { + t.Errorf("expected 5 results, got %d", len(results)) + } +} + +func TestMergeAndScore_ZeroTopK(t *testing.T) { + t.Parallel() + plan := filefinder.NewQueryPlanForTest("foo") + cands := []filefinder.CandidateForTest{{DocID: 0, Path: "foo", BaseOff: 0, BaseLen: 3, Depth: 0}} + if results := filefinder.MergeAndScoreForTest(cands, plan, filefinder.DefaultScoreParamsForTest(), 0); len(results) != 0 { + t.Errorf("expected 0 results for topK=0, got %d", len(results)) + } +} + +func TestMergeAndScore_NoMatchCandidatesDropped(t *testing.T) { + t.Parallel() + plan := filefinder.NewQueryPlanForTest("xyz") + cands := []filefinder.CandidateForTest{ + {DocID: 0, Path: "abc", BaseOff: 0, BaseLen: 3, Depth: 0}, + {DocID: 1, Path: "def", BaseOff: 0, BaseLen: 3, Depth: 0}, + } + if results := filefinder.MergeAndScoreForTest(cands, plan, filefinder.DefaultScoreParamsForTest(), 10); len(results) != 0 { + t.Errorf("expected 0 results for non-matching candidates, got %d", len(results)) + } +} + +func TestMergeAndScore_IsDirFlag(t *testing.T) { + t.Parallel() + plan := filefinder.NewQueryPlanForTest("foo") + cands := []filefinder.CandidateForTest{ + {DocID: 0, Path: "foo", BaseOff: 0, BaseLen: 3, Depth: 0, Flags: uint16(filefinder.FlagDir)}, + } + results := filefinder.MergeAndScoreForTest(cands, plan, filefinder.DefaultScoreParamsForTest(), 10) + if len(results) != 1 { + t.Fatalf("expected 1 result, got %d", len(results)) + } + if !results[0].IsDir { + t.Error("expected IsDir=true for FlagDir candidate") + } +} + +func TestMergeAndScore_EmptyCandidates(t *testing.T) { + t.Parallel() + if results := filefinder.MergeAndScoreForTest(nil, filefinder.NewQueryPlanForTest("foo"), filefinder.DefaultScoreParamsForTest(), 10); len(results) != 0 { + t.Errorf("expected 0 results for nil candidates, got %d", len(results)) + } +} + +func TestSearchSnapshot_FuzzyFallbackEndToEnd(t *testing.T) { + t.Parallel() + snap := filefinder.MakeTestSnapshot([]string{"src/handler.go", "src/middleware.go", "pkg/config.go"}) + plan := filefinder.NewQueryPlanForTest("hndlr") + results := filefinder.MergeAndScoreForTest(filefinder.SearchSnapshotForTest(plan, snap, 100), plan, filefinder.DefaultScoreParamsForTest(), 10) + if len(results) == 0 { + t.Fatal("expected fuzzy fallback to produce scored results for 'hndlr'") + } + if results[0].Path != "src/handler.go" { + t.Errorf("expected top result 'src/handler.go', got %q", results[0].Path) + } +} diff --git a/agent/filefinder/text.go b/agent/filefinder/text.go new file mode 100644 index 0000000000..a41fd581da --- /dev/null +++ b/agent/filefinder/text.go @@ -0,0 +1,288 @@ +package filefinder + +import "slices" + +func toLowerASCII(b byte) byte { + if b >= 'A' && b <= 'Z' { + return b + ('a' - 'A') + } + return b +} + +func normalizeQuery(q string) string { + b := make([]byte, 0, len(q)) + prevSpace := true + for i := 0; i < len(q); i++ { + c := q[i] + if c == '\\' { + c = '/' + } + c = toLowerASCII(c) + if c == ' ' { + if prevSpace { + continue + } + prevSpace = true + } else { + prevSpace = false + } + b = append(b, c) + } + if len(b) > 0 && b[len(b)-1] == ' ' { + b = b[:len(b)-1] + } + return string(b) +} + +func normalizePathBytes(p []byte) []byte { + j := 0 + prevSlash := false + for i := 0; i < len(p); i++ { + c := p[i] + if c == '\\' { + c = '/' + } + c = toLowerASCII(c) + if c == '/' { + if prevSlash { + continue + } + prevSlash = true + } else { + prevSlash = false + } + p[j] = c + j++ + } + return p[:j] +} + +// extractTrigrams returns deduplicated, sorted trigrams (three-byte +// subsequences) from s. Trigrams are the primary index key: a +// document matches a query only if every query trigram appears in +// the document, giving O(1) candidate filtering per trigram. +func extractTrigrams(s []byte) []uint32 { + if len(s) < 3 { + return nil + } + seen := make(map[uint32]struct{}, len(s)) + for i := 0; i <= len(s)-3; i++ { + b0 := toLowerASCII(s[i]) + b1 := toLowerASCII(s[i+1]) + b2 := toLowerASCII(s[i+2]) + gram := uint32(b0)<<16 | uint32(b1)<<8 | uint32(b2) + seen[gram] = struct{}{} + } + result := make([]uint32, 0, len(seen)) + for g := range seen { + result = append(result, g) + } + slices.Sort(result) + return result +} + +func extractBasename(path []byte) (offset int, length int) { + end := len(path) + if end > 0 && path[end-1] == '/' { + end-- + } + if end == 0 { + return 0, 0 + } + i := end - 1 + for i >= 0 && path[i] != '/' { + i-- + } + start := i + 1 + return start, end - start +} + +func extractSegments(path []byte) [][]byte { + var segments [][]byte + start := 0 + for i := 0; i <= len(path); i++ { + if i == len(path) || path[i] == '/' { + if i > start { + segments = append(segments, path[start:i]) + } + start = i + 1 + } + } + return segments +} + +func prefix1(name []byte) byte { + if len(name) == 0 { + return 0 + } + return toLowerASCII(name[0]) +} + +func prefix2(name []byte) uint16 { + if len(name) == 0 { + return 0 + } + hi := uint16(toLowerASCII(name[0])) << 8 + if len(name) < 2 { + return hi + } + return hi | uint16(toLowerASCII(name[1])) +} + +// scoreParams controls the weights for each scoring signal. +type scoreParams struct { + BasenameMatch float32 + BasenamePrefix float32 + ExactSegment float32 + BoundaryHit float32 + ContiguousRun float32 + DirTokenHit float32 + DepthPenalty float32 + LengthPenalty float32 +} + +func defaultScoreParams() scoreParams { + return scoreParams{ + BasenameMatch: 6.0, + BasenamePrefix: 3.5, + ExactSegment: 2.5, + BoundaryHit: 1.8, + ContiguousRun: 1.2, + DirTokenHit: 0.4, + DepthPenalty: 0.08, + LengthPenalty: 0.01, + } +} + +func isSubsequence(haystack, needle []byte) bool { + if len(needle) == 0 { + return true + } + ni := 0 + for _, hb := range haystack { + if toLowerASCII(hb) == toLowerASCII(needle[ni]) { + ni++ + if ni == len(needle) { + return true + } + } + } + return false +} + +func longestContiguousMatch(haystack, needle []byte) int { + if len(needle) == 0 || len(haystack) == 0 { + return 0 + } + best := 0 + ni := 0 + run := 0 + for _, hb := range haystack { + if ni < len(needle) && toLowerASCII(hb) == toLowerASCII(needle[ni]) { + run++ + ni++ + if run > best { + best = run + } + } else { + run = 0 + ni = 0 + if ni < len(needle) && toLowerASCII(hb) == toLowerASCII(needle[ni]) { + run = 1 + ni = 1 + if run > best { + best = run + } + } + } + } + return best +} + +func isBoundary(b byte) bool { + return b == '/' || b == '.' || b == '_' || b == '-' +} + +func countBoundaryHits(path []byte, query []byte) int { + if len(query) == 0 || len(path) == 0 { + return 0 + } + hits := 0 + qi := 0 + for pi := 0; pi < len(path) && qi < len(query); pi++ { + atBoundary := pi == 0 || isBoundary(path[pi-1]) + if atBoundary && toLowerASCII(path[pi]) == toLowerASCII(query[qi]) { + hits++ + qi++ + } + } + return hits +} + +func equalFoldASCII(a, b []byte) bool { + if len(a) != len(b) { + return false + } + for i := range a { + if toLowerASCII(a[i]) != toLowerASCII(b[i]) { + return false + } + } + return true +} + +func hasPrefixFoldASCII(haystack, prefix []byte) bool { + if len(prefix) > len(haystack) { + return false + } + for i := range prefix { + if toLowerASCII(haystack[i]) != toLowerASCII(prefix[i]) { + return false + } + } + return true +} + +// scorePath computes a relevance score for a candidate path +// against a query. The score combines several signals: +// basename match, basename prefix, exact segment match, +// word-boundary hits, longest contiguous run, and penalties +// for depth and length. A return value of 0 means no match +// (the query is not a subsequence of the path). +func scorePath( + path []byte, + baseOff int, + baseLen int, + depth int, + query []byte, + queryTokens [][]byte, + params scoreParams, +) float32 { + if !isSubsequence(path, query) { + return 0 + } + var score float32 + basename := path[baseOff : baseOff+baseLen] + if isSubsequence(basename, query) { + score += params.BasenameMatch + } + if hasPrefixFoldASCII(basename, query) { + score += params.BasenamePrefix + } + segments := extractSegments(path) + for _, token := range queryTokens { + for _, seg := range segments { + if equalFoldASCII(seg, token) { + score += params.ExactSegment + break + } + } + } + bh := countBoundaryHits(path, query) + score += float32(bh) * params.BoundaryHit + lcm := longestContiguousMatch(path, query) + score += float32(lcm) * params.ContiguousRun + score -= float32(depth) * params.DepthPenalty + score -= float32(len(path)) * params.LengthPenalty + return score +} diff --git a/agent/filefinder/text_test.go b/agent/filefinder/text_test.go new file mode 100644 index 0000000000..f6cc460b3b --- /dev/null +++ b/agent/filefinder/text_test.go @@ -0,0 +1,388 @@ +package filefinder_test + +import ( + "slices" + "testing" + + "github.com/coder/coder/v2/agent/filefinder" +) + +func TestNormalizeQuery(t *testing.T) { + t.Parallel() + tests := []struct { + name string + input string + want string + }{ + {"empty", "", ""}, + {"leading and trailing spaces", " hello ", "hello"}, + {"multiple internal spaces", "foo bar baz", "foo bar baz"}, + {"uppercase to lower", "FooBar", "foobar"}, + {"backslash to slash", `foo\bar\baz`, "foo/bar/baz"}, + {"mixed case and spaces", " Hello World ", "hello world"}, + {"unicode passthrough", "héllo wörld", "héllo wörld"}, + {"only spaces", " ", ""}, + {"single char", "A", "a"}, + {"slashes preserved", "/foo/bar/", "/foo/bar/"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + got := filefinder.NormalizeQueryForTest(tt.input) + if got != tt.want { + t.Errorf("normalizeQuery(%q) = %q, want %q", tt.input, got, tt.want) + } + }) + } +} + +func TestExtractTrigrams(t *testing.T) { + t.Parallel() + tests := []struct { + name string + input string + want []uint32 + }{ + {"too short", "ab", nil}, + {"exactly three bytes", "abc", []uint32{uint32('a')<<16 | uint32('b')<<8 | uint32('c')}}, + {"case insensitive", "ABC", []uint32{uint32('a')<<16 | uint32('b')<<8 | uint32('c')}}, + {"deduplication", "aaaa", []uint32{uint32('a')<<16 | uint32('a')<<8 | uint32('a')}}, + {"four bytes produces two trigrams", "abcd", []uint32{ + uint32('a')<<16 | uint32('b')<<8 | uint32('c'), + uint32('b')<<16 | uint32('c')<<8 | uint32('d'), + }}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + got := filefinder.ExtractTrigramsForTest([]byte(tt.input)) + if !slices.Equal(got, tt.want) { + t.Errorf("extractTrigrams(%q) = %v, want %v", tt.input, got, tt.want) + } + }) + } +} + +func TestExtractBasename(t *testing.T) { + t.Parallel() + tests := []struct { + name string + path string + wantOff int + wantName string + }{ + {"full path", "/foo/bar/baz.go", 9, "baz.go"}, + {"bare filename", "baz.go", 0, "baz.go"}, + {"trailing slash", "/a/b/", 3, "b"}, + {"root slash", "/", 0, ""}, + {"empty", "", 0, ""}, + {"single dir with slash", "/foo", 1, "foo"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + off, length := filefinder.ExtractBasenameForTest([]byte(tt.path)) + if off != tt.wantOff { + t.Errorf("extractBasename(%q) offset = %d, want %d", tt.path, off, tt.wantOff) + } + gotName := string([]byte(tt.path)[off : off+length]) + if gotName != tt.wantName { + t.Errorf("extractBasename(%q) name = %q, want %q", tt.path, gotName, tt.wantName) + } + }) + } +} + +func TestExtractSegments(t *testing.T) { + t.Parallel() + tests := []struct { + name string + path string + want []string + }{ + {"absolute path", "/foo/bar/baz", []string{"foo", "bar", "baz"}}, + {"relative path", "foo/bar", []string{"foo", "bar"}}, + {"trailing slash", "/a/b/", []string{"a", "b"}}, + {"multiple slashes", "//a///b//", []string{"a", "b"}}, + {"empty", "", nil}, + {"single segment", "foo", []string{"foo"}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + got := filefinder.ExtractSegmentsForTest([]byte(tt.path)) + if len(got) != len(tt.want) { + t.Fatalf("extractSegments(%q) got %d segments, want %d", tt.path, len(got), len(tt.want)) + } + for i := range got { + if string(got[i]) != tt.want[i] { + t.Errorf("extractSegments(%q)[%d] = %q, want %q", tt.path, i, got[i], tt.want[i]) + } + } + }) + } +} + +func TestPrefix1(t *testing.T) { + t.Parallel() + tests := []struct { + name string + in string + want byte + }{ + {"lowercase", "foo", 'f'}, + {"uppercase", "Foo", 'f'}, + {"empty", "", 0}, + {"digit", "1abc", '1'}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + got := filefinder.Prefix1ForTest([]byte(tt.in)) + if got != tt.want { + t.Errorf("prefix1(%q) = %d (%c), want %d (%c)", tt.in, got, got, tt.want, tt.want) + } + }) + } +} + +func TestPrefix2(t *testing.T) { + t.Parallel() + tests := []struct { + name string + in string + want uint16 + }{ + {"two chars", "ab", uint16('a')<<8 | uint16('b')}, + {"uppercase", "AB", uint16('a')<<8 | uint16('b')}, + {"single char", "A", uint16('a') << 8}, + {"empty", "", 0}, + {"longer string", "Hello", uint16('h')<<8 | uint16('e')}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + got := filefinder.Prefix2ForTest([]byte(tt.in)) + if got != tt.want { + t.Errorf("prefix2(%q) = %d, want %d", tt.in, got, tt.want) + } + }) + } +} + +func TestNormalizePathBytes(t *testing.T) { + t.Parallel() + tests := []struct { + name string + input string + want string + }{ + {"backslash to slash", `C:\Users\test`, "c:/users/test"}, + {"collapse slashes", "//foo///bar//", "/foo/bar/"}, + {"lowercase", "FooBar", "foobar"}, + {"empty", "", ""}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + buf := []byte(tt.input) + got := string(filefinder.NormalizePathBytesForTest(buf)) + if got != tt.want { + t.Errorf("normalizePathBytes(%q) = %q, want %q", tt.input, got, tt.want) + } + }) + } +} + +func TestIsSubsequence(t *testing.T) { + t.Parallel() + tests := []struct { + name string + haystack string + needle string + want bool + }{ + {"empty needle", "anything", "", true}, + {"empty both", "", "", true}, + {"empty haystack", "", "a", false}, + {"exact match", "abc", "abc", true}, + {"scattered", "axbycz", "abc", true}, + {"prefix", "abcdef", "abc", true}, + {"suffix", "xyzabc", "abc", true}, + {"case insensitive", "AbCdEf", "ace", true}, + {"case insensitive reverse", "abcdef", "ACE", true}, + {"no match", "abcdef", "xyz", false}, + {"partial match", "abcdef", "abz", false}, + {"longer needle", "ab", "abc", false}, + {"single char match", "hello", "l", true}, + {"single char no match", "hello", "z", false}, + {"path like", "src/internal/foo.go", "sif", true}, + {"path like no match", "src/internal/foo.go", "zzz", false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + got := filefinder.IsSubsequenceForTest([]byte(tt.haystack), []byte(tt.needle)) + if got != tt.want { + t.Errorf("isSubsequence(%q, %q) = %v, want %v", tt.haystack, tt.needle, got, tt.want) + } + }) + } +} + +func TestLongestContiguousMatch(t *testing.T) { + t.Parallel() + tests := []struct { + name string + haystack string + needle string + want int + }{ + {"empty needle", "abc", "", 0}, + {"empty haystack", "", "abc", 0}, + {"full match", "abc", "abc", 3}, + {"prefix match", "abcdef", "abc", 3}, + {"middle match", "xxabcyy", "abc", 3}, + {"suffix match", "xxabc", "abc", 3}, + {"partial", "axbc", "abc", 1}, + {"scattered no contiguous", "axbxcx", "abc", 1}, + {"case insensitive", "ABCdef", "abc", 3}, + {"no match", "xyz", "abc", 0}, + {"single char", "abc", "b", 1}, + {"repeated", "aababc", "abc", 3}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + got := filefinder.LongestContiguousMatchForTest([]byte(tt.haystack), []byte(tt.needle)) + if got != tt.want { + t.Errorf("longestContiguousMatch(%q, %q) = %d, want %d", tt.haystack, tt.needle, got, tt.want) + } + }) + } +} + +func TestIsBoundary(t *testing.T) { + t.Parallel() + for _, b := range []byte{'/', '.', '_', '-'} { + if !filefinder.IsBoundaryForTest(b) { + t.Errorf("isBoundary(%q) = false, want true", b) + } + } + for _, b := range []byte{'a', 'Z', '0', ' ', '('} { + if filefinder.IsBoundaryForTest(b) { + t.Errorf("isBoundary(%q) = true, want false", b) + } + } +} + +func TestCountBoundaryHits(t *testing.T) { + t.Parallel() + tests := []struct { + name string + path string + query string + want int + }{ + {"start of string", "foo/bar", "f", 1}, + {"after slash", "foo/bar", "fb", 2}, + {"after dot", "foo.bar", "fb", 2}, + {"after underscore", "foo_bar", "fb", 2}, + {"no hits", "xxxx", "y", 0}, + {"empty query", "foo", "", 0}, + {"empty path", "", "f", 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + got := filefinder.CountBoundaryHitsForTest([]byte(tt.path), []byte(tt.query)) + if got != tt.want { + t.Errorf("countBoundaryHits(%q, %q) = %d, want %d", tt.path, tt.query, got, tt.want) + } + }) + } +} + +func TestScorePath_NoSubsequenceReturnsZero(t *testing.T) { + t.Parallel() + path := []byte("src/internal/handler.go") + query := []byte("zzz") + tokens := [][]byte{[]byte("zzz")} + params := filefinder.DefaultScoreParamsForTest() + s := filefinder.ScorePathForTest(path, 13, 10, 2, query, tokens, params) + if s != 0 { + t.Errorf("expected 0 for no subsequence match, got %f", s) + } +} + +func TestScorePath_ExactBasenameOverPartial(t *testing.T) { + t.Parallel() + params := filefinder.DefaultScoreParamsForTest() + query := []byte("main") + tokens := [][]byte{query} + pathExact := []byte("src/main") + scoreExact := filefinder.ScorePathForTest(pathExact, 4, 4, 1, query, tokens, params) + pathPartial := []byte("module/amazing") + scorePartial := filefinder.ScorePathForTest(pathPartial, 7, 7, 1, query, tokens, params) + if scoreExact <= scorePartial { + t.Errorf("exact basename (%f) should score higher than partial (%f)", scoreExact, scorePartial) + } +} + +func TestScorePath_BasenamePrefixOverScattered(t *testing.T) { + t.Parallel() + params := filefinder.DefaultScoreParamsForTest() + query := []byte("han") + tokens := [][]byte{query} + pathPrefix := []byte("src/handler.go") + scorePrefix := filefinder.ScorePathForTest(pathPrefix, 4, 10, 1, query, tokens, params) + pathScattered := []byte("has/another/thing") + scoreScattered := filefinder.ScorePathForTest(pathScattered, 12, 5, 2, query, tokens, params) + if scorePrefix <= scoreScattered { + t.Errorf("basename prefix (%f) should score higher than scattered (%f)", scorePrefix, scoreScattered) + } +} + +func TestScorePath_ShallowOverDeep(t *testing.T) { + t.Parallel() + params := filefinder.DefaultScoreParamsForTest() + query := []byte("foo") + tokens := [][]byte{query} + pathShallow := []byte("src/foo.go") + scoreShallow := filefinder.ScorePathForTest(pathShallow, 4, 6, 1, query, tokens, params) + pathDeep := []byte("a/b/c/d/e/foo.go") + scoreDeep := filefinder.ScorePathForTest(pathDeep, 10, 6, 5, query, tokens, params) + if scoreShallow <= scoreDeep { + t.Errorf("shallow path (%f) should score higher than deep (%f)", scoreShallow, scoreDeep) + } +} + +func TestScorePath_ShorterOverLongerSameMatch(t *testing.T) { + t.Parallel() + params := filefinder.DefaultScoreParamsForTest() + query := []byte("foo") + tokens := [][]byte{query} + pathShort := []byte("x/foo") + scoreShort := filefinder.ScorePathForTest(pathShort, 2, 3, 1, query, tokens, params) + pathLong := []byte("x/foo_extremely_long_suffix_name") + scoreLong := filefinder.ScorePathForTest(pathLong, 2, 29, 1, query, tokens, params) + if scoreShort <= scoreLong { + t.Errorf("shorter path (%f) should score higher than longer (%f)", scoreShort, scoreLong) + } +} + +func BenchmarkScorePath(b *testing.B) { + path := []byte("src/internal/coderd/database/queries/workspaces.sql") + query := []byte("workspace") + tokens := [][]byte{query} + params := filefinder.DefaultScoreParamsForTest() + baseOff, baseLen := filefinder.ExtractBasenameForTest(path) + s := filefinder.ScorePathForTest(path, baseOff, baseLen, 4, query, tokens, params) + if s == 0 { + b.Fatal("expected non-zero score for benchmark path") + } + b.ResetTimer() + for b.Loop() { + filefinder.ScorePathForTest(path, baseOff, baseLen, 4, query, tokens, params) + } +} diff --git a/agent/filefinder/watcher_fs.go b/agent/filefinder/watcher_fs.go new file mode 100644 index 0000000000..6d6a309d58 --- /dev/null +++ b/agent/filefinder/watcher_fs.go @@ -0,0 +1,210 @@ +package filefinder + +import ( + "context" + "os" + "path/filepath" + "sync" + "time" + + "github.com/fsnotify/fsnotify" + + "cdr.dev/slog/v3" +) + +// FSEvent represents a filesystem change event. +type FSEvent struct { + Op FSEventOp + Path string + IsDir bool +} + +// FSEventOp represents the type of filesystem operation. +type FSEventOp uint8 + +// Filesystem operations reported by the watcher. +const ( + OpCreate FSEventOp = iota + OpRemove + OpRename + OpModify +) + +var skipDirs = map[string]struct{}{ + ".git": {}, "node_modules": {}, ".hg": {}, ".svn": {}, + "__pycache__": {}, ".cache": {}, ".venv": {}, "vendor": {}, ".terraform": {}, +} + +type fsWatcher struct { + w *fsnotify.Watcher + root string + events chan []FSEvent + logger slog.Logger + mu sync.Mutex + closed bool + done chan struct{} +} + +func newFSWatcher(root string, logger slog.Logger) (*fsWatcher, error) { + w, err := fsnotify.NewWatcher() + if err != nil { + return nil, err + } + return &fsWatcher{ + w: w, + root: root, + events: make(chan []FSEvent, 64), + logger: logger, + done: make(chan struct{}), + }, nil +} + +func (fw *fsWatcher) Start(ctx context.Context) { + initEvents := fw.addRecursive(fw.root) + if len(initEvents) > 0 { + select { + case fw.events <- initEvents: + case <-ctx.Done(): + return + } + } + fw.logger.Debug(ctx, "fs watcher started", slog.F("root", fw.root)) + go fw.loop(ctx) +} +func (fw *fsWatcher) Events() <-chan []FSEvent { return fw.events } +func (fw *fsWatcher) Close() error { + fw.mu.Lock() + if fw.closed { + fw.mu.Unlock() + return nil + } + fw.closed = true + fw.mu.Unlock() + err := fw.w.Close() + <-fw.done + return err +} + +func (fw *fsWatcher) loop(ctx context.Context) { + defer close(fw.done) + const batchWindow = 50 * time.Millisecond + var ( + batch []FSEvent + seen = make(map[string]struct{}) + timer *time.Timer + timerC <-chan time.Time + ) + flush := func() { + if len(batch) == 0 { + return + } + select { + case fw.events <- batch: + default: + fw.logger.Warn(ctx, "fs watcher dropping batch", slog.F("count", len(batch))) + } + batch = nil + seen = make(map[string]struct{}) + if timer != nil { + timer.Stop() + } + timer = nil + timerC = nil + } + addToBatch := func(ev FSEvent) { + if _, dup := seen[ev.Path]; dup { + return + } + seen[ev.Path] = struct{}{} + batch = append(batch, ev) + if timer == nil { + timer = time.NewTimer(batchWindow) + timerC = timer.C + } + } + for { + select { + case <-ctx.Done(): + flush() + return + case ev, ok := <-fw.w.Events: + if !ok { + flush() + return + } + fsev := translateEvent(ev) + if fsev == nil { + continue + } + if fsev.IsDir && fsev.Op == OpCreate { + for _, s := range fw.addRecursive(fsev.Path) { + addToBatch(s) + } + } + addToBatch(*fsev) + case err, ok := <-fw.w.Errors: + if !ok { + flush() + return + } + fw.logger.Warn(ctx, "fsnotify watcher error", slog.Error(err)) + case <-timerC: + flush() + } + } +} + +func (fw *fsWatcher) addRecursive(dir string) []FSEvent { + var events []FSEvent + _ = filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { + if err != nil { + return nil //nolint:nilerr // best-effort + } + base := filepath.Base(path) + if _, skip := skipDirs[base]; skip && info.IsDir() { + return filepath.SkipDir + } + if info.IsDir() { + if addErr := fw.w.Add(path); addErr != nil { + fw.logger.Debug(context.Background(), "failed to add watch", + slog.F("path", path), slog.Error(addErr)) + } + if path != dir { + events = append(events, FSEvent{Op: OpCreate, Path: path, IsDir: true}) + } + return nil + } + events = append(events, FSEvent{Op: OpCreate, Path: path, IsDir: false}) + return nil + }) + return events +} + +func translateEvent(ev fsnotify.Event) *FSEvent { + var op FSEventOp + switch { + case ev.Op&fsnotify.Create != 0: + op = OpCreate + case ev.Op&fsnotify.Remove != 0: + op = OpRemove + case ev.Op&fsnotify.Rename != 0: + op = OpRename + case ev.Op&fsnotify.Write != 0: + op = OpModify + default: + return nil + } + isDir := false + if op == OpCreate || op == OpModify { + fi, err := os.Lstat(ev.Name) + if err == nil { + isDir = fi.IsDir() + } + } + if isDir { + if _, skip := skipDirs[filepath.Base(ev.Name)]; skip { + return nil + } + } + return &FSEvent{Op: op, Path: ev.Name, IsDir: isDir} +}