mirror of
https://github.com/coder/coder.git
synced 2026-09-24 15:04:27 +08:00
Kyle CarberryandCoder
68e4155fed
Adds an in-memory trigram-indexed file finder package at `agent/filefinder`, designed to power a future `FindFiles` HTTP handler on the WorkspaceAgent. ## What it does Fast fuzzy file search with VS Code-quality matching across millions of files. Sub-millisecond search latency at 100K files. ## Architecture - **Index**: append-only docs slice with trigram + prefix posting lists - **Snapshot**: lock-free reader view via frozen slice headers + shallow-copied deleted set - **Search pipeline**: trigram intersection → fuzzy fallback (prefix bucket + subsequence) → brute-force scan (capped at 5K docs) - **Scoring**: subsequence match, basename prefix, boundary hits, contiguous runs, depth/length penalties - **Engine**: multi-root with fsnotify watcher (50ms batch coalescing), atomic snapshot publishing ## Benchmarks (10K files) | Query Type | Latency | |---|---| | exact_basename (`handler.go`) | ~43µs | | short_query (`ha`) | ~7µs | | fuzzy_basename (`hndlr`) | ~50µs | | path_structured (`internal/handler`) | ~29µs | | multi_token (`api handler`) | ~15µs | ## File inventory (11 files, 3273 lines) | File | Lines | Purpose | |---|---|---| | `text.go` | 264 | Normalization, trigram extraction, scoring | | `delta.go` | 128 | Index, Snapshot, CRUD operations | | `query.go` | 272 | Query planning, search strategies, top-K merge | | `engine.go` | 323 | Multi-root engine, watcher integration | | `watcher_fs.go` | 201 | fsnotify wrapper with batch coalescing | | `*_test.go` | 2085 | Unit tests, integration tests, benchmarks | --------- Co-authored-by: Coder <coder@users.noreply.github.com>
121 lines
2.9 KiB
Go
121 lines
2.9 KiB
Go
package filefinder_test
|
|
|
|
import (
|
|
"testing"
|
|
|
|
"github.com/coder/coder/v2/agent/filefinder"
|
|
)
|
|
|
|
func TestIndex_AddAndLen(t *testing.T) {
|
|
t.Parallel()
|
|
idx := filefinder.NewIndex()
|
|
idx.Add("foo/bar.go", 0)
|
|
idx.Add("foo/baz.go", 0)
|
|
if idx.Len() != 2 {
|
|
t.Fatalf("expected 2, got %d", idx.Len())
|
|
}
|
|
}
|
|
|
|
func TestIndex_Has(t *testing.T) {
|
|
t.Parallel()
|
|
idx := filefinder.NewIndex()
|
|
idx.Add("foo/bar.go", 0)
|
|
if !idx.Has("foo/bar.go") {
|
|
t.Fatal("expected Has to return true")
|
|
}
|
|
if idx.Has("foo/missing.go") {
|
|
t.Fatal("expected Has to return false for missing path")
|
|
}
|
|
}
|
|
|
|
func TestIndex_Remove(t *testing.T) {
|
|
t.Parallel()
|
|
idx := filefinder.NewIndex()
|
|
idx.Add("foo/bar.go", 0)
|
|
if !idx.Remove("foo/bar.go") {
|
|
t.Fatal("expected Remove to return true")
|
|
}
|
|
if idx.Has("foo/bar.go") {
|
|
t.Fatal("expected Has to return false after Remove")
|
|
}
|
|
if idx.Len() != 0 {
|
|
t.Fatalf("expected Len 0 after Remove, got %d", idx.Len())
|
|
}
|
|
}
|
|
|
|
func TestIndex_AddOverwrite(t *testing.T) {
|
|
t.Parallel()
|
|
idx := filefinder.NewIndex()
|
|
idx.Add("foo/bar.go", uint16(filefinder.FlagFile))
|
|
idx.Add("foo/bar.go", uint16(filefinder.FlagDir)) // overwrite
|
|
if idx.Len() != 1 {
|
|
t.Fatalf("expected 1 after overwrite, got %d", idx.Len())
|
|
}
|
|
// The old entry should be tombstoned.
|
|
if !filefinder.IndexIsDeleted(idx, 0) {
|
|
t.Fatal("expected old entry to be deleted")
|
|
}
|
|
if filefinder.IndexIsDeleted(idx, 1) {
|
|
t.Fatal("expected new entry to be live")
|
|
}
|
|
}
|
|
|
|
func TestIndex_Snapshot(t *testing.T) {
|
|
t.Parallel()
|
|
idx := filefinder.NewIndex()
|
|
idx.Add("foo/bar.go", 0)
|
|
idx.Add("foo/baz.go", 0)
|
|
|
|
snap := idx.Snapshot()
|
|
if filefinder.SnapshotCount(snap) != 2 {
|
|
t.Fatalf("expected snapshot count 2, got %d", filefinder.SnapshotCount(snap))
|
|
}
|
|
|
|
// Adding more docs after snapshot doesn't affect it.
|
|
idx.Add("foo/qux.go", 0)
|
|
if filefinder.SnapshotCount(snap) != 2 {
|
|
t.Fatal("snapshot count should not change after new adds")
|
|
}
|
|
}
|
|
|
|
func TestIndex_TrigramIndex(t *testing.T) {
|
|
t.Parallel()
|
|
idx := filefinder.NewIndex()
|
|
idx.Add("handler.go", 0)
|
|
|
|
// "handler.go" should produce trigrams for "handler.go".
|
|
// Check that at least one trigram exists.
|
|
if filefinder.IndexByGramLen(idx) == 0 {
|
|
t.Fatal("expected non-empty trigram index")
|
|
}
|
|
}
|
|
|
|
func TestIndex_PrefixIndex(t *testing.T) {
|
|
t.Parallel()
|
|
idx := filefinder.NewIndex()
|
|
idx.Add("handler.go", 0)
|
|
|
|
// basename is "handler.go", first byte is 'h'
|
|
if filefinder.IndexByPrefix1Len(idx, 'h') == 0 {
|
|
t.Fatal("expected prefix1['h'] to be non-empty")
|
|
}
|
|
}
|
|
|
|
func TestIndex_RemoveNonexistent(t *testing.T) {
|
|
t.Parallel()
|
|
idx := filefinder.NewIndex()
|
|
if idx.Remove("nonexistent.go") {
|
|
t.Fatal("expected Remove to return false for missing path")
|
|
}
|
|
}
|
|
|
|
func TestIndex_PathNormalization(t *testing.T) {
|
|
t.Parallel()
|
|
idx := filefinder.NewIndex()
|
|
idx.Add("Foo/Bar.go", 0)
|
|
// Should be findable with lowercase.
|
|
if !idx.Has("foo/bar.go") {
|
|
t.Fatal("expected case-insensitive Has")
|
|
}
|
|
}
|