mirror of
https://github.com/coder/coder.git
synced 2026-10-31 11:02:50 +08:00
fix: stem chat search terms with the english text search config (#28319)
Chat message search used the `simple` text search config, which folds case but does no stemming, so `refactor` did not match messages containing `refactoring`. Message bodies now index and query with the `english` config, which stems both sides. Titles, PR titles, and all other filters are unchanged. Migration `000585` adds a `chat_messages.search_tsv_config` enum column recording which config produced each stored vector and stamps existing vectors `'simple'` (a column-only update; vectors and indexes are untouched, since rewriting `search_tsv` would block message writes on large tables). The dbpurge sweep gains a bounded `ReindexStaleChatMessagesSearchTsv` pass that rewrites stale vectors newest first and stops for the process lifetime once drained. Until a row is rewritten, `GetChats` matches it with the config that produced it, so pre-migration exact-form searches keep working during the drain. Vectors written by old binaries mid rolling upgrade cannot stamp the config, stay pending, and self-heal on the next upgraded sweep. Steady-state DB load is unchanged. The down migration resets `english` vectors to `NULL` so the parent version's existing sweep rewrites them with `simple`. Closes [CODAGT-867](https://linear.app/codercom/issue/CODAGT-867/backend-search-improvements-to-handle-partial-matches) <details> <summary>Decision log</summary> - **`english` stemming over prefix matching.** On a 300k-row corpus a stemmed lookup is a single GIN entry-tree probe, while `term:*` prefix matching unions posting lists of every lexeme sharing the prefix (up to a full scan for short prefixes), is asymmetric, and loses the `websearch_to_tsquery` phrase/OR/negation UX. `pg_trgm` rejected as a heavier index with different semantics. - **Scope to message bodies only** (review). Titles and PR titles keep their `simple` FTS expressions and indexes, identical to main. - **Config-versioned queue over migration-time reset** (review). An unqualified `UPDATE ... SET search_tsv = NULL` rewrites every indexed row in the migration transaction, and an old replica could backfill reset rows with `simple` vectors that permanently leave the `search_tsv IS NULL` queue. The config column makes staleness explicit and both problems disappear. - **Enum over `text`** (review) for the config column. - **Trade-offs.** English stopwords are dropped from message search (stopword-only queries match no messages); stemming is English-specific; titles do not stem; prefix matching remains out of scope. </details> --- 🤖 This PR was generated by Coder Agents on behalf of @f0ssel.
This commit is contained in:
Generated
+13
-4
@@ -352,6 +352,11 @@ CREATE TYPE chat_message_role AS ENUM (
|
||||
'tool'
|
||||
);
|
||||
|
||||
CREATE TYPE chat_message_search_tsv_config AS ENUM (
|
||||
'simple',
|
||||
'english'
|
||||
);
|
||||
|
||||
CREATE TYPE chat_message_visibility AS ENUM (
|
||||
'user',
|
||||
'model',
|
||||
@@ -1479,6 +1484,7 @@ BEGIN
|
||||
|
||||
cmp := NEW;
|
||||
cmp.search_tsv := OLD.search_tsv;
|
||||
cmp.search_tsv_config := OLD.search_tsv_config;
|
||||
IF OLD IS NOT DISTINCT FROM cmp THEN
|
||||
RETURN NEW;
|
||||
END IF;
|
||||
@@ -1496,7 +1502,7 @@ BEGIN
|
||||
END;
|
||||
$$;
|
||||
|
||||
COMMENT ON FUNCTION set_chat_message_revision_before() IS 'Component of chatd. Updates chat_snapshot_version when any fields of chat_messages change. Excludes changes to search_tsv as it is not relevant to chatd''s processing loop.';
|
||||
COMMENT ON FUNCTION set_chat_message_revision_before() IS 'Component of chatd. Updates chat_snapshot_version when any fields of chat_messages change. Excludes changes to search_tsv and search_tsv_config as they are not relevant to chatd''s processing loop.';
|
||||
|
||||
CREATE FUNCTION sync_chat_retry_state() RETURNS trigger
|
||||
LANGUAGE plpgsql
|
||||
@@ -1548,7 +1554,7 @@ BEGIN
|
||||
SELECT DISTINCT n.chat_id
|
||||
FROM chat_message_history_new_rows n
|
||||
JOIN chat_message_history_old_rows o ON o.id = n.id
|
||||
WHERE (to_jsonb(o) - 'search_tsv') IS DISTINCT FROM (to_jsonb(n) - 'search_tsv')
|
||||
WHERE (to_jsonb(o) - 'search_tsv' - 'search_tsv_config') IS DISTINCT FROM (to_jsonb(n) - 'search_tsv' - 'search_tsv_config')
|
||||
) AS affected
|
||||
WHERE c.id = affected.chat_id
|
||||
AND (
|
||||
@@ -1559,7 +1565,7 @@ BEGIN
|
||||
END;
|
||||
$$;
|
||||
|
||||
COMMENT ON FUNCTION update_chat_history_after_message_update() IS 'Component of chatd. Updates history_version and generation_attempt on chats when chat_messages is updated. Excludes changes to search_tsv.';
|
||||
COMMENT ON FUNCTION update_chat_history_after_message_update() IS 'Component of chatd. Updates history_version and generation_attempt on chats when chat_messages is updated. Excludes changes to search_tsv and search_tsv_config.';
|
||||
|
||||
CREATE TABLE ai_gateway_keys (
|
||||
id uuid NOT NULL,
|
||||
@@ -2053,13 +2059,16 @@ CREATE TABLE chat_messages (
|
||||
provider_response_id text,
|
||||
revision bigint NOT NULL,
|
||||
reasoning_effort chat_reasoning_effort,
|
||||
search_tsv tsvector
|
||||
search_tsv tsvector,
|
||||
search_tsv_config chat_message_search_tsv_config
|
||||
);
|
||||
|
||||
COMMENT ON COLUMN chat_messages.reasoning_effort IS 'Stores the selected effort for the turn triggered by this message.';
|
||||
|
||||
COMMENT ON COLUMN chat_messages.search_tsv IS 'Used for full text search. NULL initially, populated async via background job.';
|
||||
|
||||
COMMENT ON COLUMN chat_messages.search_tsv_config IS 'Text search config that produced search_tsv. NULL means the vector was written by a binary that predates this column; the dbpurge sweep re-vectorizes rows whose config is not english.';
|
||||
|
||||
CREATE SEQUENCE chat_messages_id_seq
|
||||
START WITH 1
|
||||
INCREMENT BY 1
|
||||
|
||||
Reference in New Issue
Block a user