From 1addfd5e91630db21e8acee92f57f83670f32a30 Mon Sep 17 00:00:00 2001 From: Ben Schumacher Date: Thu, 9 Jul 2026 18:52:46 +0200 Subject: [PATCH] [MM-24529] Preserve hyphenated compound words in Postgres search (#37360) * [MM-24529] Preserve hyphenated compound words in Postgres search Postgres full-text search stripped every hyphen to a space before building a to_tsquery, so an unquoted search for "t-shirt" degraded into a loose "t AND shirt" match instead of the compound-word match the quoted version already got. Fix is Postgres-only; MySQL is gone and Elasticsearch/Bleve use a different tokenizer/query mechanism (verified empirically against a live ES instance - out of scope here). - Remove "-" from the shared specialSearchChars strip list in store.go, and add neutralizeNonWordHyphens (utils.go), which keeps a hyphen only when flanked by a letter/digit on both sides, neutralizing malformed usage (leading/trailing/standalone/repeated) that would otherwise reach to_tsquery as a parse error. - Wire the new helper into post_store.go's tsquery-building path. - file_info_store.go keeps the old hyphen-to-space behavior: real filenames glue a hyphenated name directly to an extension (e.g. "photo-2024.jpg"), which Postgres's parser tokenizes as an opaque "host"-type token that a hyphen-preserving compound query can never match - verified against Postgres. Re-skip the two FileInfo dash acceptance tests with an accurate reason instead of "Not working". - Re-enable the two previously-skipped Postgres post-search dash acceptance tests, and extend them with excluded-term, wildcard, and letters-digits-compound cases. Co-Authored-By: Claude Sonnet 5 * Revert file_info_layer.go changes Restore the two FileInfo dash acceptance tests to their original Skip:true/SkipMessage state, matching master exactly. The underlying fix (file_info_store.go keeps the old hyphen-to-space behavior since Postgres tokenizes hyphenated-filename+extension as an opaque token) is unaffected - this only reverts the test scaffolding tweak. Co-Authored-By: Claude Sonnet 5 --------- Co-authored-by: Claude Sonnet 5 --- .../channels/store/searchtest/post_layer.go | 41 +++++++++++++++---- .../store/sqlstore/file_info_store.go | 9 ++++ server/channels/store/sqlstore/post_store.go | 6 +++ server/channels/store/sqlstore/store.go | 1 - server/channels/store/sqlstore/utils.go | 29 +++++++++++++ server/channels/store/sqlstore/utils_test.go | 26 ++++++++++++ 6 files changed, 104 insertions(+), 8 deletions(-) diff --git a/server/channels/store/searchtest/post_layer.go b/server/channels/store/searchtest/post_layer.go index b29b4fac733..20c5a2e0e9b 100644 --- a/server/channels/store/searchtest/post_layer.go +++ b/server/channels/store/searchtest/post_layer.go @@ -143,8 +143,7 @@ var searchPostStoreTests = []searchTest{ { Name: "Should support terms with dash", Fn: testSupportTermsWithDash, - Tags: []string{EngineAll}, - Skip: true, + Tags: []string{EnginePostgres}, }, { Name: "Should support terms with underscore", @@ -222,11 +221,9 @@ var searchPostStoreTests = []searchTest{ Tags: []string{EnginePostgres}, }, { - Name: "Should be able to search terms with dashes", - Fn: testSearchTermsWithDashes, - Tags: []string{EngineAll}, - Skip: true, - SkipMessage: "Not working", + Name: "Should be able to search terms with dashes", + Fn: testSearchTermsWithDashes, + Tags: []string{EnginePostgres}, }, { Name: "Should be able to search terms with dots", @@ -1987,6 +1984,36 @@ func testSearchTermsWithDashes(t *testing.T, th *SearchTestHelper) { th.checkPostInSearchResults(t, p1.Id, results.Posts) th.checkPostInSearchResults(t, p2.Id, results.Posts) }) + + t.Run("Search for terms excluding a dashed term", func(t *testing.T) { + params := &model.SearchParams{Terms: "message", ExcludedTerms: "with-dash-term"} + results, err := th.Store.Post().SearchPostsForUser(th.Context, []*model.SearchParams{params}, th.User.Id, th.Team.Id, 0, 20) + require.NoError(t, err) + + require.Len(t, results.Posts, 1) + th.checkPostInSearchResults(t, p2.Id, results.Posts) + }) + + t.Run("Search for a dashed term with a wildcard", func(t *testing.T) { + params := &model.SearchParams{Terms: "with-dash-term*"} + results, err := th.Store.Post().SearchPostsForUser(th.Context, []*model.SearchParams{params}, th.User.Id, th.Team.Id, 0, 20) + require.NoError(t, err) + + require.Len(t, results.Posts, 1) + th.checkPostInSearchResults(t, p1.Id, results.Posts) + }) + + t.Run("Search for a letters-digits dashed term", func(t *testing.T) { + p3, err := th.createPost(th.User.Id, th.ChannelBasic.Id, "the code is FX-042", "", model.PostTypeDefault, 0, false) + require.NoError(t, err) + + params := &model.SearchParams{Terms: "FX-042"} + results, err := th.Store.Post().SearchPostsForUser(th.Context, []*model.SearchParams{params}, th.User.Id, th.Team.Id, 0, 20) + require.NoError(t, err) + + require.Len(t, results.Posts, 1) + th.checkPostInSearchResults(t, p3.Id, results.Posts) + }) } func testSearchTermsWithDots(t *testing.T, th *SearchTestHelper) { diff --git a/server/channels/store/sqlstore/file_info_store.go b/server/channels/store/sqlstore/file_info_store.go index 3f8318dbe19..326bd9edc94 100644 --- a/server/channels/store/sqlstore/file_info_store.go +++ b/server/channels/store/sqlstore/file_info_store.go @@ -630,6 +630,15 @@ func (fs SqlFileInfoStore) Search(rctx request.CTX, paramsList []*model.SearchPa excludedTerms = strings.Replace(excludedTerms, c, " ", -1) } + // Unlike post message search, filenames commonly have a hyphenated + // name glued directly to an extension (e.g. "photo-2024.jpg"), which + // PostgreSQL's parser classifies as an opaque "host"-type token that + // to_tsquery can never match against a hyphen-preserving compound + // query. So, unlike post_store.go, hyphens are still replaced with + // spaces here rather than preserved. + terms = strings.Replace(terms, "-", " ", -1) + excludedTerms = strings.Replace(excludedTerms, "-", " ", -1) + if terms == "" && excludedTerms == "" { // we've already confirmed that we have a channel or user to search for } else { diff --git a/server/channels/store/sqlstore/post_store.go b/server/channels/store/sqlstore/post_store.go index f5979c0aecc..4fc48f670aa 100644 --- a/server/channels/store/sqlstore/post_store.go +++ b/server/channels/store/sqlstore/post_store.go @@ -2292,6 +2292,12 @@ func (s *SqlPostStore) search(teamId string, userId string, params *model.Search // It also adds complexity as we would only need that index for CJK deployments. baseQuery = s.buildCJKSearchClause(baseQuery, searchType, terms, excludedTerms, params.OrTerms) } else { + // Preserve internal hyphens (e.g. "t-shirt") as compound-word searches, + // while neutralizing malformed hyphen usage that would otherwise be + // passed to to_tsquery. + terms = neutralizeNonWordHyphens(terms) + excludedTerms = neutralizeNonWordHyphens(excludedTerms) + // Parse text for wildcards terms = wildCardRegex.ReplaceAllLiteralString(terms, ":* ") excludedTerms = wildCardRegex.ReplaceAllLiteralString(excludedTerms, ":* ") diff --git a/server/channels/store/sqlstore/store.go b/server/channels/store/sqlstore/store.go index 8e5935b31ea..b31e54478bc 100644 --- a/server/channels/store/sqlstore/store.go +++ b/server/channels/store/sqlstore/store.go @@ -394,7 +394,6 @@ var specialSearchChars = []string{ "<", ">", "+", - "-", "(", ")", "~", diff --git a/server/channels/store/sqlstore/utils.go b/server/channels/store/sqlstore/utils.go index 4bac11d2b59..aa489dd0e67 100644 --- a/server/channels/store/sqlstore/utils.go +++ b/server/channels/store/sqlstore/utils.go @@ -200,6 +200,35 @@ type rowScanner interface { Err() error } +// neutralizeNonWordHyphens replaces any '-' that isn't flanked by a word +// rune (letter or digit) on both sides with a space, so malformed hyphen +// usage (leading/trailing/standalone/repeated) can't reach to_tsquery, while +// compound words like "t-shirt" are preserved. +func neutralizeNonWordHyphens(s string) string { + if !strings.ContainsRune(s, '-') { + return s + } + runes := []rune(s) + for i, r := range runes { + if r != '-' { + continue + } + hasLeft := i > 0 && isWordRune(runes[i-1]) + hasRight := i < len(runes)-1 && isWordRune(runes[i+1]) + if !hasLeft || !hasRight { + runes[i] = ' ' + } + } + return string(runes) +} + +// isWordRune reports whether r can be part of a word for hyphen-flanking +// purposes. Combining marks (e.g. a decomposed accent) count too, since they +// attach to the preceding base letter rather than acting as a boundary. +func isWordRune(r rune) bool { + return unicode.IsLetter(r) || unicode.IsDigit(r) || unicode.IsMark(r) +} + // scanRowsIntoMap scans SQL rows into a map, using a provided scanner function to extract key-value pairs func scanRowsIntoMap[K comparable, V any](rows rowScanner, scanner func(rows rowScanner) (K, V, error), defaults map[K]V) (map[K]V, error) { results := make(map[K]V, len(defaults)) diff --git a/server/channels/store/sqlstore/utils_test.go b/server/channels/store/sqlstore/utils_test.go index 44e39c47860..7fe2e11eb43 100644 --- a/server/channels/store/sqlstore/utils_test.go +++ b/server/channels/store/sqlstore/utils_test.go @@ -11,6 +11,32 @@ import ( "github.com/stretchr/testify/require" ) +func TestNeutralizeNonWordHyphens(t *testing.T) { + testCases := []struct { + name string + input string + expected string + }{ + {"compound word kept", "t-shirt", "t-shirt"}, + {"multiple hyphens kept", "a-b-c", "a-b-c"}, + {"digits kept", "covid-19", "covid-19"}, + {"unicode letters kept", "café-au-lait", "café-au-lait"}, + {"NFD combining mark before hyphen kept", "café-au-lait", "café-au-lait"}, + {"leading hyphen neutralized", "-5", " 5"}, + {"trailing hyphen neutralized", "foo-", "foo "}, + {"standalone hyphen neutralized", "-", " "}, + {"hyphen between spaces neutralized", "a - b", "a b"}, + {"repeated bare hyphens neutralized", "--", " "}, + {"empty string", "", ""}, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + require.Equal(t, tc.expected, neutralizeNonWordHyphens(tc.input)) + }) + } +} + func TestChunkSlice(t *testing.T) { if enableFullyParallelTests { t.Parallel()