[MM-24529] Preserve hyphenated compound words in Postgres search (#37360)

* [MM-24529] Preserve hyphenated compound words in Postgres search

Postgres full-text search stripped every hyphen to a space before
building a to_tsquery, so an unquoted search for "t-shirt" degraded
into a loose "t AND shirt" match instead of the compound-word match
the quoted version already got. Fix is Postgres-only; MySQL is gone
and Elasticsearch/Bleve use a different tokenizer/query mechanism
(verified empirically against a live ES instance - out of scope here).

- Remove "-" from the shared specialSearchChars strip list in
  store.go, and add neutralizeNonWordHyphens (utils.go), which keeps
  a hyphen only when flanked by a letter/digit on both sides,
  neutralizing malformed usage (leading/trailing/standalone/repeated)
  that would otherwise reach to_tsquery as a parse error.
- Wire the new helper into post_store.go's tsquery-building path.
- file_info_store.go keeps the old hyphen-to-space behavior: real
  filenames glue a hyphenated name directly to an extension (e.g.
  "photo-2024.jpg"), which Postgres's parser tokenizes as an opaque
  "host"-type token that a hyphen-preserving compound query can never
  match - verified against Postgres. Re-skip the two FileInfo dash
  acceptance tests with an accurate reason instead of "Not working".
- Re-enable the two previously-skipped Postgres post-search dash
  acceptance tests, and extend them with excluded-term, wildcard, and
  letters-digits-compound cases.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>

* Revert file_info_layer.go changes

Restore the two FileInfo dash acceptance tests to their original
Skip:true/SkipMessage state, matching master exactly. The underlying
fix (file_info_store.go keeps the old hyphen-to-space behavior since
Postgres tokenizes hyphenated-filename+extension as an opaque token)
is unaffected - this only reverts the test scaffolding tweak.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>

---------

Co-authored-by: Claude Sonnet 5 <noreply@anthropic.com>

server/channels/store/searchtest/post_layer.go

@@ -143,8 +143,7 @@ var searchPostStoreTests = []searchTest{
 	{
 		Name: "Should support terms with dash",
 		Fn:   testSupportTermsWithDash,
-		Tags: []string{EngineAll},
-		Skip: true,
+		Tags: []string{EnginePostgres},
 	},
 	{
 		Name: "Should support terms with underscore",
@@ -222,11 +221,9 @@ var searchPostStoreTests = []searchTest{
 		Tags: []string{EnginePostgres},
 	},
 	{
-		Name:        "Should be able to search terms with dashes",
-		Fn:          testSearchTermsWithDashes,
-		Tags:        []string{EngineAll},
-		Skip:        true,
-		SkipMessage: "Not working",
+		Name: "Should be able to search terms with dashes",
+		Fn:   testSearchTermsWithDashes,
+		Tags: []string{EnginePostgres},
 	},
 	{
 		Name: "Should be able to search terms with dots",
@@ -1987,6 +1984,36 @@ func testSearchTermsWithDashes(t *testing.T, th *SearchTestHelper) {
 		th.checkPostInSearchResults(t, p1.Id, results.Posts)
 		th.checkPostInSearchResults(t, p2.Id, results.Posts)
 	})
+
+	t.Run("Search for terms excluding a dashed term", func(t *testing.T) {
+		params := &model.SearchParams{Terms: "message", ExcludedTerms: "with-dash-term"}
+		results, err := th.Store.Post().SearchPostsForUser(th.Context, []*model.SearchParams{params}, th.User.Id, th.Team.Id, 0, 20)
+		require.NoError(t, err)
+
+		require.Len(t, results.Posts, 1)
+		th.checkPostInSearchResults(t, p2.Id, results.Posts)
+	})
+
+	t.Run("Search for a dashed term with a wildcard", func(t *testing.T) {
+		params := &model.SearchParams{Terms: "with-dash-term*"}
+		results, err := th.Store.Post().SearchPostsForUser(th.Context, []*model.SearchParams{params}, th.User.Id, th.Team.Id, 0, 20)
+		require.NoError(t, err)
+
+		require.Len(t, results.Posts, 1)
+		th.checkPostInSearchResults(t, p1.Id, results.Posts)
+	})
+
+	t.Run("Search for a letters-digits dashed term", func(t *testing.T) {
+		p3, err := th.createPost(th.User.Id, th.ChannelBasic.Id, "the code is FX-042", "", model.PostTypeDefault, 0, false)
+		require.NoError(t, err)
+
+		params := &model.SearchParams{Terms: "FX-042"}
+		results, err := th.Store.Post().SearchPostsForUser(th.Context, []*model.SearchParams{params}, th.User.Id, th.Team.Id, 0, 20)
+		require.NoError(t, err)
+
+		require.Len(t, results.Posts, 1)
+		th.checkPostInSearchResults(t, p3.Id, results.Posts)
+	})
 }
 
 func testSearchTermsWithDots(t *testing.T, th *SearchTestHelper) {

server/channels/store/sqlstore/file_info_store.go

@@ -630,6 +630,15 @@ func (fs SqlFileInfoStore) Search(rctx request.CTX, paramsList []*model.SearchPa
 			excludedTerms = strings.Replace(excludedTerms, c, " ", -1)
 		}
 
+		// Unlike post message search, filenames commonly have a hyphenated
+		// name glued directly to an extension (e.g. "photo-2024.jpg"), which
+		// PostgreSQL's parser classifies as an opaque "host"-type token that
+		// to_tsquery can never match against a hyphen-preserving compound
+		// query. So, unlike post_store.go, hyphens are still replaced with
+		// spaces here rather than preserved.
+		terms = strings.Replace(terms, "-", " ", -1)
+		excludedTerms = strings.Replace(excludedTerms, "-", " ", -1)
+
 		if terms == "" && excludedTerms == "" {
 			// we've already confirmed that we have a channel or user to search for
 		} else {

server/channels/store/sqlstore/post_store.go

@@ -2292,6 +2292,12 @@ func (s *SqlPostStore) search(teamId string, userId string, params *model.Search
 		// It also adds complexity as we would only need that index for CJK deployments.
 		baseQuery = s.buildCJKSearchClause(baseQuery, searchType, terms, excludedTerms, params.OrTerms)
 	} else {
+		// Preserve internal hyphens (e.g. "t-shirt") as compound-word searches,
+		// while neutralizing malformed hyphen usage that would otherwise be
+		// passed to to_tsquery.
+		terms = neutralizeNonWordHyphens(terms)
+		excludedTerms = neutralizeNonWordHyphens(excludedTerms)
+
 		// Parse text for wildcards
 		terms = wildCardRegex.ReplaceAllLiteralString(terms, ":* ")
 		excludedTerms = wildCardRegex.ReplaceAllLiteralString(excludedTerms, ":* ")

server/channels/store/sqlstore/store.go

@@ -394,7 +394,6 @@ var specialSearchChars = []string{
 	"<",
 	">",
 	"+",
-	"-",
 	"(",
 	")",
 	"~",

server/channels/store/sqlstore/utils.go

@@ -200,6 +200,35 @@ type rowScanner interface {
 	Err() error
 }
 
+// neutralizeNonWordHyphens replaces any '-' that isn't flanked by a word
+// rune (letter or digit) on both sides with a space, so malformed hyphen
+// usage (leading/trailing/standalone/repeated) can't reach to_tsquery, while
+// compound words like "t-shirt" are preserved.
+func neutralizeNonWordHyphens(s string) string {
+	if !strings.ContainsRune(s, '-') {
+		return s
+	}
+	runes := []rune(s)
+	for i, r := range runes {
+		if r != '-' {
+			continue
+		}
+		hasLeft := i > 0 && isWordRune(runes[i-1])
+		hasRight := i < len(runes)-1 && isWordRune(runes[i+1])
+		if !hasLeft || !hasRight {
+			runes[i] = ' '
+		}
+	}
+	return string(runes)
+}
+
+// isWordRune reports whether r can be part of a word for hyphen-flanking
+// purposes. Combining marks (e.g. a decomposed accent) count too, since they
+// attach to the preceding base letter rather than acting as a boundary.
+func isWordRune(r rune) bool {
+	return unicode.IsLetter(r) || unicode.IsDigit(r) || unicode.IsMark(r)
+}
+
 // scanRowsIntoMap scans SQL rows into a map, using a provided scanner function to extract key-value pairs
 func scanRowsIntoMap[K comparable, V any](rows rowScanner, scanner func(rows rowScanner) (K, V, error), defaults map[K]V) (map[K]V, error) {
 	results := make(map[K]V, len(defaults))

server/channels/store/sqlstore/utils_test.go

@@ -11,6 +11,32 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
+func TestNeutralizeNonWordHyphens(t *testing.T) {
+	testCases := []struct {
+		name     string
+		input    string
+		expected string
+	}{
+		{"compound word kept", "t-shirt", "t-shirt"},
+		{"multiple hyphens kept", "a-b-c", "a-b-c"},
+		{"digits kept", "covid-19", "covid-19"},
+		{"unicode letters kept", "café-au-lait", "café-au-lait"},
+		{"NFD combining mark before hyphen kept", "café-au-lait", "café-au-lait"},
+		{"leading hyphen neutralized", "-5", " 5"},
+		{"trailing hyphen neutralized", "foo-", "foo "},
+		{"standalone hyphen neutralized", "-", " "},
+		{"hyphen between spaces neutralized", "a - b", "a   b"},
+		{"repeated bare hyphens neutralized", "--", "  "},
+		{"empty string", "", ""},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			require.Equal(t, tc.expected, neutralizeNonWordHyphens(tc.input))
+		})
+	}
+}
+
 func TestChunkSlice(t *testing.T) {
 	if enableFullyParallelTests {
 		t.Parallel()