diff --git a/docs/main/administration-guide/configure/environment-configuration-settings.mdx b/docs/main/administration-guide/configure/environment-configuration-settings.mdx index e0b7f4952d6..b4fd9f80770 100644 --- a/docs/main/administration-guide/configure/environment-configuration-settings.mdx +++ b/docs/main/administration-guide/configure/environment-configuration-settings.mdx @@ -1446,6 +1446,23 @@ This setting has no effect when [Compliance Mode](/administration-guide/configur +### Enable file indexing + + ++++ + + + + + + +

Configure whether Mattermost indexes file attachments in Elasticsearch or AWS OpenSearch.

  • true: (Default) New and updated file attachments are indexed, and bulk indexing jobs include files.
  • false: New or updated file attachments aren't indexed, and bulk indexing jobs skip files. File search continues to return previously indexed files, and files deleted from Mattermost are still removed from the index.
  • System Config path: Environment > Elasticsearch
  • config.json setting: ElasticsearchSettings > EnableFileIndexing > true
  • Environment variable: MM_ELASTICSEARCHSETTINGS_ENABLEFILEINDEXING
+ +Files uploaded while this setting is **false** don't appear in file search results. After setting it back to **true**, run a bulk index from **System Console > Environment > Elasticsearch** to make those files searchable. + ### Post index replicas diff --git a/server/channels/app/job.go b/server/channels/app/job.go index 067ff1ed9e1..f505323518b 100644 --- a/server/channels/app/job.go +++ b/server/channels/app/job.go @@ -6,6 +6,7 @@ package app import ( "errors" "net/http" + "strconv" "strings" "github.com/mattermost/mattermost/server/public/model" @@ -91,6 +92,14 @@ func (a *App) CreateJob(rctx request.CTX, job *model.Job) (*model.Job, *model.Ap case model.JobTypeAccessControlTeamSync: // Route team ABAC jobs to specialized deduplication handler return a.CreateAccessControlTeamSyncJob(rctx, job.Data) + case model.JobTypeElasticsearchPostIndexing: + if _, ok := job.Data["index_files"]; !ok { + if job.Data == nil { + job.Data = model.StringMap{} + } + job.Data["index_files"] = strconv.FormatBool(*a.Config().ElasticsearchSettings.EnableFileIndexing) + } + return a.Srv().Jobs.CreateJob(rctx, job.Type, job.Data) default: return a.Srv().Jobs.CreateJob(rctx, job.Type, job.Data) } diff --git a/server/channels/app/job_test.go b/server/channels/app/job_test.go index 7a37ebfc56e..bda078710f7 100644 --- a/server/channels/app/job_test.go +++ b/server/channels/app/job_test.go @@ -10,6 +10,8 @@ import ( "github.com/stretchr/testify/require" "github.com/mattermost/mattermost/server/public/model" + "github.com/mattermost/mattermost/server/public/shared/mlog" + "github.com/mattermost/mattermost/server/v8/channels/jobs" ) func TestGetJob(t *testing.T) { @@ -771,6 +773,45 @@ func TestSessionHasPermissionToReadJob(t *testing.T) { } } +func TestCreateElasticsearchPostIndexingJob(t *testing.T) { + mainHelper.Parallel(t) + th := Setup(t) + + worker := jobs.NewSimpleWorker(model.JobTypeElasticsearchPostIndexing, th.Server.Jobs, func(mlog.LoggerIFace, *model.Job) error { return nil }, func(*model.Config) bool { return false }) + th.Server.Jobs.RegisterJobType(model.JobTypeElasticsearchPostIndexing, worker, nil) + + createJob := func(t *testing.T, data model.StringMap) *model.Job { + job, appErr := th.App.CreateJob(th.Context, &model.Job{Type: model.JobTypeElasticsearchPostIndexing, Data: data}) + require.Nil(t, appErr) + t.Cleanup(func() { + _, err := th.App.Srv().Store().Job().Delete(job.Id) + require.NoError(t, err) + }) + return job + } + + t.Run("indexes files by default when file indexing is enabled", func(t *testing.T) { + th.App.UpdateConfig(func(cfg *model.Config) { *cfg.ElasticsearchSettings.EnableFileIndexing = true }) + + job := createJob(t, nil) + assert.Equal(t, "true", job.Data["index_files"]) + }) + + t.Run("skips files by default when file indexing is disabled", func(t *testing.T) { + th.App.UpdateConfig(func(cfg *model.Config) { *cfg.ElasticsearchSettings.EnableFileIndexing = false }) + + job := createJob(t, nil) + assert.Equal(t, "false", job.Data["index_files"]) + }) + + t.Run("keeps an explicit index_files value", func(t *testing.T) { + th.App.UpdateConfig(func(cfg *model.Config) { *cfg.ElasticsearchSettings.EnableFileIndexing = false }) + + job := createJob(t, model.StringMap{"index_files": "true"}) + assert.Equal(t, "true", job.Data["index_files"]) + }) +} + func TestGetJobByType(t *testing.T) { mainHelper.Parallel(t) th := Setup(t) diff --git a/server/channels/app/platform/testdata/support_packet/sanitized_config.json b/server/channels/app/platform/testdata/support_packet/sanitized_config.json index 34074e85f11..6ead8054954 100644 --- a/server/channels/app/platform/testdata/support_packet/sanitized_config.json +++ b/server/channels/app/platform/testdata/support_packet/sanitized_config.json @@ -569,7 +569,8 @@ "ClientKey": null, "Trace": null, "IgnoredPurgeIndexes": null, - "EnableSearchPublicChannelsWithoutMembership": null + "EnableSearchPublicChannelsWithoutMembership": null, + "EnableFileIndexing": null }, "DataRetentionSettings": { "EnableMessageDeletion": null, diff --git a/server/channels/store/searchlayer/file_info_layer.go b/server/channels/store/searchlayer/file_info_layer.go index 8468d0fd571..fe76eeb7a10 100644 --- a/server/channels/store/searchlayer/file_info_layer.go +++ b/server/channels/store/searchlayer/file_info_layer.go @@ -19,6 +19,10 @@ type SearchFileInfoStore struct { } func (s SearchFileInfoStore) indexFile(rctx request.CTX, file *model.FileInfo) { + if !*s.rootStore.getConfig().ElasticsearchSettings.EnableFileIndexing { + return + } + for _, engine := range s.rootStore.searchEngine.GetActiveEngines() { if engine.IsIndexingEnabled() { runIndexFn(rctx, engine, func(engineCopy searchengine.SearchEngineInterface) { diff --git a/server/channels/store/searchlayer/file_info_layer_test.go b/server/channels/store/searchlayer/file_info_layer_test.go new file mode 100644 index 00000000000..36528264555 --- /dev/null +++ b/server/channels/store/searchlayer/file_info_layer_test.go @@ -0,0 +1,71 @@ +// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. +// See LICENSE.txt for license information. + +package searchlayer_test + +import ( + "testing" + + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + + "github.com/mattermost/mattermost/server/public/model" + "github.com/mattermost/mattermost/server/public/shared/request" + "github.com/mattermost/mattermost/server/v8/channels/store/searchlayer" + storemocks "github.com/mattermost/mattermost/server/v8/channels/store/storetest/mocks" + "github.com/mattermost/mattermost/server/v8/platform/services/searchengine" + "github.com/mattermost/mattermost/server/v8/platform/services/searchengine/mocks" +) + +func TestFileIndexing(t *testing.T) { + setup := func(t *testing.T, enableFileIndexing bool) (*searchlayer.SearchStore, *storemocks.FileInfoStore, *mocks.SearchEngineInterface) { + cfg := &model.Config{} + cfg.SetDefaults() + cfg.ElasticsearchSettings.EnableFileIndexing = model.NewPointer(enableFileIndexing) + + fileInfoStore := &storemocks.FileInfoStore{} + baseStore := &storemocks.Store{} + baseStore.On("FileInfo").Return(fileInfoStore) + baseStore.On("Channel").Return(&storemocks.ChannelStore{}) + baseStore.On("Post").Return(&storemocks.PostStore{}) + baseStore.On("Team").Return(&storemocks.TeamStore{}) + baseStore.On("User").Return(&storemocks.UserStore{}) + + es := &mocks.SearchEngineInterface{} + es.On("IsActive").Return(true) + es.On("IsHealthy").Return(true) + es.On("IsIndexingEnabled").Return(true).Maybe() + es.On("IsSearchEnabled").Return(true).Maybe() + es.On("IsIndexingSync").Return(true).Maybe() + es.On("RefreshIndexes", mock.Anything).Return(nil).Maybe() + es.On("GetName").Return("mock").Maybe() + + broker := searchengine.NewBroker(cfg) + broker.ElasticsearchEngine = es + + return searchlayer.NewSearchLayer(baseStore, broker, cfg), fileInfoStore, es + } + + file := &model.FileInfo{Id: model.NewId(), PostId: model.NewId(), ChannelId: model.NewId()} + + t.Run("indexes saved files when file indexing is enabled", func(t *testing.T) { + layer, fileInfoStore, es := setup(t, true) + fileInfoStore.On("Save", mock.Anything, file).Return(file, nil) + es.On("IndexFile", file, file.ChannelId).Return(nil) + + _, err := layer.FileInfo().Save(request.TestContext(t), file) + require.NoError(t, err) + + es.AssertCalled(t, "IndexFile", file, file.ChannelId) + }) + + t.Run("does not index saved files when file indexing is disabled", func(t *testing.T) { + layer, fileInfoStore, es := setup(t, false) + fileInfoStore.On("Save", mock.Anything, file).Return(file, nil) + + _, err := layer.FileInfo().Save(request.TestContext(t), file) + require.NoError(t, err) + + es.AssertNotCalled(t, "IndexFile", mock.Anything, mock.Anything) + }) +} diff --git a/server/enterprise/elasticsearch/elasticsearch/aggregation_job.go b/server/enterprise/elasticsearch/elasticsearch/aggregation_job.go index 31c73f5198e..5677d358ee1 100644 --- a/server/enterprise/elasticsearch/elasticsearch/aggregation_job.go +++ b/server/enterprise/elasticsearch/elasticsearch/aggregation_job.go @@ -224,8 +224,9 @@ func (worker *ElasticsearchAggregatorWorker) DoJob(job *model.Job) { rctx, model.JobTypeElasticsearchPostIndexing, map[string]string{ - "start_time": strconv.FormatInt(reindexingStartDate.UnixNano()/int64(time.Millisecond), 10), - "end_time": strconv.FormatInt(reindexingEndDate.UnixNano()/int64(time.Millisecond), 10), + "start_time": strconv.FormatInt(reindexingStartDate.UnixNano()/int64(time.Millisecond), 10), + "end_time": strconv.FormatInt(reindexingEndDate.UnixNano()/int64(time.Millisecond), 10), + "index_files": strconv.FormatBool(*worker.jobServer.Config().ElasticsearchSettings.EnableFileIndexing), }, ); appErr != nil { logger.Error("Worker: Failed to create indexing job.", mlog.Err(appErr)) diff --git a/server/enterprise/elasticsearch/opensearch/aggregation_job.go b/server/enterprise/elasticsearch/opensearch/aggregation_job.go index 981e2209ba2..be1e39105b8 100644 --- a/server/enterprise/elasticsearch/opensearch/aggregation_job.go +++ b/server/enterprise/elasticsearch/opensearch/aggregation_job.go @@ -223,8 +223,9 @@ func (worker *OpensearchAggregatorWorker) DoJob(job *model.Job) { rctx, model.JobTypeElasticsearchPostIndexing, map[string]string{ - "start_time": strconv.FormatInt(reindexingStartDate.UnixNano()/int64(time.Millisecond), 10), - "end_time": strconv.FormatInt(reindexingEndDate.UnixNano()/int64(time.Millisecond), 10), + "start_time": strconv.FormatInt(reindexingStartDate.UnixNano()/int64(time.Millisecond), 10), + "end_time": strconv.FormatInt(reindexingEndDate.UnixNano()/int64(time.Millisecond), 10), + "index_files": strconv.FormatBool(*worker.jobServer.Config().ElasticsearchSettings.EnableFileIndexing), }, ); appErr != nil { logger.Error("Worker: Failed to create indexing job.", mlog.Err(appErr)) diff --git a/server/public/model/config.go b/server/public/model/config.go index ec115d2c455..c8e61184b98 100644 --- a/server/public/model/config.go +++ b/server/public/model/config.go @@ -3302,6 +3302,7 @@ type ElasticsearchSettings struct { Trace *string `access:"environment_elasticsearch,write_restrictable,cloud_restrictable"` IgnoredPurgeIndexes *string `access:"environment_elasticsearch,write_restrictable,cloud_restrictable"` // telemetry: none EnableSearchPublicChannelsWithoutMembership *bool `access:"environment_elasticsearch,write_restrictable,cloud_restrictable"` + EnableFileIndexing *bool `access:"environment_elasticsearch,write_restrictable,cloud_restrictable"` } func (s *ElasticsearchSettings) SetDefaults() { @@ -3420,6 +3421,10 @@ func (s *ElasticsearchSettings) SetDefaults() { if s.EnableSearchPublicChannelsWithoutMembership == nil { s.EnableSearchPublicChannelsWithoutMembership = new(true) } + + if s.EnableFileIndexing == nil { + s.EnableFileIndexing = new(true) + } } type DataRetentionSettings struct { diff --git a/server/public/model/config_test.go b/server/public/model/config_test.go index 8a661606488..42caf20edc6 100644 --- a/server/public/model/config_test.go +++ b/server/public/model/config_test.go @@ -5168,6 +5168,18 @@ func TestElasticsearchSettingsSetDefaults(t *testing.T) { s.SetDefaults() require.False(t, *s.EnableSearchPublicChannelsWithoutMembership) }) + + t.Run("EnableFileIndexing defaults to true when nil", func(t *testing.T) { + s := ElasticsearchSettings{} + s.SetDefaults() + require.True(t, *s.EnableFileIndexing) + }) + + t.Run("EnableFileIndexing preserves explicit false", func(t *testing.T) { + s := ElasticsearchSettings{EnableFileIndexing: NewPointer(false)} + s.SetDefaults() + require.False(t, *s.EnableFileIndexing) + }) } func TestElasticsearchSettingsIsValid(t *testing.T) { diff --git a/webapp/channels/src/components/admin_console/__snapshots__/elasticsearch_settings.test.tsx.snap b/webapp/channels/src/components/admin_console/__snapshots__/elasticsearch_settings.test.tsx.snap index bd20ed8cd84..17cbe70655e 100644 --- a/webapp/channels/src/components/admin_console/__snapshots__/elasticsearch_settings.test.tsx.snap +++ b/webapp/channels/src/components/admin_console/__snapshots__/elasticsearch_settings.test.tsx.snap @@ -733,6 +733,54 @@ exports[`components/ElasticSearchSettings should match snapshot, disabled 1`] = +
+ + Enable file indexing: + +
+ + +
+ When false, new or updated file attachments are not indexed. File search continues to return previously indexed files, but files uploaded while this setting is false are not searchable until a bulk index is run after re-enabling it. +
+
+
+
+ + Enable file indexing: + +
+ + +
+ When false, new or updated file attachments are not indexed. File search continues to return previously indexed files, but files uploaded while this setting is false are not searchable until a bulk index is run after re-enabling it. +
+
+
+
+ + Enable file indexing: + +
+ + +
+ When false, new or updated file attachments are not indexed. File search continues to return previously indexed files, but files uploaded while this setting is false are not searchable until a bulk index is run after re-enabling it. +
+
+
{ EnableSearching: false, EnableAutocomplete: false, EnableSearchPublicChannelsWithoutMembership: false, + EnableFileIndexing: true, IgnoredPurgeIndexes: '', }, }; @@ -62,6 +63,7 @@ describe('components/ElasticSearchSettings', () => { EnableSearching: false, EnableAutocomplete: false, EnableSearchPublicChannelsWithoutMembership: false, + EnableFileIndexing: true, IgnoredPurgeIndexes: '', }, }; @@ -90,6 +92,7 @@ describe('components/ElasticSearchSettings', () => { EnableSearching: false, EnableAutocomplete: false, EnableSearchPublicChannelsWithoutMembership: false, + EnableFileIndexing: true, IgnoredPurgeIndexes: '', }, }; @@ -118,6 +121,7 @@ describe('components/ElasticSearchSettings', () => { EnableSearching: false, EnableAutocomplete: false, EnableSearchPublicChannelsWithoutMembership: false, + EnableFileIndexing: true, IgnoredPurgeIndexes: '', }, }; @@ -146,6 +150,7 @@ describe('components/ElasticSearchSettings', () => { EnableSearching: false, EnableAutocomplete: false, EnableSearchPublicChannelsWithoutMembership: false, + EnableFileIndexing: true, IgnoredPurgeIndexes: '', }, }; @@ -174,6 +179,7 @@ describe('components/ElasticSearchSettings', () => { EnableSearching: false, EnableAutocomplete: false, EnableSearchPublicChannelsWithoutMembership: false, + EnableFileIndexing: true, IgnoredPurgeIndexes: '', }, }; diff --git a/webapp/channels/src/components/admin_console/elasticsearch_settings.tsx b/webapp/channels/src/components/admin_console/elasticsearch_settings.tsx index d7f48bd62ea..eac359dbfda 100644 --- a/webapp/channels/src/components/admin_console/elasticsearch_settings.tsx +++ b/webapp/channels/src/components/admin_console/elasticsearch_settings.tsx @@ -38,6 +38,7 @@ interface State extends BaseState { enableSearching: boolean; enableAutocomplete: boolean; enableSearchPublicChannelsWithoutMembership: boolean; + enableFileIndexing: boolean; configTested: boolean; canSave: boolean; canPurgeAndIndex: boolean; @@ -119,6 +120,7 @@ export default class ElasticsearchSettings extends OLDAdminSettings + + } + helpText={ + + } + value={this.state.enableFileIndexing} + disabled={this.props.isDisabled || !this.state.enableIndexing || !this.state.configTested} + onChange={this.handleSettingChanged} + setByEnv={this.isSetByEnv('ElasticsearchSettings.EnableFileIndexing')} + /> ); }; diff --git a/webapp/channels/src/i18n/en.json b/webapp/channels/src/i18n/en.json index 6b4487df626..bd46a485bd3 100644 --- a/webapp/channels/src/i18n/en.json +++ b/webapp/channels/src/i18n/en.json @@ -4911,6 +4911,14 @@ "defaultMessage": "Enable Elasticsearch for autocomplete queries:", "description": "Toggle setting name on the Elasticsearch admin page for using Elasticsearch in autocomplete; ends with a colon." }, + "admin.elasticsearch.enableFileIndexingDescription": { + "defaultMessage": "When false, new or updated file attachments are not indexed. File search continues to return previously indexed files, but files uploaded while this setting is false are not searchable until a bulk index is run after re-enabling it.", + "description": "Help text under the file indexing toggle on the Elasticsearch admin page, explaining that turning it off stops indexing new or updated file attachments while file search keeps returning previously indexed files." + }, + "admin.elasticsearch.enableFileIndexingTitle": { + "defaultMessage": "Enable file indexing:", + "description": "Toggle setting name on the Elasticsearch admin page for indexing file attachments; ends with a colon." + }, "admin.elasticsearch.enableIndexingDescription": { "defaultMessage": "When true, indexing of new posts occurs automatically. Search queries will use database search until \"Enable Elasticsearch for search queries\" is enabled. Learn more about Elasticsearch in our documentation.", "description": "Admin console help text for the Enable Elasticsearch Indexing toggle; wraps the sentence linking to documentation, the quoted setting name refers to a sibling setting." diff --git a/webapp/platform/types/src/config.ts b/webapp/platform/types/src/config.ts index 0130c62f616..5a42bb4770a 100644 --- a/webapp/platform/types/src/config.ts +++ b/webapp/platform/types/src/config.ts @@ -942,6 +942,7 @@ export type ElasticsearchSettings = { Trace: string; IgnoredPurgeIndexes: string; EnableSearchPublicChannelsWithoutMembership: boolean; + EnableFileIndexing: boolean; }; export type DataRetentionSettings = {