Add ElasticsearchSettings.EnableFileIndexing to disable file indexing (#39058)

* Add ElasticsearchSettings.EnableFileIndexing to disable file indexing

* Document ElasticsearchSettings.EnableFileIndexing

* Decide index_files when creating indexing jobs

Instead of the indexing worker reading EnableFileIndexing, job creators
set index_files from the setting: App.CreateJob defaults it when the
caller omits it, and the post aggregation workers set it explicitly.
The worker is unchanged, so running and resumed jobs keep the decision
they were created with, and an explicit index_files=true still indexes
files when the setting is off.

docs/main/administration-guide/configure/environment-configuration-settings.mdx

@@ -1446,6 +1446,23 @@ This setting has no effect when [Compliance Mode](/administration-guide/configur
 
 </Note>
 
+### Enable file indexing
+
+<table>
+<colgroup>
+<col style={{width: '41%'}} />
+<col style={{width: '58%'}} />
+</colgroup>
+<tbody>
+<tr>
+<td><p>Configure whether Mattermost indexes file attachments in Elasticsearch or AWS OpenSearch.</p><ul><li><strong>true</strong>: <strong>(Default)</strong> New and updated file attachments are indexed, and bulk indexing jobs include files.</li><li><strong>false</strong>: New or updated file attachments aren't indexed, and bulk indexing jobs skip files. File search continues to return previously indexed files, and files deleted from Mattermost are still removed from the index.</li></ul></td>
+<td><ul><li>System Config path: <strong>Environment &gt; Elasticsearch</strong></li><li><code>config.json</code> setting: <code>ElasticsearchSettings</code> &gt; <code>EnableFileIndexing</code> &gt; <code>true</code></li><li>Environment variable: <code>MM_ELASTICSEARCHSETTINGS_ENABLEFILEINDEXING</code></li></ul></td>
+</tr>
+</tbody>
+</table>
+
+Files uploaded while this setting is **false** don't appear in file search results. After setting it back to **true**, run a bulk index from **System Console > Environment > Elasticsearch** to make those files searchable.
+
 ### Post index replicas
 
 <table>

server/channels/app/job.go

@@ -6,6 +6,7 @@ package app
 import (
 	"errors"
 	"net/http"
+	"strconv"
 	"strings"
 
 	"github.com/mattermost/mattermost/server/public/model"
@@ -91,6 +92,14 @@ func (a *App) CreateJob(rctx request.CTX, job *model.Job) (*model.Job, *model.Ap
 	case model.JobTypeAccessControlTeamSync:
 		// Route team ABAC jobs to specialized deduplication handler
 		return a.CreateAccessControlTeamSyncJob(rctx, job.Data)
+	case model.JobTypeElasticsearchPostIndexing:
+		if _, ok := job.Data["index_files"]; !ok {
+			if job.Data == nil {
+				job.Data = model.StringMap{}
+			}
+			job.Data["index_files"] = strconv.FormatBool(*a.Config().ElasticsearchSettings.EnableFileIndexing)
+		}
+		return a.Srv().Jobs.CreateJob(rctx, job.Type, job.Data)
 	default:
 		return a.Srv().Jobs.CreateJob(rctx, job.Type, job.Data)
 	}

server/channels/app/job_test.go

@@ -10,6 +10,8 @@ import (
 	"github.com/stretchr/testify/require"
 
 	"github.com/mattermost/mattermost/server/public/model"
+	"github.com/mattermost/mattermost/server/public/shared/mlog"
+	"github.com/mattermost/mattermost/server/v8/channels/jobs"
 )
 
 func TestGetJob(t *testing.T) {
@@ -771,6 +773,45 @@ func TestSessionHasPermissionToReadJob(t *testing.T) {
 	}
 }
 
+func TestCreateElasticsearchPostIndexingJob(t *testing.T) {
+	mainHelper.Parallel(t)
+	th := Setup(t)
+
+	worker := jobs.NewSimpleWorker(model.JobTypeElasticsearchPostIndexing, th.Server.Jobs, func(mlog.LoggerIFace, *model.Job) error { return nil }, func(*model.Config) bool { return false })
+	th.Server.Jobs.RegisterJobType(model.JobTypeElasticsearchPostIndexing, worker, nil)
+
+	createJob := func(t *testing.T, data model.StringMap) *model.Job {
+		job, appErr := th.App.CreateJob(th.Context, &model.Job{Type: model.JobTypeElasticsearchPostIndexing, Data: data})
+		require.Nil(t, appErr)
+		t.Cleanup(func() {
+			_, err := th.App.Srv().Store().Job().Delete(job.Id)
+			require.NoError(t, err)
+		})
+		return job
+	}
+
+	t.Run("indexes files by default when file indexing is enabled", func(t *testing.T) {
+		th.App.UpdateConfig(func(cfg *model.Config) { *cfg.ElasticsearchSettings.EnableFileIndexing = true })
+
+		job := createJob(t, nil)
+		assert.Equal(t, "true", job.Data["index_files"])
+	})
+
+	t.Run("skips files by default when file indexing is disabled", func(t *testing.T) {
+		th.App.UpdateConfig(func(cfg *model.Config) { *cfg.ElasticsearchSettings.EnableFileIndexing = false })
+
+		job := createJob(t, nil)
+		assert.Equal(t, "false", job.Data["index_files"])
+	})
+
+	t.Run("keeps an explicit index_files value", func(t *testing.T) {
+		th.App.UpdateConfig(func(cfg *model.Config) { *cfg.ElasticsearchSettings.EnableFileIndexing = false })
+
+		job := createJob(t, model.StringMap{"index_files": "true"})
+		assert.Equal(t, "true", job.Data["index_files"])
+	})
+}
+
 func TestGetJobByType(t *testing.T) {
 	mainHelper.Parallel(t)
 	th := Setup(t)

server/channels/app/platform/testdata/support_packet/sanitized_config.json

@@ -569,7 +569,8 @@
         "ClientKey": null,
         "Trace": null,
         "IgnoredPurgeIndexes": null,
-        "EnableSearchPublicChannelsWithoutMembership": null
+        "EnableSearchPublicChannelsWithoutMembership": null,
+        "EnableFileIndexing": null
     },
     "DataRetentionSettings": {
         "EnableMessageDeletion": null,

server/channels/store/searchlayer/file_info_layer.go

@@ -19,6 +19,10 @@ type SearchFileInfoStore struct {
 }
 
 func (s SearchFileInfoStore) indexFile(rctx request.CTX, file *model.FileInfo) {
+	if !*s.rootStore.getConfig().ElasticsearchSettings.EnableFileIndexing {
+		return
+	}
+
 	for _, engine := range s.rootStore.searchEngine.GetActiveEngines() {
 		if engine.IsIndexingEnabled() {
 			runIndexFn(rctx, engine, func(engineCopy searchengine.SearchEngineInterface) {

server/channels/store/searchlayer/file_info_layer_test.go

@@ -0,0 +1,71 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package searchlayer_test
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/mock"
+	"github.com/stretchr/testify/require"
+
+	"github.com/mattermost/mattermost/server/public/model"
+	"github.com/mattermost/mattermost/server/public/shared/request"
+	"github.com/mattermost/mattermost/server/v8/channels/store/searchlayer"
+	storemocks "github.com/mattermost/mattermost/server/v8/channels/store/storetest/mocks"
+	"github.com/mattermost/mattermost/server/v8/platform/services/searchengine"
+	"github.com/mattermost/mattermost/server/v8/platform/services/searchengine/mocks"
+)
+
+func TestFileIndexing(t *testing.T) {
+	setup := func(t *testing.T, enableFileIndexing bool) (*searchlayer.SearchStore, *storemocks.FileInfoStore, *mocks.SearchEngineInterface) {
+		cfg := &model.Config{}
+		cfg.SetDefaults()
+		cfg.ElasticsearchSettings.EnableFileIndexing = model.NewPointer(enableFileIndexing)
+
+		fileInfoStore := &storemocks.FileInfoStore{}
+		baseStore := &storemocks.Store{}
+		baseStore.On("FileInfo").Return(fileInfoStore)
+		baseStore.On("Channel").Return(&storemocks.ChannelStore{})
+		baseStore.On("Post").Return(&storemocks.PostStore{})
+		baseStore.On("Team").Return(&storemocks.TeamStore{})
+		baseStore.On("User").Return(&storemocks.UserStore{})
+
+		es := &mocks.SearchEngineInterface{}
+		es.On("IsActive").Return(true)
+		es.On("IsHealthy").Return(true)
+		es.On("IsIndexingEnabled").Return(true).Maybe()
+		es.On("IsSearchEnabled").Return(true).Maybe()
+		es.On("IsIndexingSync").Return(true).Maybe()
+		es.On("RefreshIndexes", mock.Anything).Return(nil).Maybe()
+		es.On("GetName").Return("mock").Maybe()
+
+		broker := searchengine.NewBroker(cfg)
+		broker.ElasticsearchEngine = es
+
+		return searchlayer.NewSearchLayer(baseStore, broker, cfg), fileInfoStore, es
+	}
+
+	file := &model.FileInfo{Id: model.NewId(), PostId: model.NewId(), ChannelId: model.NewId()}
+
+	t.Run("indexes saved files when file indexing is enabled", func(t *testing.T) {
+		layer, fileInfoStore, es := setup(t, true)
+		fileInfoStore.On("Save", mock.Anything, file).Return(file, nil)
+		es.On("IndexFile", file, file.ChannelId).Return(nil)
+
+		_, err := layer.FileInfo().Save(request.TestContext(t), file)
+		require.NoError(t, err)
+
+		es.AssertCalled(t, "IndexFile", file, file.ChannelId)
+	})
+
+	t.Run("does not index saved files when file indexing is disabled", func(t *testing.T) {
+		layer, fileInfoStore, es := setup(t, false)
+		fileInfoStore.On("Save", mock.Anything, file).Return(file, nil)
+
+		_, err := layer.FileInfo().Save(request.TestContext(t), file)
+		require.NoError(t, err)
+
+		es.AssertNotCalled(t, "IndexFile", mock.Anything, mock.Anything)
+	})
+}

server/enterprise/elasticsearch/elasticsearch/aggregation_job.go

@@ -224,8 +224,9 @@ func (worker *ElasticsearchAggregatorWorker) DoJob(job *model.Job) {
 		rctx,
 		model.JobTypeElasticsearchPostIndexing,
 		map[string]string{
-			"start_time": strconv.FormatInt(reindexingStartDate.UnixNano()/int64(time.Millisecond), 10),
-			"end_time":   strconv.FormatInt(reindexingEndDate.UnixNano()/int64(time.Millisecond), 10),
+			"start_time":  strconv.FormatInt(reindexingStartDate.UnixNano()/int64(time.Millisecond), 10),
+			"end_time":    strconv.FormatInt(reindexingEndDate.UnixNano()/int64(time.Millisecond), 10),
+			"index_files": strconv.FormatBool(*worker.jobServer.Config().ElasticsearchSettings.EnableFileIndexing),
 		},
 	); appErr != nil {
 		logger.Error("Worker: Failed to create indexing job.", mlog.Err(appErr))

server/enterprise/elasticsearch/opensearch/aggregation_job.go

@@ -223,8 +223,9 @@ func (worker *OpensearchAggregatorWorker) DoJob(job *model.Job) {
 		rctx,
 		model.JobTypeElasticsearchPostIndexing,
 		map[string]string{
-			"start_time": strconv.FormatInt(reindexingStartDate.UnixNano()/int64(time.Millisecond), 10),
-			"end_time":   strconv.FormatInt(reindexingEndDate.UnixNano()/int64(time.Millisecond), 10),
+			"start_time":  strconv.FormatInt(reindexingStartDate.UnixNano()/int64(time.Millisecond), 10),
+			"end_time":    strconv.FormatInt(reindexingEndDate.UnixNano()/int64(time.Millisecond), 10),
+			"index_files": strconv.FormatBool(*worker.jobServer.Config().ElasticsearchSettings.EnableFileIndexing),
 		},
 	); appErr != nil {
 		logger.Error("Worker: Failed to create indexing job.", mlog.Err(appErr))

server/public/model/config.go

@@ -3302,6 +3302,7 @@ type ElasticsearchSettings struct {
 	Trace                                       *string `access:"environment_elasticsearch,write_restrictable,cloud_restrictable"`
 	IgnoredPurgeIndexes                         *string `access:"environment_elasticsearch,write_restrictable,cloud_restrictable"` // telemetry: none
 	EnableSearchPublicChannelsWithoutMembership *bool   `access:"environment_elasticsearch,write_restrictable,cloud_restrictable"`
+	EnableFileIndexing                          *bool   `access:"environment_elasticsearch,write_restrictable,cloud_restrictable"`
 }
 
 func (s *ElasticsearchSettings) SetDefaults() {
@@ -3420,6 +3421,10 @@ func (s *ElasticsearchSettings) SetDefaults() {
 	if s.EnableSearchPublicChannelsWithoutMembership == nil {
 		s.EnableSearchPublicChannelsWithoutMembership = new(true)
 	}
+
+	if s.EnableFileIndexing == nil {
+		s.EnableFileIndexing = new(true)
+	}
 }
 
 type DataRetentionSettings struct {

server/public/model/config_test.go

@@ -5168,6 +5168,18 @@ func TestElasticsearchSettingsSetDefaults(t *testing.T) {
 		s.SetDefaults()
 		require.False(t, *s.EnableSearchPublicChannelsWithoutMembership)
 	})
+
+	t.Run("EnableFileIndexing defaults to true when nil", func(t *testing.T) {
+		s := ElasticsearchSettings{}
+		s.SetDefaults()
+		require.True(t, *s.EnableFileIndexing)
+	})
+
+	t.Run("EnableFileIndexing preserves explicit false", func(t *testing.T) {
+		s := ElasticsearchSettings{EnableFileIndexing: NewPointer(false)}
+		s.SetDefaults()
+		require.False(t, *s.EnableFileIndexing)
+	})
 }
 
 func TestElasticsearchSettingsIsValid(t *testing.T) {

webapp/channels/src/components/admin_console/__snapshots__/elasticsearch_settings.test.tsx.snap

@@ -733,6 +733,54 @@ exports[`components/ElasticSearchSettings should match snapshot, disabled 1`] =
               </div>
             </div>
           </fieldset>
+          <fieldset
+            class="form-group"
+            data-testid="enableFileIndexing"
+            id="enableFileIndexing"
+          >
+            <legend
+              class="control-label form-legend col-sm-4"
+            >
+              Enable file indexing:
+            </legend>
+            <div
+              class="col-sm-8"
+            >
+              <label
+                class="Label-eJcoMl kxVoZJ"
+              >
+                <input
+                  checked=""
+                  data-testid="enableFileIndexingtrue"
+                  disabled=""
+                  id="enableFileIndexingtrue"
+                  name="enableFileIndexing"
+                  type="radio"
+                  value="true"
+                />
+                True
+              </label>
+              <label
+                class="Label-eJcoMl kxVoZJ"
+              >
+                <input
+                  data-testid="enableFileIndexingfalse"
+                  disabled=""
+                  id="enableFileIndexingfalse"
+                  name="enableFileIndexing"
+                  type="radio"
+                  value="false"
+                />
+                False
+              </label>
+              <div
+                class="help-text"
+                data-testid="enableFileIndexinghelp-text"
+              >
+                When false, new or updated file attachments are not indexed. File search continues to return previously indexed files, but files uploaded while this setting is false are not searchable until a bulk index is run after re-enabling it.
+              </div>
+            </div>
+          </fieldset>
         </div>
       </div>
       <div
@@ -1469,6 +1517,52 @@ exports[`components/ElasticSearchSettings should match snapshot, enabled 1`] = `
               </div>
             </div>
           </fieldset>
+          <fieldset
+            class="form-group"
+            data-testid="enableFileIndexing"
+            id="enableFileIndexing"
+          >
+            <legend
+              class="control-label form-legend col-sm-4"
+            >
+              Enable file indexing:
+            </legend>
+            <div
+              class="col-sm-8"
+            >
+              <label
+                class="Label-eJcoMl iShgPr"
+              >
+                <input
+                  checked=""
+                  data-testid="enableFileIndexingtrue"
+                  id="enableFileIndexingtrue"
+                  name="enableFileIndexing"
+                  type="radio"
+                  value="true"
+                />
+                True
+              </label>
+              <label
+                class="Label-eJcoMl iShgPr"
+              >
+                <input
+                  data-testid="enableFileIndexingfalse"
+                  id="enableFileIndexingfalse"
+                  name="enableFileIndexing"
+                  type="radio"
+                  value="false"
+                />
+                False
+              </label>
+              <div
+                class="help-text"
+                data-testid="enableFileIndexinghelp-text"
+              >
+                When false, new or updated file attachments are not indexed. File search continues to return previously indexed files, but files uploaded while this setting is false are not searchable until a bulk index is run after re-enabling it.
+              </div>
+            </div>
+          </fieldset>
         </div>
       </div>
       <div
@@ -2214,6 +2308,52 @@ exports[`components/ElasticSearchSettings should match snapshot, sniff enabled 1
               </div>
             </div>
           </fieldset>
+          <fieldset
+            class="form-group"
+            data-testid="enableFileIndexing"
+            id="enableFileIndexing"
+          >
+            <legend
+              class="control-label form-legend col-sm-4"
+            >
+              Enable file indexing:
+            </legend>
+            <div
+              class="col-sm-8"
+            >
+              <label
+                class="Label-eJcoMl iShgPr"
+              >
+                <input
+                  checked=""
+                  data-testid="enableFileIndexingtrue"
+                  id="enableFileIndexingtrue"
+                  name="enableFileIndexing"
+                  type="radio"
+                  value="true"
+                />
+                True
+              </label>
+              <label
+                class="Label-eJcoMl iShgPr"
+              >
+                <input
+                  data-testid="enableFileIndexingfalse"
+                  id="enableFileIndexingfalse"
+                  name="enableFileIndexing"
+                  type="radio"
+                  value="false"
+                />
+                False
+              </label>
+              <div
+                class="help-text"
+                data-testid="enableFileIndexinghelp-text"
+              >
+                When false, new or updated file attachments are not indexed. File search continues to return previously indexed files, but files uploaded while this setting is false are not searchable until a bulk index is run after re-enabling it.
+              </div>
+            </div>
+          </fieldset>
         </div>
       </div>
       <div

webapp/channels/src/components/admin_console/elasticsearch_settings.test.tsx

@@ -34,6 +34,7 @@ describe('components/ElasticSearchSettings', () => {
                 EnableSearching: false,
                 EnableAutocomplete: false,
                 EnableSearchPublicChannelsWithoutMembership: false,
+                EnableFileIndexing: true,
                 IgnoredPurgeIndexes: '',
             },
         };
@@ -62,6 +63,7 @@ describe('components/ElasticSearchSettings', () => {
                 EnableSearching: false,
                 EnableAutocomplete: false,
                 EnableSearchPublicChannelsWithoutMembership: false,
+                EnableFileIndexing: true,
                 IgnoredPurgeIndexes: '',
             },
         };
@@ -90,6 +92,7 @@ describe('components/ElasticSearchSettings', () => {
                 EnableSearching: false,
                 EnableAutocomplete: false,
                 EnableSearchPublicChannelsWithoutMembership: false,
+                EnableFileIndexing: true,
                 IgnoredPurgeIndexes: '',
             },
         };
@@ -118,6 +121,7 @@ describe('components/ElasticSearchSettings', () => {
                 EnableSearching: false,
                 EnableAutocomplete: false,
                 EnableSearchPublicChannelsWithoutMembership: false,
+                EnableFileIndexing: true,
                 IgnoredPurgeIndexes: '',
             },
         };
@@ -146,6 +150,7 @@ describe('components/ElasticSearchSettings', () => {
                 EnableSearching: false,
                 EnableAutocomplete: false,
                 EnableSearchPublicChannelsWithoutMembership: false,
+                EnableFileIndexing: true,
                 IgnoredPurgeIndexes: '',
             },
         };
@@ -174,6 +179,7 @@ describe('components/ElasticSearchSettings', () => {
                 EnableSearching: false,
                 EnableAutocomplete: false,
                 EnableSearchPublicChannelsWithoutMembership: false,
+                EnableFileIndexing: true,
                 IgnoredPurgeIndexes: '',
             },
         };

webapp/channels/src/components/admin_console/elasticsearch_settings.tsx

@@ -38,6 +38,7 @@ interface State extends BaseState {
     enableSearching: boolean;
     enableAutocomplete: boolean;
     enableSearchPublicChannelsWithoutMembership: boolean;
+    enableFileIndexing: boolean;
     configTested: boolean;
     canSave: boolean;
     canPurgeAndIndex: boolean;
@@ -119,6 +120,7 @@ export default class ElasticsearchSettings extends OLDAdminSettings<Props, State
         config.ElasticsearchSettings.EnableAutocomplete = this.state.enableAutocomplete;
         config.ElasticsearchSettings.IgnoredPurgeIndexes = this.state.ignoredPurgeIndexes;
         config.ElasticsearchSettings.EnableSearchPublicChannelsWithoutMembership = this.state.enableSearchPublicChannelsWithoutMembership;
+        config.ElasticsearchSettings.EnableFileIndexing = this.state.enableFileIndexing;
 
         return config;
     };
@@ -138,6 +140,7 @@ export default class ElasticsearchSettings extends OLDAdminSettings<Props, State
             enableSearching: config.ElasticsearchSettings.EnableSearching,
             enableAutocomplete: config.ElasticsearchSettings.EnableAutocomplete,
             enableSearchPublicChannelsWithoutMembership: config.ElasticsearchSettings.EnableSearchPublicChannelsWithoutMembership,
+            enableFileIndexing: config.ElasticsearchSettings.EnableFileIndexing,
             configTested: true,
             canSave: true,
             canPurgeAndIndex: config.ElasticsearchSettings.EnableIndexing,
@@ -564,6 +567,25 @@ export default class ElasticsearchSettings extends OLDAdminSettings<Props, State
                     onChange={this.handleSettingChanged}
                     setByEnv={this.isSetByEnv('ElasticsearchSettings.EnableSearchPublicChannelsWithoutMembership')}
                 />
+                <BooleanSetting
+                    id='enableFileIndexing'
+                    label={
+                        <FormattedMessage
+                            id='admin.elasticsearch.enableFileIndexingTitle'
+                            defaultMessage='Enable file indexing:'
+                        />
+                    }
+                    helpText={
+                        <FormattedMessage
+                            id='admin.elasticsearch.enableFileIndexingDescription'
+                            defaultMessage='When false, new or updated file attachments are not indexed. File search continues to return previously indexed files, but files uploaded while this setting is false are not searchable until a bulk index is run after re-enabling it.'
+                        />
+                    }
+                    value={this.state.enableFileIndexing}
+                    disabled={this.props.isDisabled || !this.state.enableIndexing || !this.state.configTested}
+                    onChange={this.handleSettingChanged}
+                    setByEnv={this.isSetByEnv('ElasticsearchSettings.EnableFileIndexing')}
+                />
             </SettingsGroup>
         );
     };

webapp/channels/src/i18n/en.json

@@ -4911,6 +4911,14 @@
     "defaultMessage": "Enable Elasticsearch for autocomplete queries:",
     "description": "Toggle setting name on the Elasticsearch admin page for using Elasticsearch in autocomplete; ends with a colon."
   },
+  "admin.elasticsearch.enableFileIndexingDescription": {
+    "defaultMessage": "When false, new or updated file attachments are not indexed. File search continues to return previously indexed files, but files uploaded while this setting is false are not searchable until a bulk index is run after re-enabling it.",
+    "description": "Help text under the file indexing toggle on the Elasticsearch admin page, explaining that turning it off stops indexing new or updated file attachments while file search keeps returning previously indexed files."
+  },
+  "admin.elasticsearch.enableFileIndexingTitle": {
+    "defaultMessage": "Enable file indexing:",
+    "description": "Toggle setting name on the Elasticsearch admin page for indexing file attachments; ends with a colon."
+  },
   "admin.elasticsearch.enableIndexingDescription": {
     "defaultMessage": "When true, indexing of new posts occurs automatically. Search queries will use database search until \"Enable Elasticsearch for search queries\" is enabled. <link>Learn more about Elasticsearch in our documentation.</link>",
     "description": "Admin console help text for the Enable Elasticsearch Indexing toggle; <link></link> wraps the sentence linking to documentation, the quoted setting name refers to a sibling setting."

webapp/platform/types/src/config.ts

@@ -942,6 +942,7 @@ export type ElasticsearchSettings = {
     Trace: string;
     IgnoredPurgeIndexes: string;
     EnableSearchPublicChannelsWithoutMembership: boolean;
+    EnableFileIndexing: boolean;
 };
 
 export type DataRetentionSettings = {