opensource/mattermost
3
0
Fork 0
mirror of https://github.com/mattermost/mattermost.git synced 2026-09-21 05:54:10 +08:00
Code Issues Packages Projects Releases Wiki Activity

MM-67505 Add AnalyticsQueryTimeout setting and use when refreshing materialized views (#35906)

Browse Source 
* MM-67505 Add AnalyticsQueryTimeout setting and use when refreshing materialized views

* Fix last minute i18n change

* Disallow 0 values for AnalyticsQueryTimeout

* Fix E2E test config

* Fix post store tests crashing

* Update snapshot and revert accidental changes to it
This commit is contained in:
Harrison Healey
2026-04-13 09:27:20 -04:00
committed by GitHub
parent 4dcf6916f7
commit 8e14c8ed58
15 changed files with 99 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files
e2e-tests/cypress/tests/support/api/cloud_default_config.json
+1
View File
@@ -292,6 +292,7 @@
"PublicCertificateFile": "",
"PrivateKeyFile": "",
"QueryTimeout": 60,
"AnalyticsQueryTimeout": 300,
"MaxPageSize": 500,
"LoginFieldName": "",
"LoginButtonColor": "#0000",
e2e-tests/cypress/tests/support/api/on_prem_default_config.json
+1
View File
@@ -152,6 +152,7 @@
"Trace": false,
"AtRestEncryptKey": "",
"QueryTimeout": 30,
"AnalyticsQueryTimeout": 300,
"DisableDatabaseSearch": false,
"MigrationsStatementTimeoutSeconds": 100000,
"ReplicaLagSettings": [],
e2e-tests/playwright/lib/src/server/default_config.ts
+1
View File
@@ -259,6 +259,7 @@ const defaultServerConfig: AdminConfig = {
Trace: false,
AtRestEncryptKey: '',
QueryTimeout: 30,
AnalyticsQueryTimeout: 300,
DisableDatabaseSearch: false,
MigrationsStatementTimeoutSeconds: 100000,
ReplicaLagSettings: [],
server/channels/store/sqlstore/file_info_store.go
+4 -1
View File
@@ -794,12 +794,15 @@ func (fs SqlFileInfoStore) RestoreForPostByIds(rctx request.CTX, postId string,
}
func (fs SqlFileInfoStore) RefreshFileStats() error {
ctx, cancel := fs.analyticsContext()
defer cancel()
// CONCURRENTLY is not used deliberately because as per Postgres docs,
// not using CONCURRENTLY takes less resources and completes faster
// at the expense of locking the mat view. Since viewing admin console
// is not a very frequent activity, we accept the tradeoff to let the
// refresh happen as fast as possible.
if _, err := fs.GetMaster().Exec("REFRESH MATERIALIZED VIEW file_stats"); err != nil {
if _, err := fs.GetMaster().ExecContext(ctx, "REFRESH MATERIALIZED VIEW file_stats"); err != nil {
return errors.Wrap(err, "error refreshing materialized view file_stats")
}
server/channels/store/sqlstore/post_store.go
+9 -2
View File
@@ -3267,11 +3267,18 @@ func (s *SqlPostStore) RefreshPostStats() error {
// at the expense of locking the mat view. Since viewing admin console
// is not a very frequent activity, we accept the tradeoff to let the
// refresh happen as fast as possible.
if _, err := s.GetMaster().Exec("REFRESH MATERIALIZED VIEW posts_by_team_day"); err != nil {
postsCtx, postsCancel := s.analyticsContext()
defer postsCancel()
if _, err := s.GetMaster().ExecContext(postsCtx, "REFRESH MATERIALIZED VIEW posts_by_team_day"); err != nil {
return errors.Wrap(err, "error refreshing materialized view posts_by_team_day")
}
if _, err := s.GetMaster().Exec("REFRESH MATERIALIZED VIEW bot_posts_by_team_day"); err != nil {
botPostsCtx, botPostsCancel := s.analyticsContext()
defer botPostsCancel()
if _, err := s.GetMaster().ExecContext(botPostsCtx, "REFRESH MATERIALIZED VIEW bot_posts_by_team_day"); err != nil {
return errors.Wrap(err, "error refreshing materialized view bot_posts_by_team_day")
}
server/channels/store/sqlstore/store.go
+5
View File
@@ -4,6 +4,7 @@
package sqlstore
import (
"context"
"database/sql"
"fmt"
"path"
@@ -474,6 +475,10 @@ func (ss *SqlStore) GetReplica() *sqlxDBWrapper {
return ss.GetMaster()
}
func (ss *SqlStore) analyticsContext() (context.Context, context.CancelFunc) {
return context.WithTimeout(context.Background(), time.Duration(*ss.settings.AnalyticsQueryTimeout)*time.Second)
}
func (ss *SqlStore) monitorReplicas() {
t := time.NewTicker(time.Duration(*ss.settings.ReplicaMonitorIntervalSeconds) * time.Second)
defer func() {
server/channels/store/sqlstore/user_store.go
+5 -1
View File
@@ -2374,9 +2374,13 @@ func (us SqlUserStore) GetUsersWithInvalidEmails(page int, perPage int, restrict
}
func (us SqlUserStore) RefreshPostStatsForUsers() error {
if _, err := us.GetMaster().Exec("REFRESH MATERIALIZED VIEW poststats"); err != nil {
ctx, cancel := us.analyticsContext()
defer cancel()
if _, err := us.GetMaster().ExecContext(ctx, "REFRESH MATERIALIZED VIEW poststats"); err != nil {
return errors.Wrap(err, "users_refresh_post_stats_exec")
}
return nil
}
server/channels/store/storetest/settings.go
+2
View File
@@ -90,6 +90,7 @@ func databaseSettings(driver, dataSource string) *model.SqlSettings {
Trace: model.NewPointer(false),
AtRestEncryptKey: model.NewPointer(model.NewRandomString(32)),
QueryTimeout: new(int),
AnalyticsQueryTimeout: new(int),
MigrationsStatementTimeoutSeconds: new(int),
}
*settings.MaxIdleConns = 10
@@ -97,6 +98,7 @@ func databaseSettings(driver, dataSource string) *model.SqlSettings {
*settings.ConnMaxIdleTimeMilliseconds = 300000
*settings.MaxOpenConns = 100
*settings.QueryTimeout = 60
*settings.AnalyticsQueryTimeout = 300
*settings.MigrationsStatementTimeoutSeconds = 60
return settings
server/i18n/en.json
+4
View File
@@ -11028,6 +11028,10 @@
"id": "model.config.is_valid.sitename_length.app_error",
"translation": "Site name must be less than or equal to {{.MaxLength}} characters."
},
{
"id": "model.config.is_valid.sql_analytics_query_timeout.app_error",
"translation": "Invalid analytics query timeout for SQL settings. Must be a positive number."
},
{
"id": "model.config.is_valid.sql_conn_max_idle_time_milliseconds.app_error",
"translation": "Invalid connection maximum idle time for SQL settings. Must be a non-negative number."
server/public/model/config.go
+9
View File
@@ -1491,6 +1491,7 @@ type SqlSettings struct {
Trace *bool `access:"environment_database,write_restrictable,cloud_restrictable"`
AtRestEncryptKey *string `access:"environment_database,write_restrictable,cloud_restrictable"` // telemetry: none
QueryTimeout *int `access:"environment_database,write_restrictable,cloud_restrictable"`
AnalyticsQueryTimeout *int `access:"environment_database,write_restrictable,cloud_restrictable"`
DisableDatabaseSearch *bool `access:"environment_database,write_restrictable,cloud_restrictable"`
MigrationsStatementTimeoutSeconds *int `access:"environment_database,write_restrictable,cloud_restrictable"`
ReplicaLagSettings []*ReplicaLagSettings `access:"environment_database,write_restrictable,cloud_restrictable"` // telemetry: none
@@ -1548,6 +1549,10 @@ func (s *SqlSettings) SetDefaults(isUpdate bool) {
s.QueryTimeout = NewPointer(30)
}
if s.AnalyticsQueryTimeout == nil {
s.AnalyticsQueryTimeout = NewPointer(300)
}
if s.DisableDatabaseSearch == nil {
s.DisableDatabaseSearch = NewPointer(false)
}
@@ -4366,6 +4371,10 @@ func (s *SqlSettings) isValid() *AppError {
return NewAppError("Config.IsValid", "model.config.is_valid.sql_query_timeout.app_error", nil, "", http.StatusBadRequest)
}
if *s.AnalyticsQueryTimeout <= 0 {
return NewAppError("Config.IsValid", "model.config.is_valid.sql_analytics_query_timeout.app_error", nil, "", http.StatusBadRequest)
}
if *s.DataSource == "" {
return NewAppError("Config.IsValid", "model.config.is_valid.sql_data_src.app_error", nil, "", http.StatusBadRequest)
}
webapp/channels/src/components/admin_console/__snapshots__/database_settings.test.tsx.snap
+31
View File
@@ -168,6 +168,37 @@ exports[`components/DatabaseSettings should match snapshot 1`] = `
</div>
</div>
</div>
<div
class="form-group"
data-testid="analyticsQueryTimeout"
>
<label
class="control-label col-sm-4"
data-testid="analyticsQueryTimeoutlabel"
for="analyticsQueryTimeout"
>
Analytics Query Timeout:
</label>
<div
class="col-sm-8"
>
<input
class="form-control"
data-testid="analyticsQueryTimeoutinput"
id="analyticsQueryTimeout"
maxlength="-1"
placeholder="E.g.: \\"300\\""
type="text"
value="300"
/>
<div
class="help-text"
data-testid="analyticsQueryTimeouthelp-text"
>
The number of seconds to wait for a response from the database after opening a connection and sending certain analytics queries. This setting only applies to long queries which are run in the background to populate some information in the Team and Site Statistics pages.
</div>
</div>
</div>
<div
class="form-group"
data-testid="connMaxLifetimeMilliseconds"
webapp/channels/src/components/admin_console/database_settings.test.tsx
+1
View File
@@ -35,6 +35,7 @@ describe('components/DatabaseSettings', () => {
DisableDatabaseSearch: true,
DataSource: 'postgres://mmuser:mostest@localhost/mattermost_test?sslmode=disable\u0026connect_timeout=10',
QueryTimeout: 10,
AnalyticsQueryTimeout: 300,
ConnMaxLifetimeMilliseconds: 10,
ConnMaxIdleTimeMilliseconds: 20,
},
webapp/channels/src/components/admin_console/database_settings.tsx
+22
View File
@@ -36,6 +36,7 @@ interface State extends BaseState {
trace: boolean;
disableDatabaseSearch: boolean;
queryTimeout: number;
analyticsQueryTimeout: number;
connMaxLifetimeMilliseconds: number;
connMaxIdleTimeMilliseconds: number;
minimumHashtagLength: number;
@@ -62,6 +63,8 @@ const messages = defineMessages({
maxOpenDescription: {id: 'admin.sql.maxOpenDescription', defaultMessage: 'Maximum number of open connections held open to the database.'},
queryTimeoutTitle: {id: 'admin.sql.queryTimeoutTitle', defaultMessage: 'Query Timeout:'},
queryTimeoutDescription: {id: 'admin.sql.queryTimeoutDescription', defaultMessage: 'The number of seconds to wait for a response from the database after opening a connection and sending the query. Errors that you see in the UI or in the logs as a result of a query timeout can vary depending on the type of query.'},
analyticsQueryTimeoutTitle: {id: 'admin.sql.analyticsQueryTimeoutTitle', defaultMessage: 'Analytics Query Timeout:'},
analyticsQueryTimeoutDescription: {id: 'admin.sql.analyticsQueryTimeoutDescription', defaultMessage: 'The number of seconds to wait for a response from the database after opening a connection and sending certain analytics queries. This setting only applies to long queries which are run in the background to populate some information in the Team and Site Statistics pages.'},
connMaxLifetimeTitle: {id: 'admin.sql.connMaxLifetimeTitle', defaultMessage: 'Maximum Connection Lifetime:'},
connMaxLifetimeDescription: {id: 'admin.sql.connMaxLifetimeDescription', defaultMessage: 'Maximum lifetime for a connection to the database in milliseconds.'},
connMaxIdleTimeTitle: {id: 'admin.sql.connMaxIdleTimeTitle', defaultMessage: 'Maximum Connection Idle Time:'},
@@ -91,6 +94,8 @@ export const searchableStrings: Array<string|MessageDescriptor|[MessageDescripto
messages.maxOpenDescription,
messages.queryTimeoutTitle,
messages.queryTimeoutDescription,
messages.analyticsQueryTimeoutTitle,
messages.analyticsQueryTimeoutDescription,
messages.connMaxLifetimeTitle,
messages.connMaxLifetimeDescription,
messages.connMaxIdleTimeTitle,
@@ -117,6 +122,7 @@ export default class DatabaseSettings extends OLDAdminSettings<Props, State> {
config.SqlSettings.Trace = this.state.trace;
config.SqlSettings.DisableDatabaseSearch = this.state.disableDatabaseSearch;
config.SqlSettings.QueryTimeout = this.parseIntNonZero(this.state.queryTimeout);
config.SqlSettings.AnalyticsQueryTimeout = this.parseIntNonZero(this.state.analyticsQueryTimeout);
config.SqlSettings.ConnMaxLifetimeMilliseconds = this.parseIntNonNegative(this.state.connMaxLifetimeMilliseconds);
config.SqlSettings.ConnMaxIdleTimeMilliseconds = this.parseIntNonNegative(this.state.connMaxIdleTimeMilliseconds);
config.ServiceSettings.MinimumHashtagLength = this.parseIntNonZero(this.state.minimumHashtagLength, 3, 2);
@@ -144,6 +150,7 @@ export default class DatabaseSettings extends OLDAdminSettings<Props, State> {
trace: config.SqlSettings.Trace,
disableDatabaseSearch: config.SqlSettings.DisableDatabaseSearch,
queryTimeout: config.SqlSettings.QueryTimeout,
analyticsQueryTimeout: config.SqlSettings.AnalyticsQueryTimeout,
connMaxLifetimeMilliseconds: config.SqlSettings.ConnMaxLifetimeMilliseconds,
connMaxIdleTimeMilliseconds: config.SqlSettings.ConnMaxIdleTimeMilliseconds,
minimumHashtagLength: config.ServiceSettings.MinimumHashtagLength,
@@ -284,6 +291,21 @@ export default class DatabaseSettings extends OLDAdminSettings<Props, State> {
disabled={this.props.isDisabled}
type='text'
/>
<TextSetting
id='analyticsQueryTimeout'
label={
<FormattedMessage {...messages.analyticsQueryTimeoutTitle}/>
}
placeholder={defineMessage({id: 'admin.sql.analyticsQueryTimeoutExample', defaultMessage: 'E.g.: "300"'})}
helpText={
<FormattedMessage {...messages.analyticsQueryTimeoutDescription}/>
}
value={this.state.analyticsQueryTimeout}
onChange={this.handleChange}
setByEnv={this.isSetByEnv('SqlSettings.AnalyticsQueryTimeout')}
disabled={this.props.isDisabled}
type='text'
/>
<TextSetting
id='connMaxLifetimeMilliseconds'
label={
webapp/channels/src/i18n/en.json
+3
View File
@@ -2912,6 +2912,9 @@
"admin.site.posts": "Posts",
"admin.site.public_links": "Public Links",
"admin.site.usersAndTeams": "Users and Teams",
"admin.sql.analyticsQueryTimeoutDescription": "The number of seconds to wait for a response from the database after opening a connection and sending certain analytics queries. This setting only applies to long queries which are run in the background to populate some information in the Team and Site Statistics pages.",
"admin.sql.analyticsQueryTimeoutExample": "E.g.: \"300\"",
"admin.sql.analyticsQueryTimeoutTitle": "Analytics Query Timeout:",
"admin.sql.connMaxIdleTimeDescription": "Maximum idle time for a connection to the database in milliseconds.",
"admin.sql.connMaxIdleTimeExample": "E.g.: \"300000\"",
"admin.sql.connMaxIdleTimeTitle": "Maximum Connection Idle Time:",
webapp/platform/types/src/config.ts
+1
View File
@@ -486,6 +486,7 @@ export type SqlSettings = {
Trace: boolean;
AtRestEncryptKey: string;
QueryTimeout: number;
AnalyticsQueryTimeout: number;
DisableDatabaseSearch: boolean;
MigrationsStatementTimeoutSeconds: number;
ReplicaLagSettings: ReplicaLagSetting[];