From 41e88b74ac90db4e1afd22902827cbca48959dcc Mon Sep 17 00:00:00 2001 From: Agniva De Sarker Date: Thu, 31 Jul 2025 09:35:29 +0530 Subject: [PATCH] MM-61407: Remove Bleve (#33430) https://mattermost.atlassian.net/browse/MM-61407 ```release-note NONE ``` * webapp i18n ```release-note NONE ``` * Fix e2e tests ```release-note NONE ``` * fix roles in e2e tests ```release-note NONE ``` * some review comments ```release-note NONE ``` * add back permissions to deprecated list ```release-note NONE ``` --- .../fixtures/console-example-inputs.json | 11 +- .../fixtures/system-roles-console-access.json | 8 +- .../system_console/search_box_spec.js | 4 +- .../support/api/on_prem_default_config.json | 7 - e2e-tests/cypress/tests/support/api/role.js | 4 +- .../cypress/tests/utils/admin_console.js | 6 - .../lib/src/server/default_config.ts | 7 - server/.golangci.yml | 3 - server/channels/api4/api.go | 4 - server/channels/api4/apitestlib.go | 4 - server/channels/api4/bleve.go | 33 - server/channels/api4/bleve_test.go | 47 - server/channels/app/job.go | 5 - server/channels/app/job_test.go | 27 +- server/channels/app/permissions_migrations.go | 13 +- server/channels/app/platform/searchengine.go | 5 - server/channels/app/platform/service.go | 6 - server/channels/app/searchengine.go | 12 - server/channels/app/server.go | 8 - server/channels/app/support_packet.go | 4 - server/channels/app/support_packet_test.go | 8 +- .../store/searchtest/channel_layer.go | 10 +- .../store/searchtest/file_info_layer.go | 2 +- .../channels/store/searchtest/post_layer.go | 2 +- server/channels/store/searchtest/testlib.go | 1 - .../channels/store/searchtest/user_layer.go | 8 +- .../testdata/mysql_migration_warmup.sql | 4 +- .../testdata/postgres_migration_warmup.sql | 4 +- server/go.mod | 25 - server/go.sum | 51 - server/i18n/en.json | 200 ---- .../searchengine/bleveengine/bleve.go | 337 ------- .../searchengine/bleveengine/bleve_test.go | 235 ----- .../searchengine/bleveengine/common.go | 163 ---- .../bleveengine/indexer/indexing_job.go | 638 ------------- .../bleveengine/indexer/indexing_job_test.go | 75 -- .../searchengine/bleveengine/search.go | 880 ------------------ .../searchengine/bleveengine/testlib.go | 23 - .../services/searchengine/searchengine.go | 12 - .../searchengine/searchengine_test.go | 9 - .../platform/services/telemetry/telemetry.go | 8 - server/public/model/client4.go | 16 - server/public/model/config.go | 61 -- server/public/model/config_test.go | 3 +- server/public/model/job.go | 2 - server/public/model/permission.go | 16 +- server/public/model/role.go | 6 - server/public/model/support_packet.go | 1 - server/public/model/utils_test.go | 3 +- webapp/channels/src/actions/admin_actions.jsx | 14 - .../bleve_settings.test.tsx.snap | 387 -------- .../database_settings.test.tsx.snap | 2 +- .../admin_console/admin_definition.tsx | 15 - .../__snapshots__/admin_sidebar.test.tsx.snap | 99 -- .../admin_console/bleve_settings.test.tsx | 51 - .../admin_console/bleve_settings.tsx | 243 ----- .../admin_console/database_settings.tsx | 2 +- .../system_role_permissions.test.tsx.snap | 6 - .../system_roles/system_role/strings.tsx | 6 - .../system_role/system_role_permissions.tsx | 1 - webapp/channels/src/i18n/en.json | 20 - webapp/platform/types/src/config.ts | 9 - 62 files changed, 41 insertions(+), 3835 deletions(-) delete mode 100644 server/channels/api4/bleve.go delete mode 100644 server/channels/api4/bleve_test.go delete mode 100644 server/platform/services/searchengine/bleveengine/bleve.go delete mode 100644 server/platform/services/searchengine/bleveengine/bleve_test.go delete mode 100644 server/platform/services/searchengine/bleveengine/common.go delete mode 100644 server/platform/services/searchengine/bleveengine/indexer/indexing_job.go delete mode 100644 server/platform/services/searchengine/bleveengine/indexer/indexing_job_test.go delete mode 100644 server/platform/services/searchengine/bleveengine/search.go delete mode 100644 server/platform/services/searchengine/bleveengine/testlib.go delete mode 100644 webapp/channels/src/components/admin_console/__snapshots__/bleve_settings.test.tsx.snap delete mode 100644 webapp/channels/src/components/admin_console/bleve_settings.test.tsx delete mode 100644 webapp/channels/src/components/admin_console/bleve_settings.tsx diff --git a/e2e-tests/cypress/tests/fixtures/console-example-inputs.json b/e2e-tests/cypress/tests/fixtures/console-example-inputs.json index 70dfa7fd62f..e8b51da3e60 100644 --- a/e2e-tests/cypress/tests/fixtures/console-example-inputs.json +++ b/e2e-tests/cypress/tests/fixtures/console-example-inputs.json @@ -412,14 +412,5 @@ "selector": "" } ] - }, - { - "section": "experimental.bleve", - "disabledInputs": [ - { - "path": "/admin_console/experimental/blevesearch", - "selector": "indexDirinput" - } - ] } -] \ No newline at end of file +] diff --git a/e2e-tests/cypress/tests/fixtures/system-roles-console-access.json b/e2e-tests/cypress/tests/fixtures/system-roles-console-access.json index 739c5ecefe4..751e3ff2ce8 100644 --- a/e2e-tests/cypress/tests/fixtures/system-roles-console-access.json +++ b/e2e-tests/cypress/tests/fixtures/system-roles-console-access.json @@ -310,11 +310,5 @@ "system_manager": "none", "system_user_manager": "none", "system_read_only_admin": "read" - }, - { - "section": "experimental.bleve", - "system_manager": "none", - "system_user_manager": "none", - "system_read_only_admin": "read" } -] \ No newline at end of file +] diff --git a/e2e-tests/cypress/tests/integration/channels/enterprise/system_console/search_box_spec.js b/e2e-tests/cypress/tests/integration/channels/enterprise/system_console/search_box_spec.js index 8bb483aa2ab..9bb392b2bc2 100644 --- a/e2e-tests/cypress/tests/integration/channels/enterprise/system_console/search_box_spec.js +++ b/e2e-tests/cypress/tests/integration/channels/enterprise/system_console/search_box_spec.js @@ -46,14 +46,14 @@ describe('System console', () => { cy.visit('/admin_console'); // * Scroll to the last item of the page and ensure it can be clicked - cy.findByTestId('experimental.bleve').scrollIntoView().click(); + cy.findByTestId('experimental.feature_flags').scrollIntoView().click(); }); it('MM-T1634 - Search box should remain visible / in the header as you scroll down the settings list in the left-hand-side', () => { cy.visit('/admin_console'); // * Scroll to bottom of left hand side - cy.findByTestId('experimental.bleve').scrollIntoView().click(); + cy.findByTestId('experimental.feature_flags').scrollIntoView().click(); // * To check if the sidebar is in view, try to click it cy.get('#adminSidebarFilter').should('be.visible').click(); diff --git a/e2e-tests/cypress/tests/support/api/on_prem_default_config.json b/e2e-tests/cypress/tests/support/api/on_prem_default_config.json index 064768f2833..13dec8e655b 100644 --- a/e2e-tests/cypress/tests/support/api/on_prem_default_config.json +++ b/e2e-tests/cypress/tests/support/api/on_prem_default_config.json @@ -521,13 +521,6 @@ "Trace": "", "IgnoredPurgeIndexes": "" }, - "BleveSettings": { - "IndexDir": "", - "EnableIndexing": false, - "EnableSearching": false, - "EnableAutocomplete": false, - "BatchSize": 10000 - }, "DataRetentionSettings": { "EnableMessageDeletion": false, "EnableFileDeletion": false, diff --git a/e2e-tests/cypress/tests/support/api/role.js b/e2e-tests/cypress/tests/support/api/role.js index 4180581d36f..61ccbfed028 100644 --- a/e2e-tests/cypress/tests/support/api/role.js +++ b/e2e-tests/cypress/tests/support/api/role.js @@ -17,13 +17,13 @@ export const defaultRolesPermissions = { playbook_member: 'playbook_public_view playbook_public_manage_members playbook_public_manage_properties playbook_private_view playbook_private_manage_members playbook_private_manage_properties run_create', run_admin: 'run_manage_properties run_manage_members', run_member: 'run_view', - system_admin: 'sysconsole_write_environment_elasticsearch playbook_public_manage_properties sysconsole_write_authentication_ldap run_view manage_jobs manage_roles playbook_public_create manage_public_channel_properties sysconsole_read_plugins delete_post purge_elasticsearch_indexes sysconsole_read_integrations_bot_accounts read_data_retention_job manage_private_channel_members create_elasticsearch_post_indexing_job manage_elasticsearch_post_indexing_job sysconsole_read_authentication_guest_access create_elasticsearch_post_aggregation_job manage_elasticsearch_post_aggregation_job join_public_teams sysconsole_read_site_public_links add_saml_idp_cert sysconsole_write_site_announcement_banner sysconsole_write_site_notices sysconsole_read_experimental_feature_flags sysconsole_read_site_users_and_teams manage_slash_commands sysconsole_read_authentication_ldap read_channel read_channel_content sysconsole_write_authentication_password list_users_without_team sysconsole_read_authentication_email add_saml_public_cert playbook_private_create promote_guest sysconsole_read_user_management_system_roles manage_public_channel_members create_data_retention_job manage_data_retention_job add_saml_private_cert sysconsole_write_user_management_users sysconsole_read_compliance_compliance_monitoring playbook_public_manage_members sysconsole_write_environment_database sysconsole_write_user_management_teams playbook_private_manage_roles read_public_channel sysconsole_write_plugins sysconsole_read_authentication_openid sysconsole_write_user_management_groups sysconsole_write_site_file_sharing_and_downloads playbook_private_manage_properties sysconsole_read_site_customization join_public_channels add_user_to_team restore_custom_group download_compliance_export_result sysconsole_write_user_management_system_roles sysconsole_write_environment_session_lengths create_custom_group manage_private_channel_properties create_post_public remove_ldap_private_cert sysconsole_write_site_public_links import_team sysconsole_read_environment_developer sysconsole_read_environment_database sysconsole_read_environment_web_server sysconsole_read_environment_mobile_security sysconsole_write_environment_mobile_security use_channel_mentions view_team remove_others_reactions sysconsole_read_environment_session_lengths sysconsole_write_integrations_bot_accounts playbook_public_view use_group_mentions sysconsole_write_environment_web_server add_ldap_private_cert read_public_channel_groups invite_guest sysconsole_read_environment_smtp create_post sysconsole_read_about_edition_and_license sysconsole_read_authentication_signup sysconsole_read_authentication_saml sysconsole_read_environment_file_storage sysconsole_write_experimental_feature_flags sysconsole_write_site_localization sysconsole_write_environment_rate_limiting sysconsole_read_environment_rate_limiting sysconsole_read_products_boards get_saml_cert_status sysconsole_read_environment_high_availability manage_secure_connections read_compliance_export_job sysconsole_write_compliance_custom_terms_of_service read_user_access_token edit_post sysconsole_write_environment_logging sysconsole_read_environment_push_notification_server sysconsole_write_site_customization read_other_users_teams read_elasticsearch_post_aggregation_job sysconsole_write_compliance_data_retention_policy sysconsole_read_user_management_permissions sysconsole_read_site_emoji sysconsole_read_compliance_data_retention_policy read_license_information sysconsole_read_experimental_features read_deleted_posts sysconsole_read_environment_logging sysconsole_read_reporting_site_statistics test_elasticsearch sysconsole_read_site_posts add_reaction sysconsole_write_authentication_signup manage_outgoing_webhooks create_post_ephemeral sysconsole_read_environment_image_proxy invite_user manage_others_outgoing_webhooks create_user_access_token sysconsole_write_environment_image_proxy sysconsole_write_products_boards read_elasticsearch_post_indexing_job purge_bleve_indexes sysconsole_write_environment_performance_monitoring sysconsole_write_authentication_guest_access sysconsole_read_compliance_custom_terms_of_service edit_others_posts sysconsole_write_billing get_saml_metadata_from_idp sysconsole_write_authentication_saml create_post_bleve_indexes_job manage_post_bleve_indexes_job invalidate_caches sysconsole_write_experimental_bleve view_members manage_others_bots run_create join_private_teams convert_private_channel_to_public read_audits assign_bot read_jobs remove_user_from_team revoke_user_access_token manage_team sysconsole_read_reporting_server_logs get_public_link manage_others_slash_commands manage_system delete_public_channel read_private_channel_groups sysconsole_read_authentication_mfa delete_emojis list_private_teams create_emojis sysconsole_read_billing sysconsole_write_site_emoji invalidate_email_invite sysconsole_write_environment_file_storage sysconsole_write_compliance_compliance_monitoring remove_saml_public_cert sysconsole_read_compliance_compliance_export sysconsole_read_site_localization manage_team_roles list_public_teams get_logs sysconsole_write_integrations_integration_management sysconsole_read_integrations_cors manage_oauth manage_outgoing_oauth_connections delete_others_emojis sysconsole_write_integrations_gif manage_incoming_webhooks sysconsole_write_authentication_email create_private_channel playbook_private_make_public manage_bots add_ldap_public_cert remove_ldap_public_cert sysconsole_write_site_notifications sysconsole_write_environment_developer playbook_private_manage_members sysconsole_read_user_management_teams edit_custom_group remove_reaction playbook_public_manage_roles sysconsole_write_reporting_server_logs read_others_bots sysconsole_write_site_posts sysconsole_read_site_notifications sysconsole_read_authentication_password playbook_private_view manage_system_wide_oauth get_analytics list_team_channels sysconsole_write_user_management_channels delete_private_channel manage_custom_group_members test_s3 create_ldap_sync_job manage_ldap_sync_job sysconsole_read_integrations_integration_management test_site_url recycle_database_connections sysconsole_read_site_announcement_banner test_email manage_shared_channels read_bots sysconsole_write_environment_smtp sysconsole_read_experimental_bleve sysconsole_write_environment_push_notification_server sysconsole_write_user_management_permissions sysconsole_read_environment_elasticsearch sysconsole_write_reporting_site_statistics sysconsole_write_site_users_and_teams demote_to_guest create_team test_ldap remove_saml_idp_cert delete_others_posts edit_other_users sysconsole_write_reporting_team_statistics sysconsole_read_integrations_gif sysconsole_read_site_notices sysconsole_write_about_edition_and_license manage_others_incoming_webhooks run_manage_members create_bot sysconsole_write_authentication_mfa sysconsole_read_user_management_users assign_system_admin_role sysconsole_write_experimental_features edit_brand create_group_channel sysconsole_write_authentication_openid create_direct_channel manage_license_information reload_config manage_channel_roles sysconsole_read_user_management_groups create_compliance_export_job manage_compliance_export_job read_ldap_sync_job upload_file sysconsole_read_site_file_sharing_and_downloads delete_custom_group sysconsole_read_user_management_channels sysconsole_write_compliance_compliance_export remove_saml_private_cert sysconsole_read_environment_performance_monitoring create_public_channel sysconsole_write_integrations_cors sysconsole_write_environment_high_availability playbook_public_make_private run_manage_properties sysconsole_read_reporting_team_statistics convert_public_channel_to_private add_bookmark_public_channel edit_bookmark_public_channel delete_bookmark_public_channel order_bookmark_public_channel add_bookmark_private_channel edit_bookmark_private_channel delete_bookmark_private_channel order_bookmark_private_channel', + system_admin: 'sysconsole_write_environment_elasticsearch playbook_public_manage_properties sysconsole_write_authentication_ldap run_view manage_jobs manage_roles playbook_public_create manage_public_channel_properties sysconsole_read_plugins delete_post purge_elasticsearch_indexes sysconsole_read_integrations_bot_accounts read_data_retention_job manage_private_channel_members create_elasticsearch_post_indexing_job manage_elasticsearch_post_indexing_job sysconsole_read_authentication_guest_access create_elasticsearch_post_aggregation_job manage_elasticsearch_post_aggregation_job join_public_teams sysconsole_read_site_public_links add_saml_idp_cert sysconsole_write_site_announcement_banner sysconsole_write_site_notices sysconsole_read_experimental_feature_flags sysconsole_read_site_users_and_teams manage_slash_commands sysconsole_read_authentication_ldap read_channel read_channel_content sysconsole_write_authentication_password list_users_without_team sysconsole_read_authentication_email add_saml_public_cert playbook_private_create promote_guest sysconsole_read_user_management_system_roles manage_public_channel_members create_data_retention_job manage_data_retention_job add_saml_private_cert sysconsole_write_user_management_users sysconsole_read_compliance_compliance_monitoring playbook_public_manage_members sysconsole_write_environment_database sysconsole_write_user_management_teams playbook_private_manage_roles read_public_channel sysconsole_write_plugins sysconsole_read_authentication_openid sysconsole_write_user_management_groups sysconsole_write_site_file_sharing_and_downloads playbook_private_manage_properties sysconsole_read_site_customization join_public_channels add_user_to_team restore_custom_group download_compliance_export_result sysconsole_write_user_management_system_roles sysconsole_write_environment_session_lengths create_custom_group manage_private_channel_properties create_post_public remove_ldap_private_cert sysconsole_write_site_public_links import_team sysconsole_read_environment_developer sysconsole_read_environment_database sysconsole_read_environment_web_server sysconsole_read_environment_mobile_security sysconsole_write_environment_mobile_security use_channel_mentions view_team remove_others_reactions sysconsole_read_environment_session_lengths sysconsole_write_integrations_bot_accounts playbook_public_view use_group_mentions sysconsole_write_environment_web_server add_ldap_private_cert read_public_channel_groups invite_guest sysconsole_read_environment_smtp create_post sysconsole_read_about_edition_and_license sysconsole_read_authentication_signup sysconsole_read_authentication_saml sysconsole_read_environment_file_storage sysconsole_write_experimental_feature_flags sysconsole_write_site_localization sysconsole_write_environment_rate_limiting sysconsole_read_environment_rate_limiting sysconsole_read_products_boards get_saml_cert_status sysconsole_read_environment_high_availability manage_secure_connections read_compliance_export_job sysconsole_write_compliance_custom_terms_of_service read_user_access_token edit_post sysconsole_write_environment_logging sysconsole_read_environment_push_notification_server sysconsole_write_site_customization read_other_users_teams read_elasticsearch_post_aggregation_job sysconsole_write_compliance_data_retention_policy sysconsole_read_user_management_permissions sysconsole_read_site_emoji sysconsole_read_compliance_data_retention_policy read_license_information sysconsole_read_experimental_features read_deleted_posts sysconsole_read_environment_logging sysconsole_read_reporting_site_statistics test_elasticsearch sysconsole_read_site_posts add_reaction sysconsole_write_authentication_signup manage_outgoing_webhooks create_post_ephemeral sysconsole_read_environment_image_proxy invite_user manage_others_outgoing_webhooks create_user_access_token sysconsole_write_environment_image_proxy sysconsole_write_products_boards read_elasticsearch_post_indexing_job sysconsole_write_environment_performance_monitoring sysconsole_write_authentication_guest_access sysconsole_read_compliance_custom_terms_of_service edit_others_posts sysconsole_write_billing get_saml_metadata_from_idp sysconsole_write_authentication_saml invalidate_caches view_members manage_others_bots run_create join_private_teams convert_private_channel_to_public read_audits assign_bot read_jobs remove_user_from_team revoke_user_access_token manage_team sysconsole_read_reporting_server_logs get_public_link manage_others_slash_commands manage_system delete_public_channel read_private_channel_groups sysconsole_read_authentication_mfa delete_emojis list_private_teams create_emojis sysconsole_read_billing sysconsole_write_site_emoji invalidate_email_invite sysconsole_write_environment_file_storage sysconsole_write_compliance_compliance_monitoring remove_saml_public_cert sysconsole_read_compliance_compliance_export sysconsole_read_site_localization manage_team_roles list_public_teams get_logs sysconsole_write_integrations_integration_management sysconsole_read_integrations_cors manage_oauth manage_outgoing_oauth_connections delete_others_emojis sysconsole_write_integrations_gif manage_incoming_webhooks sysconsole_write_authentication_email create_private_channel playbook_private_make_public manage_bots add_ldap_public_cert remove_ldap_public_cert sysconsole_write_site_notifications sysconsole_write_environment_developer playbook_private_manage_members sysconsole_read_user_management_teams edit_custom_group remove_reaction playbook_public_manage_roles sysconsole_write_reporting_server_logs read_others_bots sysconsole_write_site_posts sysconsole_read_site_notifications sysconsole_read_authentication_password playbook_private_view manage_system_wide_oauth get_analytics list_team_channels sysconsole_write_user_management_channels delete_private_channel manage_custom_group_members test_s3 create_ldap_sync_job manage_ldap_sync_job sysconsole_read_integrations_integration_management test_site_url recycle_database_connections sysconsole_read_site_announcement_banner test_email manage_shared_channels read_bots sysconsole_write_environment_smtp sysconsole_write_environment_push_notification_server sysconsole_write_user_management_permissions sysconsole_read_environment_elasticsearch sysconsole_write_reporting_site_statistics sysconsole_write_site_users_and_teams demote_to_guest create_team test_ldap remove_saml_idp_cert delete_others_posts edit_other_users sysconsole_write_reporting_team_statistics sysconsole_read_integrations_gif sysconsole_read_site_notices sysconsole_write_about_edition_and_license manage_others_incoming_webhooks run_manage_members create_bot sysconsole_write_authentication_mfa sysconsole_read_user_management_users assign_system_admin_role sysconsole_write_experimental_features edit_brand create_group_channel sysconsole_write_authentication_openid create_direct_channel manage_license_information reload_config manage_channel_roles sysconsole_read_user_management_groups create_compliance_export_job manage_compliance_export_job read_ldap_sync_job upload_file sysconsole_read_site_file_sharing_and_downloads delete_custom_group sysconsole_read_user_management_channels sysconsole_write_compliance_compliance_export remove_saml_private_cert sysconsole_read_environment_performance_monitoring create_public_channel sysconsole_write_integrations_cors sysconsole_write_environment_high_availability playbook_public_make_private run_manage_properties sysconsole_read_reporting_team_statistics convert_public_channel_to_private add_bookmark_public_channel edit_bookmark_public_channel delete_bookmark_public_channel order_bookmark_public_channel add_bookmark_private_channel edit_bookmark_private_channel delete_bookmark_private_channel order_bookmark_private_channel', system_custom_group_admin: 'create_custom_group edit_custom_group delete_custom_group restore_custom_group manage_custom_group_members', system_guest: 'create_group_channel create_direct_channel', system_manager: 'sysconsole_read_site_announcement_banner manage_private_channel_properties edit_brand read_private_channel_groups manage_private_channel_members manage_team_roles sysconsole_write_environment_session_lengths sysconsole_read_site_emoji sysconsole_write_environment_developer sysconsole_read_user_management_groups sysconsole_write_user_management_groups sysconsole_write_environment_rate_limiting delete_private_channel sysconsole_read_environment_performance_monitoring sysconsole_read_environment_rate_limiting sysconsole_write_user_management_teams sysconsole_write_integrations_integration_management sysconsole_write_site_public_links sysconsole_read_authentication_ldap sysconsole_write_integrations_cors reload_config sysconsole_write_user_management_channels sysconsole_read_environment_high_availability sysconsole_read_site_users_and_teams sysconsole_read_user_management_teams sysconsole_write_site_users_and_teams sysconsole_read_site_customization sysconsole_write_environment_high_availability sysconsole_read_integrations_bot_accounts sysconsole_read_authentication_guest_access sysconsole_read_site_public_links read_elasticsearch_post_indexing_job sysconsole_read_user_management_channels sysconsole_read_reporting_team_statistics invalidate_caches sysconsole_read_authentication_signup read_elasticsearch_post_aggregation_job sysconsole_write_environment_smtp manage_public_channel_members list_public_teams add_user_to_team sysconsole_read_environment_web_server sysconsole_read_site_localization get_logs sysconsole_write_site_posts sysconsole_write_integrations_bot_accounts sysconsole_write_user_management_permissions sysconsole_read_environment_elasticsearch sysconsole_read_environment_smtp list_private_teams read_public_channel_groups sysconsole_write_environment_file_storage sysconsole_write_integrations_gif manage_public_channel_properties sysconsole_write_environment_performance_monitoring sysconsole_write_site_notifications sysconsole_read_site_notifications sysconsole_read_environment_image_proxy sysconsole_write_site_announcement_banner sysconsole_write_site_emoji test_site_url sysconsole_read_integrations_gif sysconsole_write_environment_logging convert_public_channel_to_private get_analytics sysconsole_read_user_management_permissions sysconsole_write_environment_image_proxy test_elasticsearch recycle_database_connections sysconsole_write_site_localization sysconsole_read_reporting_server_logs create_elasticsearch_post_indexing_job manage_elasticsearch_post_indexing_job sysconsole_read_reporting_site_statistics test_ldap delete_public_channel sysconsole_write_environment_push_notification_server read_license_information sysconsole_write_products_boards sysconsole_read_about_edition_and_license convert_private_channel_to_public sysconsole_read_integrations_integration_management create_elasticsearch_post_aggregation_job manage_elasticsearch_post_aggregation_job purge_elasticsearch_indexes sysconsole_read_environment_database join_public_teams sysconsole_read_authentication_email sysconsole_read_environment_push_notification_server view_team read_channel sysconsole_read_authentication_password read_ldap_sync_job sysconsole_read_integrations_cors sysconsole_read_environment_logging manage_team sysconsole_read_authentication_openid read_public_channel sysconsole_write_environment_elasticsearch sysconsole_read_plugins manage_channel_roles remove_user_from_team test_email sysconsole_write_site_file_sharing_and_downloads test_s3 sysconsole_read_site_file_sharing_and_downloads sysconsole_read_site_notices sysconsole_read_environment_file_storage join_private_teams sysconsole_read_products_boards sysconsole_read_environment_session_lengths sysconsole_write_environment_database sysconsole_read_authentication_saml sysconsole_read_authentication_mfa sysconsole_write_site_notices sysconsole_write_environment_web_server sysconsole_read_site_posts sysconsole_read_environment_developer sysconsole_write_site_customization sysconsole_read_environment_mobile_security sysconsole_write_environment_mobile_security manage_outgoing_oauth_connections', system_post_all: 'use_group_mentions use_channel_mentions create_post', system_post_all_public: 'use_group_mentions use_channel_mentions create_post_public', - system_read_only_admin: 'sysconsole_read_authentication_guest_access download_compliance_export_result sysconsole_read_compliance_data_retention_policy get_logs sysconsole_read_environment_file_storage read_channel sysconsole_read_integrations_integration_management sysconsole_read_compliance_custom_terms_of_service sysconsole_read_site_notices sysconsole_read_environment_rate_limiting sysconsole_read_about_edition_and_license read_public_channel sysconsole_read_experimental_features test_ldap sysconsole_read_user_management_permissions read_elasticsearch_post_aggregation_job sysconsole_read_environment_image_proxy sysconsole_read_compliance_compliance_export sysconsole_read_integrations_bot_accounts sysconsole_read_authentication_openid sysconsole_read_site_posts sysconsole_read_user_management_users sysconsole_read_experimental_feature_flags sysconsole_read_reporting_team_statistics sysconsole_read_site_localization read_private_channel_groups sysconsole_read_site_file_sharing_and_downloads sysconsole_read_user_management_channels sysconsole_read_authentication_email read_data_retention_job read_audits sysconsole_read_plugins view_team get_analytics sysconsole_read_user_management_groups sysconsole_read_experimental_bleve sysconsole_read_products_boards read_compliance_export_job sysconsole_read_environment_logging sysconsole_read_authentication_signup sysconsole_read_environment_smtp sysconsole_read_environment_session_lengths sysconsole_read_environment_developer sysconsole_read_environment_high_availability sysconsole_read_environment_mobile_security read_ldap_sync_job sysconsole_read_environment_performance_monitoring sysconsole_read_authentication_saml read_public_channel_groups sysconsole_read_integrations_gif sysconsole_read_authentication_mfa list_public_teams sysconsole_read_environment_database list_private_teams sysconsole_read_authentication_ldap sysconsole_read_compliance_compliance_monitoring sysconsole_read_site_notifications sysconsole_read_site_announcement_banner read_other_users_teams sysconsole_read_authentication_password sysconsole_read_environment_push_notification_server sysconsole_read_site_users_and_teams sysconsole_read_site_public_links sysconsole_read_site_emoji sysconsole_read_environment_elasticsearch read_license_information sysconsole_read_integrations_cors sysconsole_read_user_management_teams sysconsole_read_reporting_server_logs sysconsole_read_site_customization sysconsole_read_reporting_site_statistics sysconsole_read_environment_web_server read_elasticsearch_post_indexing_job', + system_read_only_admin: 'sysconsole_read_authentication_guest_access download_compliance_export_result sysconsole_read_compliance_data_retention_policy get_logs sysconsole_read_environment_file_storage read_channel sysconsole_read_integrations_integration_management sysconsole_read_compliance_custom_terms_of_service sysconsole_read_site_notices sysconsole_read_environment_rate_limiting sysconsole_read_about_edition_and_license read_public_channel sysconsole_read_experimental_features test_ldap sysconsole_read_user_management_permissions read_elasticsearch_post_aggregation_job sysconsole_read_environment_image_proxy sysconsole_read_compliance_compliance_export sysconsole_read_integrations_bot_accounts sysconsole_read_authentication_openid sysconsole_read_site_posts sysconsole_read_user_management_users sysconsole_read_experimental_feature_flags sysconsole_read_reporting_team_statistics sysconsole_read_site_localization read_private_channel_groups sysconsole_read_site_file_sharing_and_downloads sysconsole_read_user_management_channels sysconsole_read_authentication_email read_data_retention_job read_audits sysconsole_read_plugins view_team get_analytics sysconsole_read_user_management_groups sysconsole_read_products_boards read_compliance_export_job sysconsole_read_environment_logging sysconsole_read_authentication_signup sysconsole_read_environment_smtp sysconsole_read_environment_session_lengths sysconsole_read_environment_developer sysconsole_read_environment_high_availability sysconsole_read_environment_mobile_security read_ldap_sync_job sysconsole_read_environment_performance_monitoring sysconsole_read_authentication_saml read_public_channel_groups sysconsole_read_integrations_gif sysconsole_read_authentication_mfa list_public_teams sysconsole_read_environment_database list_private_teams sysconsole_read_authentication_ldap sysconsole_read_compliance_compliance_monitoring sysconsole_read_site_notifications sysconsole_read_site_announcement_banner read_other_users_teams sysconsole_read_authentication_password sysconsole_read_environment_push_notification_server sysconsole_read_site_users_and_teams sysconsole_read_site_public_links sysconsole_read_site_emoji sysconsole_read_environment_elasticsearch read_license_information sysconsole_read_integrations_cors sysconsole_read_user_management_teams sysconsole_read_reporting_server_logs sysconsole_read_site_customization sysconsole_read_reporting_site_statistics sysconsole_read_environment_web_server read_elasticsearch_post_indexing_job', system_user: 'delete_custom_group create_emojis edit_custom_group create_direct_channel view_members join_public_teams restore_custom_group create_custom_group manage_custom_group_members delete_emojis list_public_teams create_team create_group_channel', system_user_access_token: 'create_user_access_token read_user_access_token revoke_user_access_token', system_user_manager: 'sysconsole_read_authentication_password sysconsole_read_authentication_openid sysconsole_write_user_management_groups list_private_teams sysconsole_read_user_management_groups sysconsole_read_authentication_email manage_public_channel_properties delete_private_channel sysconsole_read_authentication_signup read_private_channel_groups sysconsole_read_user_management_teams test_ldap read_channel view_team manage_team sysconsole_write_user_management_teams manage_channel_roles sysconsole_read_authentication_saml sysconsole_read_authentication_guest_access convert_private_channel_to_public sysconsole_read_user_management_permissions join_public_teams sysconsole_write_user_management_channels read_public_channel_groups sysconsole_read_user_management_channels list_public_teams manage_team_roles join_private_teams manage_public_channel_members convert_public_channel_to_private remove_user_from_team sysconsole_read_authentication_ldap manage_private_channel_properties delete_public_channel manage_private_channel_members read_public_channel add_user_to_team sysconsole_read_authentication_mfa read_ldap_sync_job', diff --git a/e2e-tests/cypress/tests/utils/admin_console.js b/e2e-tests/cypress/tests/utils/admin_console.js index 670f6cd2842..b18c0ffc406 100644 --- a/e2e-tests/cypress/tests/utils/admin_console.js +++ b/e2e-tests/cypress/tests/utils/admin_console.js @@ -359,10 +359,4 @@ export const adminConsoleNavigation = [ sidebar: 'Feature Flags', url: 'admin_console/experimental/feature_flags', }, - { - type: ['team', 'e20'], - header: 'Bleve', - sidebar: 'Bleve', - url: 'admin_console/experimental/blevesearch', - }, ]; diff --git a/e2e-tests/playwright/lib/src/server/default_config.ts b/e2e-tests/playwright/lib/src/server/default_config.ts index 920cbd71b63..9f793718a83 100644 --- a/e2e-tests/playwright/lib/src/server/default_config.ts +++ b/e2e-tests/playwright/lib/src/server/default_config.ts @@ -646,13 +646,6 @@ const defaultServerConfig: AdminConfig = { Trace: '', IgnoredPurgeIndexes: '', }, - BleveSettings: { - IndexDir: '', - EnableIndexing: false, - EnableSearching: false, - EnableAutocomplete: false, - BatchSize: 10000, - }, DataRetentionSettings: { EnableMessageDeletion: false, EnableFileDeletion: false, diff --git a/server/.golangci.yml b/server/.golangci.yml index 2536d8a68f1..1f22b2151b2 100644 --- a/server/.golangci.yml +++ b/server/.golangci.yml @@ -156,9 +156,6 @@ linters: platform/services/remotecluster/send_test.go|\ platform/services/remotecluster/sendfile.go|\ platform/services/remotecluster/sendprofileImage_test.go|\ - platform/services/searchengine/bleveengine/bleve_test.go|\ - platform/services/searchengine/bleveengine/common.go|\ - platform/services/searchengine/bleveengine/indexer/indexing_job.go|\ platform/services/sharedchannel/attachment.go|\ platform/services/sharedchannel/channelinvite.go|\ platform/services/sharedchannel/permalink_test.go|\ diff --git a/server/channels/api4/api.go b/server/channels/api4/api.go index 5794e924358..ab22a3884b0 100644 --- a/server/channels/api4/api.go +++ b/server/channels/api4/api.go @@ -93,8 +93,6 @@ type Routes struct { Elasticsearch *mux.Router // 'api/v4/elasticsearch' - Bleve *mux.Router // 'api/v4/bleve' - DataRetention *mux.Router // 'api/v4/data_retention' Brand *mux.Router // 'api/v4/brand' @@ -256,7 +254,6 @@ func Init(srv *app.Server) (*API, error) { api.BaseRoutes.Reactions = api.BaseRoutes.APIRoot.PathPrefix("/reactions").Subrouter() api.BaseRoutes.Jobs = api.BaseRoutes.APIRoot.PathPrefix("/jobs").Subrouter() api.BaseRoutes.Elasticsearch = api.BaseRoutes.APIRoot.PathPrefix("/elasticsearch").Subrouter() - api.BaseRoutes.Bleve = api.BaseRoutes.APIRoot.PathPrefix("/bleve").Subrouter() api.BaseRoutes.DataRetention = api.BaseRoutes.APIRoot.PathPrefix("/data_retention").Subrouter() api.BaseRoutes.Emojis = api.BaseRoutes.APIRoot.PathPrefix("/emoji").Subrouter() @@ -331,7 +328,6 @@ func Init(srv *app.Server) (*API, error) { api.InitCluster() api.InitLdap() api.InitElasticsearch() - api.InitBleve() api.InitDataRetention() api.InitBrand() api.InitJob() diff --git a/server/channels/api4/apitestlib.go b/server/channels/api4/apitestlib.go index 6e275982ec8..50f3cebb2f2 100644 --- a/server/channels/api4/apitestlib.go +++ b/server/channels/api4/apitestlib.go @@ -174,10 +174,6 @@ func setupTestHelper(tb testing.TB, dbStore store.Store, sqlSettings *model.SqlS Store: dbStore, } - if s.Platform().SearchEngine != nil && s.Platform().SearchEngine.BleveEngine != nil && searchEngine != nil { - searchEngine.BleveEngine = s.Platform().SearchEngine.BleveEngine - } - if searchEngine != nil { th.App.SetSearchEngine(searchEngine) } diff --git a/server/channels/api4/bleve.go b/server/channels/api4/bleve.go deleted file mode 100644 index f7700dd17cb..00000000000 --- a/server/channels/api4/bleve.go +++ /dev/null @@ -1,33 +0,0 @@ -// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. -// See LICENSE.txt for license information. - -package api4 - -import ( - "net/http" - - "github.com/mattermost/mattermost/server/public/model" -) - -func (api *API) InitBleve() { - api.BaseRoutes.Bleve.Handle("/purge_indexes", api.APISessionRequired(purgeBleveIndexes)).Methods(http.MethodPost) -} - -func purgeBleveIndexes(c *Context, w http.ResponseWriter, r *http.Request) { - auditRec := c.MakeAuditRecord(model.AuditEventPurgeBleveIndexes, model.AuditStatusFail) - defer c.LogAuditRec(auditRec) - - if !c.App.SessionHasPermissionToAndNotRestrictedAdmin(*c.AppContext.Session(), model.PermissionPurgeBleveIndexes) { - c.SetPermissionError(model.PermissionPurgeBleveIndexes) - return - } - - if err := c.App.PurgeBleveIndexes(c.AppContext); err != nil { - c.Err = err - return - } - - auditRec.Success() - - ReturnStatusOK(w) -} diff --git a/server/channels/api4/bleve_test.go b/server/channels/api4/bleve_test.go deleted file mode 100644 index 4e6866ff2fc..00000000000 --- a/server/channels/api4/bleve_test.go +++ /dev/null @@ -1,47 +0,0 @@ -// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. -// See LICENSE.txt for license information. - -package api4 - -import ( - "context" - "testing" - - "github.com/stretchr/testify/require" - - "github.com/mattermost/mattermost/server/public/model" -) - -func TestBlevePurgeIndexes(t *testing.T) { - mainHelper.Parallel(t) - th := Setup(t) - defer th.TearDown() - - t.Run("as system user", func(t *testing.T) { - resp, err := th.Client.PurgeBleveIndexes(context.Background()) - require.Error(t, err) - CheckForbiddenStatus(t, resp) - }) - - t.Run("as system user with write experimental permission", func(t *testing.T) { - th.AddPermissionToRole(model.PermissionPurgeBleveIndexes.Id, model.SystemUserRoleId) - defer th.RemovePermissionFromRole(model.PermissionSysconsoleWriteExperimental.Id, model.SystemUserRoleId) - resp, err := th.Client.PurgeBleveIndexes(context.Background()) - require.NoError(t, err) - CheckOKStatus(t, resp) - }) - - t.Run("as system admin", func(t *testing.T) { - resp, err := th.SystemAdminClient.PurgeBleveIndexes(context.Background()) - require.NoError(t, err) - CheckOKStatus(t, resp) - }) - - t.Run("as restricted system admin", func(t *testing.T) { - th.App.UpdateConfig(func(cfg *model.Config) { *cfg.ExperimentalSettings.RestrictSystemAdmin = true }) - - resp, err := th.SystemAdminClient.PurgeBleveIndexes(context.Background()) - require.Error(t, err) - CheckForbiddenStatus(t, resp) - }) -} diff --git a/server/channels/app/job.go b/server/channels/app/job.go index 6e62a040ce8..068e452c552 100644 --- a/server/channels/app/job.go +++ b/server/channels/app/job.go @@ -74,8 +74,6 @@ func (a *App) UpdateJobStatus(c request.CTX, job *model.Job, newStatus string) * func (a *App) SessionHasPermissionToCreateJob(session model.Session, job *model.Job) (bool, *model.Permission) { switch job.Type { - case model.JobTypeBlevePostIndexing: - return a.SessionHasPermissionTo(session, model.PermissionCreatePostBleveIndexesJob), model.PermissionCreatePostBleveIndexesJob case model.JobTypeDataRetention: return a.SessionHasPermissionTo(session, model.PermissionCreateDataRetentionJob), model.PermissionCreateDataRetentionJob case model.JobTypeMessageExport: @@ -110,8 +108,6 @@ func (a *App) SessionHasPermissionToManageJob(session model.Session, job *model. var permission *model.Permission switch job.Type { - case model.JobTypeBlevePostIndexing: - permission = model.PermissionManagePostBleveIndexesJob case model.JobTypeDataRetention: permission = model.PermissionManageDataRetentionJob case model.JobTypeMessageExport: @@ -159,7 +155,6 @@ func (a *App) SessionHasPermissionToReadJob(session model.Session, jobType strin case model.JobTypeLdapSync: return a.SessionHasPermissionTo(session, model.PermissionReadLdapSyncJob), model.PermissionReadLdapSyncJob case - model.JobTypeBlevePostIndexing, model.JobTypeMigrations, model.JobTypePlugins, model.JobTypeProductNotices, diff --git a/server/channels/app/job_test.go b/server/channels/app/job_test.go index 0cff6b96cbb..2f05576165a 100644 --- a/server/channels/app/job_test.go +++ b/server/channels/app/job_test.go @@ -42,11 +42,6 @@ func TestSessionHasPermissionToCreateJob(t *testing.T) { defer th.TearDown() jobs := []model.Job{ - { - Id: model.NewId(), - Type: model.JobTypeBlevePostIndexing, - CreateAt: 1000, - }, { Id: model.NewId(), Type: model.JobTypeDataRetention, @@ -65,14 +60,10 @@ func TestSessionHasPermissionToCreateJob(t *testing.T) { }{ { Job: jobs[0], - PermissionRequired: model.PermissionCreatePostBleveIndexesJob, - }, - { - Job: jobs[1], PermissionRequired: model.PermissionCreateDataRetentionJob, }, { - Job: jobs[2], + Job: jobs[1], PermissionRequired: model.PermissionCreateComplianceExportJob, }, } @@ -104,24 +95,10 @@ func TestSessionHasPermissionToCreateJob(t *testing.T) { ctx := sqlstore.WithMaster(context.Background()) role, _ := th.App.GetRoleByName(ctx, model.SystemReadOnlyAdminRoleId) - role.Permissions = append(role.Permissions, model.PermissionCreatePostBleveIndexesJob.Id) - - _, err := th.App.UpdateRole(role) - require.Nil(t, err) - - // Now system read only admin should have ability to create a Belve Post Index job but not the others - for _, testCase := range testCases { - hasPermission, permissionRequired := th.App.SessionHasPermissionToCreateJob(session, &testCase.Job) - expectedHasPermission := testCase.Job.Type == model.JobTypeBlevePostIndexing - assert.Equal(t, expectedHasPermission, hasPermission) - require.NotNil(t, permissionRequired) - assert.Equal(t, testCase.PermissionRequired.Id, permissionRequired.Id) - } - role.Permissions = append(role.Permissions, model.PermissionCreateDataRetentionJob.Id) role.Permissions = append(role.Permissions, model.PermissionCreateComplianceExportJob.Id) - _, err = th.App.UpdateRole(role) + _, err := th.App.UpdateRole(role) require.Nil(t, err) // Now system read only admin should have ability to create all jobs diff --git a/server/channels/app/permissions_migrations.go b/server/channels/app/permissions_migrations.go index e132a323b6f..623a38f66d3 100644 --- a/server/channels/app/permissions_migrations.go +++ b/server/channels/app/permissions_migrations.go @@ -614,17 +614,12 @@ func (a *App) getAddExperimentalSubsectionPermissions() (permissionsMap, error) // Give the new subsection READ permissions to any user with READ_EXPERIMENTAL permissionTransformation{ On: permissionExists(model.PermissionSysconsoleReadExperimental.Id), - Add: []string{model.PermissionSysconsoleReadExperimentalBleve.Id, model.PermissionSysconsoleReadExperimentalFeatures.Id, model.PermissionSysconsoleReadExperimentalFeatureFlags.Id}, + Add: []string{model.PermissionSysconsoleReadExperimentalFeatures.Id, model.PermissionSysconsoleReadExperimentalFeatureFlags.Id}, }, // Give the new subsection WRITE permissions to any user with WRITE_EXPERIMENTAL permissionTransformation{ On: permissionExists(model.PermissionSysconsoleWriteExperimental.Id), - Add: []string{model.PermissionSysconsoleWriteExperimentalBleve.Id, model.PermissionSysconsoleWriteExperimentalFeatures.Id, model.PermissionSysconsoleWriteExperimentalFeatureFlags.Id}, - }, - // Give the ancillary permissions MANAGE_JOBS and PURGE_BLEVE_INDEXES to anyone with WRITE_EXPERIMENTAL_BLEVE - permissionTransformation{ - On: permissionExists(model.PermissionSysconsoleWriteExperimentalBleve.Id), - Add: []string{model.PermissionCreatePostBleveIndexesJob.Id, model.PermissionPurgeBleveIndexes.Id}, + Add: []string{model.PermissionSysconsoleWriteExperimentalFeatures.Id, model.PermissionSysconsoleWriteExperimentalFeatureFlags.Id}, }, }, nil } @@ -1097,10 +1092,6 @@ func (a *App) getAddManageJobAncillaryPermissionsMigration() (permissionsMap, er On: permissionExists(model.PermissionSysconsoleWriteComplianceDataRetentionPolicy.Id), Add: []string{model.PermissionManageDataRetentionJob.Id}, }, - permissionTransformation{ - On: permissionExists(model.PermissionSysconsoleWriteExperimentalBleve.Id), - Add: []string{model.PermissionManagePostBleveIndexesJob.Id}, - }, permissionTransformation{ On: permissionExists(model.PermissionSysconsoleWriteComplianceComplianceExport.Id), Add: []string{model.PermissionManageComplianceExportJob.Id}, diff --git a/server/channels/app/platform/searchengine.go b/server/channels/app/platform/searchengine.go index c4499c279e6..f035da1d3ea 100644 --- a/server/channels/app/platform/searchengine.go +++ b/server/channels/app/platform/searchengine.go @@ -86,9 +86,4 @@ func (ps *PlatformService) StopSearchEngine() { ps.Log().Error("Failed to stop Elasticsearch engine", mlog.Err(err)) } } - if ps.SearchEngine != nil && ps.SearchEngine.BleveEngine != nil && ps.SearchEngine.BleveEngine.IsActive() { - if err := ps.SearchEngine.BleveEngine.Stop(); err != nil { - ps.Log().Error("Failed to stop Bleve Engine", mlog.Err(err)) - } - } } diff --git a/server/channels/app/platform/service.go b/server/channels/app/platform/service.go index 6d50300b3a6..799837dbd08 100644 --- a/server/channels/app/platform/service.go +++ b/server/channels/app/platform/service.go @@ -30,7 +30,6 @@ import ( "github.com/mattermost/mattermost/server/v8/einterfaces" "github.com/mattermost/mattermost/server/v8/platform/services/cache" "github.com/mattermost/mattermost/server/v8/platform/services/searchengine" - "github.com/mattermost/mattermost/server/v8/platform/services/searchengine/bleveengine" "github.com/mattermost/mattermost/server/v8/platform/shared/filestore" ) @@ -232,11 +231,6 @@ func New(sc ServiceConfig, options ...Option) (*PlatformService, error) { // Step 3: Search Engine searchEngine := searchengine.NewBroker(ps.Config()) - bleveEngine := bleveengine.NewBleveEngine(ps.Config()) - if err := bleveEngine.Start(); err != nil { - return nil, err - } - searchEngine.RegisterBleveEngine(bleveEngine) ps.SearchEngine = searchEngine // Step 4: Init Enterprise diff --git a/server/channels/app/searchengine.go b/server/channels/app/searchengine.go index 2df9937e565..15774d47c28 100644 --- a/server/channels/app/searchengine.go +++ b/server/channels/app/searchengine.go @@ -53,18 +53,6 @@ func (a *App) PurgeElasticsearchIndexes(c request.CTX, indexes []string) *model. return appErr } -func (a *App) PurgeBleveIndexes(c request.CTX) *model.AppError { - engine := a.SearchEngine().BleveEngine - if engine == nil { - err := model.NewAppError("PurgeBleveIndexes", "searchengine.bleve.disabled.error", nil, "", http.StatusNotImplemented) - return err - } - if err := engine.PurgeIndexes(c); err != nil { - return err - } - return nil -} - func (a *App) ActiveSearchBackend() string { return a.ch.srv.platform.SearchEngine.ActiveEngine() } diff --git a/server/channels/app/server.go b/server/channels/app/server.go index 80ddadb5bde..485ceb8dcd7 100644 --- a/server/channels/app/server.go +++ b/server/channels/app/server.go @@ -70,8 +70,6 @@ import ( "github.com/mattermost/mattermost/server/v8/platform/services/awsmeter" "github.com/mattermost/mattermost/server/v8/platform/services/cache" "github.com/mattermost/mattermost/server/v8/platform/services/remotecluster" - "github.com/mattermost/mattermost/server/v8/platform/services/searchengine/bleveengine" - "github.com/mattermost/mattermost/server/v8/platform/services/searchengine/bleveengine/indexer" "github.com/mattermost/mattermost/server/v8/platform/services/sharedchannel" "github.com/mattermost/mattermost/server/v8/platform/services/telemetry" "github.com/mattermost/mattermost/server/v8/platform/services/upgrader" @@ -1478,12 +1476,6 @@ func (s *Server) initJobs() { s.Jobs.RegisterJobType(model.JobTypeAccessControlSync, builder.MakeWorker(), builder.MakeScheduler()) } - s.Jobs.RegisterJobType( - model.JobTypeBlevePostIndexing, - indexer.MakeWorker(s.Jobs, s.platform.SearchEngine.BleveEngine.(*bleveengine.BleveEngine)), - nil, - ) - s.Jobs.RegisterJobType( model.JobTypeMigrations, migrations.MakeWorker(s.Jobs, s.Store()), diff --git a/server/channels/app/support_packet.go b/server/channels/app/support_packet.go index af0a30b4e89..612c4fd9cd8 100644 --- a/server/channels/app/support_packet.go +++ b/server/channels/app/support_packet.go @@ -260,10 +260,6 @@ func (a *App) getSupportPacketJobList(rctx request.CTX) (*model.FileData, error) if err != nil { rErr = multierror.Append(errors.Wrap(err, "error while getting ES post aggregation jobs")) } - jobs.BlevePostIndexingJobs, err = a.Srv().Store().Job().GetAllByTypePage(rctx, model.JobTypeBlevePostIndexing, 0, numberOfJobsRuns) - if err != nil { - rErr = multierror.Append(errors.Wrap(err, "error while getting bleve post indexing jobs")) - } jobs.MigrationJobs, err = a.Srv().Store().Job().GetAllByTypePage(rctx, model.JobTypeMigrations, 0, numberOfJobsRuns) if err != nil { rErr = multierror.Append(errors.Wrap(err, "error while getting migration jobs")) diff --git a/server/channels/app/support_packet_test.go b/server/channels/app/support_packet_test.go index 0c3d396a5f2..7c88175a83a 100644 --- a/server/channels/app/support_packet_test.go +++ b/server/channels/app/support_packet_test.go @@ -502,7 +502,6 @@ func TestGetSupportPacketJobList(t *testing.T) { assert.Empty(t, jobs.MessageExportJobs) assert.Empty(t, jobs.ElasticPostIndexingJobs) assert.Empty(t, jobs.ElasticPostAggregationJobs) - assert.Empty(t, jobs.BlevePostIndexingJobs) assert.Empty(t, jobs.MigrationJobs) }) @@ -527,7 +526,6 @@ func TestGetSupportPacketJobList(t *testing.T) { getJob(model.JobTypeMessageExport), getJob(model.JobTypeElasticsearchPostIndexing), getJob(model.JobTypeElasticsearchPostAggregation), - getJob(model.JobTypeBlevePostIndexing), getJob(model.JobTypeMigrations), } @@ -577,13 +575,9 @@ func TestGetSupportPacketJobList(t *testing.T) { require.Len(t, jobs.ElasticPostAggregationJobs, 1, "Should have 1 elasticsearch post aggregation job") verifyJob(t, expectedJobs[4], jobs.ElasticPostAggregationJobs[0]) - // Verify bleve post indexing jobs - require.Len(t, jobs.BlevePostIndexingJobs, 1, "Should have 1 bleve post indexing job") - verifyJob(t, expectedJobs[5], jobs.BlevePostIndexingJobs[0]) - // Verify migration jobs require.Len(t, jobs.MigrationJobs, 1, "Should have 1 migration job") - verifyJob(t, expectedJobs[6], jobs.MigrationJobs[0]) + verifyJob(t, expectedJobs[5], jobs.MigrationJobs[0]) }) } diff --git a/server/channels/store/searchtest/channel_layer.go b/server/channels/store/searchtest/channel_layer.go index 76e8eed0ef6..8b97cf62c2e 100644 --- a/server/channels/store/searchtest/channel_layer.go +++ b/server/channels/store/searchtest/channel_layer.go @@ -17,7 +17,7 @@ var searchChannelStoreTests = []searchTest{ { Name: "Should be able to autocomplete a channel by name", Fn: testAutocompleteChannelByName, - Tags: []string{EngineElasticSearch, EngineBleve}, + Tags: []string{EngineElasticSearch}, }, { Name: "Should be able to autocomplete a channel by name (Postgres)", @@ -32,7 +32,7 @@ var searchChannelStoreTests = []searchTest{ { Name: "Should be able to autocomplete a channel by a part of its name when has parts splitted by - character", Fn: testAutocompleteChannelByNameSplittedWithDashChar, - Tags: []string{EngineElasticSearch, EngineBleve}, + Tags: []string{EngineElasticSearch}, }, { Name: "Should be able to autocomplete a channel by a part of its name when has parts splitted by - character (Postgres)", @@ -42,12 +42,12 @@ var searchChannelStoreTests = []searchTest{ { Name: "Should be able to autocomplete a channel by a part of its name when has parts splitted by _ character", Fn: testAutocompleteChannelByNameSplittedWithUnderscoreChar, - Tags: []string{EngineElasticSearch, EngineBleve}, + Tags: []string{EngineElasticSearch}, }, { Name: "Should be able to autocomplete a channel by a part of its display name when has parts splitted by whitespace character", Fn: testAutocompleteChannelByDisplayNameSplittedByWhitespaces, - Tags: []string{EngineElasticSearch, EngineBleve}, + Tags: []string{EngineElasticSearch}, }, { Name: "Should be able to autocomplete retrieving all channels if the term is empty", @@ -57,7 +57,7 @@ var searchChannelStoreTests = []searchTest{ { Name: "Should be able to autocomplete channels in a case insensitive manner", Fn: testSearchChannelsInCaseInsensitiveManner, - Tags: []string{EngineElasticSearch, EngineBleve}, + Tags: []string{EngineElasticSearch}, }, { Name: "Should be able to autocomplete channels in a case insensitive manner (Postgres)", diff --git a/server/channels/store/searchtest/file_info_layer.go b/server/channels/store/searchtest/file_info_layer.go index 6705cfe6f09..d46ad830c18 100644 --- a/server/channels/store/searchtest/file_info_layer.go +++ b/server/channels/store/searchtest/file_info_layer.go @@ -23,7 +23,7 @@ var searchFileInfoStoreTests = []searchTest{ { Name: "Should be able to search posts using pagination", Fn: testFileInfoSearchFileInfosWithPagination, - Tags: []string{EngineElasticSearch, EngineBleve}, + Tags: []string{EngineElasticSearch}, }, { Name: "Should be able to search for exact phrases in quotes", diff --git a/server/channels/store/searchtest/post_layer.go b/server/channels/store/searchtest/post_layer.go index 481112aad5e..7d8eeef4d40 100644 --- a/server/channels/store/searchtest/post_layer.go +++ b/server/channels/store/searchtest/post_layer.go @@ -23,7 +23,7 @@ var searchPostStoreTests = []searchTest{ { Name: "Should be able to search posts using pagination", Fn: testSearchPostsWithPagination, - Tags: []string{EngineElasticSearch, EngineBleve}, + Tags: []string{EngineElasticSearch}, }, { Name: "Should return pinned and unpinned posts", diff --git a/server/channels/store/searchtest/testlib.go b/server/channels/store/searchtest/testlib.go index 720d5008ebb..4eeb449bd5e 100644 --- a/server/channels/store/searchtest/testlib.go +++ b/server/channels/store/searchtest/testlib.go @@ -14,7 +14,6 @@ const ( EngineAll = "all" EnginePostgres = "postgres" EngineElasticSearch = "elasticsearch" - EngineBleve = "bleve" ) type SearchTestEngine struct { diff --git a/server/channels/store/searchtest/user_layer.go b/server/channels/store/searchtest/user_layer.go index d2fce935292..dee6a0635a6 100644 --- a/server/channels/store/searchtest/user_layer.go +++ b/server/channels/store/searchtest/user_layer.go @@ -27,7 +27,7 @@ var searchUserStoreTests = []searchTest{ { Name: "Should honor team restrictions when autocompleting users", Fn: testHonorTeamRestrictionsAutocompletingUsers, - Tags: []string{EngineElasticSearch, EngineBleve}, + Tags: []string{EngineElasticSearch}, }, { Name: "Should return nothing if the user can't access the channels of a given search", @@ -71,17 +71,17 @@ var searchUserStoreTests = []searchTest{ { Name: "Should be able to autocomplete a user by part of its username splitted by Dot", Fn: testAutocompleteUserByUsernameWithDot, - Tags: []string{EngineElasticSearch, EngineBleve}, + Tags: []string{EngineElasticSearch}, }, { Name: "Should be able to autocomplete a user by part of its username splitted by underscore", Fn: testAutocompleteUserByUsernameWithUnderscore, - Tags: []string{EngineElasticSearch, EngineBleve}, + Tags: []string{EngineElasticSearch}, }, { Name: "Should be able to autocomplete a user by part of its username splitted by hyphen", Fn: testAutocompleteUserByUsernameWithHyphen, - Tags: []string{EngineElasticSearch, EngineBleve}, + Tags: []string{EngineElasticSearch}, }, { Name: "Should escape the percentage character", diff --git a/server/channels/testlib/testdata/mysql_migration_warmup.sql b/server/channels/testlib/testdata/mysql_migration_warmup.sql index eaafb2d368d..b76f793c431 100644 --- a/server/channels/testlib/testdata/mysql_migration_warmup.sql +++ b/server/channels/testlib/testdata/mysql_migration_warmup.sql @@ -77,11 +77,11 @@ INSERT INTO `Roles` VALUES ('cmqctq1egt877y9ua9pdsknoiw','team_post_all_public', INSERT INTO `Roles` VALUES ('dmmkxxmi3b8pdgcq9pjtf6mfao','playbook_member','authentication.roles.playbook_member.name','authentication.roles.playbook_member.description',1662271985811,1662271986918,0,' playbook_private_manage_members playbook_private_manage_properties run_create playbook_public_view playbook_public_manage_members playbook_public_manage_properties playbook_private_view',1,1); INSERT INTO `Roles` VALUES ('dwjkqj9bj7r4xr8zb18z3tg53y','playbook_admin','authentication.roles.playbook_admin.name','authentication.roles.playbook_admin.description',1662271985841,1662271986921,0,' playbook_private_manage_roles playbook_private_manage_properties playbook_public_make_private playbook_public_manage_members playbook_public_manage_roles playbook_public_manage_properties playbook_private_manage_members',1,1); INSERT INTO `Roles` VALUES ('hh56iy3patffuc3h76soondcga','channel_admin','authentication.roles.channel_admin.name','authentication.roles.channel_admin.description',0,1662271986924,0,' manage_channel_roles use_group_mentions',1,1); -INSERT INTO `Roles` VALUES ('hkcrew7wttb5fbuw3ime6g7nzc','system_read_only_admin','authentication.roles.system_read_only_admin.name','authentication.roles.system_read_only_admin.description',0,1662271986928,0,' sysconsole_read_environment_database sysconsole_read_experimental_features sysconsole_read_compliance_compliance_export sysconsole_read_environment_performance_monitoring sysconsole_read_environment_file_storage sysconsole_read_user_management_channels read_public_channel_groups read_elasticsearch_post_aggregation_job sysconsole_read_integrations_integration_management sysconsole_read_environment_push_notification_server read_compliance_export_job sysconsole_read_user_management_teams sysconsole_read_environment_logging sysconsole_read_about_edition_and_license sysconsole_read_site_customization sysconsole_read_reporting_site_statistics sysconsole_read_site_emoji sysconsole_read_authentication_guest_access test_ldap read_audits sysconsole_read_site_posts download_compliance_export_result sysconsole_read_compliance_compliance_monitoring sysconsole_read_site_announcement_banner sysconsole_read_integrations_gif sysconsole_read_authentication_email sysconsole_read_site_file_sharing_and_downloads sysconsole_read_experimental_bleve sysconsole_read_compliance_data_retention_policy read_channel sysconsole_read_experimental_feature_flags sysconsole_read_environment_image_proxy view_team sysconsole_read_authentication_openid sysconsole_read_environment_web_server sysconsole_read_integrations_cors read_ldap_sync_job sysconsole_read_authentication_saml get_analytics read_private_channel_groups sysconsole_read_reporting_team_statistics sysconsole_read_compliance_custom_terms_of_service sysconsole_read_authentication_ldap sysconsole_read_environment_smtp read_other_users_teams sysconsole_read_user_management_permissions sysconsole_read_environment_session_lengths read_public_channel read_data_retention_job sysconsole_read_user_management_groups sysconsole_read_environment_high_availability sysconsole_read_site_public_links sysconsole_read_authentication_password sysconsole_read_environment_rate_limiting list_public_teams sysconsole_read_site_users_and_teams sysconsole_read_authentication_signup get_logs read_license_information sysconsole_read_site_notices list_private_teams read_elasticsearch_post_indexing_job sysconsole_read_site_notifications sysconsole_read_authentication_mfa sysconsole_read_integrations_bot_accounts sysconsole_read_reporting_server_logs sysconsole_read_site_localization sysconsole_read_environment_elasticsearch sysconsole_read_user_management_users sysconsole_read_plugins sysconsole_read_environment_developer',0,1); +INSERT INTO `Roles` VALUES ('hkcrew7wttb5fbuw3ime6g7nzc','system_read_only_admin','authentication.roles.system_read_only_admin.name','authentication.roles.system_read_only_admin.description',0,1662271986928,0,' sysconsole_read_environment_database sysconsole_read_experimental_features sysconsole_read_compliance_compliance_export sysconsole_read_environment_performance_monitoring sysconsole_read_environment_file_storage sysconsole_read_user_management_channels read_public_channel_groups read_elasticsearch_post_aggregation_job sysconsole_read_integrations_integration_management sysconsole_read_environment_push_notification_server read_compliance_export_job sysconsole_read_user_management_teams sysconsole_read_environment_logging sysconsole_read_about_edition_and_license sysconsole_read_site_customization sysconsole_read_reporting_site_statistics sysconsole_read_site_emoji sysconsole_read_authentication_guest_access test_ldap read_audits sysconsole_read_site_posts download_compliance_export_result sysconsole_read_compliance_compliance_monitoring sysconsole_read_site_announcement_banner sysconsole_read_integrations_gif sysconsole_read_authentication_email sysconsole_read_site_file_sharing_and_downloads sysconsole_read_compliance_data_retention_policy read_channel sysconsole_read_experimental_feature_flags sysconsole_read_environment_image_proxy view_team sysconsole_read_authentication_openid sysconsole_read_environment_web_server sysconsole_read_integrations_cors read_ldap_sync_job sysconsole_read_authentication_saml get_analytics read_private_channel_groups sysconsole_read_reporting_team_statistics sysconsole_read_compliance_custom_terms_of_service sysconsole_read_authentication_ldap sysconsole_read_environment_smtp read_other_users_teams sysconsole_read_user_management_permissions sysconsole_read_environment_session_lengths read_public_channel read_data_retention_job sysconsole_read_user_management_groups sysconsole_read_environment_high_availability sysconsole_read_site_public_links sysconsole_read_authentication_password sysconsole_read_environment_rate_limiting list_public_teams sysconsole_read_site_users_and_teams sysconsole_read_authentication_signup get_logs read_license_information sysconsole_read_site_notices list_private_teams read_elasticsearch_post_indexing_job sysconsole_read_site_notifications sysconsole_read_authentication_mfa sysconsole_read_integrations_bot_accounts sysconsole_read_reporting_server_logs sysconsole_read_site_localization sysconsole_read_environment_elasticsearch sysconsole_read_user_management_users sysconsole_read_plugins sysconsole_read_environment_developer',0,1); INSERT INTO `Roles` VALUES ('iiwt9pt6wiyb9e1enixtxs5yme','run_admin','authentication.roles.run_admin.name','authentication.roles.run_admin.description',1662271985864,1662271986932,0,' run_manage_properties run_manage_members',1,1); INSERT INTO `Roles` VALUES ('jg1f1xfh3bb73pua938orwg9ie','system_guest','authentication.roles.global_guest.name','authentication.roles.global_guest.description',1605167829015,1662271986937,0,' create_direct_channel create_group_channel',1,1); INSERT INTO `Roles` VALUES ('k891n5tpd3n9peue79azejjocy','system_post_all_public','authentication.roles.system_post_all_public.name','authentication.roles.system_post_all_public.description',0,1662271986941,0,' use_channel_mentions create_post_public',0,1); -INSERT INTO `Roles` VALUES ('kb6r9i58x7dxdb3srfohd66sse','system_admin','authentication.roles.global_admin.name','authentication.roles.global_admin.description',0,1662271986948,0,' list_public_teams edit_brand manage_private_channel_properties sysconsole_read_user_management_teams playbook_public_create manage_others_bots invalidate_caches manage_shared_channels sysconsole_write_environment_logging manage_others_outgoing_webhooks sysconsole_read_reporting_team_statistics sysconsole_read_plugins list_team_channels use_group_mentions sysconsole_read_site_users_and_teams sysconsole_write_site_localization get_analytics sysconsole_read_experimental_bleve manage_team_roles sysconsole_read_site_localization edit_post sysconsole_write_user_management_channels test_elasticsearch list_private_teams add_ldap_public_cert join_public_teams manage_slash_commands manage_others_incoming_webhooks manage_public_channel_members sysconsole_read_environment_elasticsearch sysconsole_write_site_customization delete_others_emojis run_manage_members create_emojis sysconsole_write_authentication_email sysconsole_write_compliance_compliance_export add_saml_private_cert create_bot sysconsole_write_environment_rate_limiting add_saml_public_cert edit_other_users sysconsole_write_integrations_integration_management read_user_access_token create_elasticsearch_post_indexing_job sysconsole_write_user_management_users assign_system_admin_role sysconsole_write_user_management_groups sysconsole_read_authentication_guest_access sysconsole_write_about_edition_and_license sysconsole_read_authentication_ldap sysconsole_read_experimental_feature_flags sysconsole_read_integrations_cors sysconsole_read_user_management_groups join_public_channels sysconsole_read_experimental_features test_ldap sysconsole_write_environment_elasticsearch sysconsole_write_reporting_server_logs sysconsole_read_environment_image_proxy sysconsole_read_site_announcement_banner sysconsole_read_reporting_site_statistics sysconsole_write_authentication_mfa sysconsole_read_authentication_openid purge_bleve_indexes playbook_public_manage_members delete_emojis sysconsole_write_environment_file_storage sysconsole_write_reporting_site_statistics playbook_private_manage_members import_team sysconsole_write_environment_web_server sysconsole_write_authentication_password read_public_channel_groups create_compliance_export_job sysconsole_read_authentication_password list_users_without_team sysconsole_read_authentication_mfa add_ldap_private_cert create_data_retention_job read_license_information sysconsole_write_authentication_signup sysconsole_read_environment_push_notification_server edit_others_posts download_compliance_export_result create_ldap_sync_job sysconsole_write_authentication_ldap sysconsole_write_plugins read_data_retention_job sysconsole_write_compliance_data_retention_policy sysconsole_read_site_public_links manage_bots manage_system sysconsole_write_compliance_custom_terms_of_service playbook_public_manage_roles playbook_public_manage_properties playbook_private_create sysconsole_write_experimental_bleve sysconsole_read_authentication_email promote_guest get_saml_cert_status add_user_to_team sysconsole_write_site_users_and_teams create_custom_group manage_private_channel_members read_jobs sysconsole_write_experimental_features read_other_users_teams sysconsole_write_reporting_team_statistics sysconsole_read_environment_file_storage create_post_bleve_indexes_job sysconsole_read_site_file_sharing_and_downloads playbook_private_make_public playbook_public_view create_user_access_token create_public_channel read_channel sysconsole_read_user_management_channels sysconsole_read_user_management_permissions read_public_channel sysconsole_read_compliance_custom_terms_of_service sysconsole_write_site_emoji sysconsole_read_integrations_gif sysconsole_read_site_customization sysconsole_write_integrations_cors invite_user create_direct_channel sysconsole_write_user_management_teams run_create manage_custom_group_members read_ldap_sync_job sysconsole_read_site_notifications playbook_private_manage_properties sysconsole_read_integrations_bot_accounts convert_public_channel_to_private invalidate_email_invite reload_config get_saml_metadata_from_idp manage_secure_connections delete_private_channel sysconsole_read_about_edition_and_license convert_private_channel_to_public sysconsole_read_environment_developer recycle_database_connections remove_saml_private_cert manage_oauth sysconsole_write_environment_database sysconsole_write_site_notifications sysconsole_write_authentication_guest_access sysconsole_write_compliance_compliance_monitoring sysconsole_write_environment_image_proxy create_post_public manage_jobs remove_user_from_team delete_others_posts create_post_ephemeral playbook_private_view create_elasticsearch_post_aggregation_job remove_reaction add_reaction sysconsole_write_environment_high_availability sysconsole_write_authentication_openid sysconsole_write_user_management_permissions add_saml_idp_cert sysconsole_read_site_posts view_members sysconsole_write_environment_smtp sysconsole_read_authentication_saml create_post use_channel_mentions create_team playbook_private_manage_roles get_public_link sysconsole_write_billing manage_system_wide_oauth sysconsole_read_environment_database sysconsole_write_environment_session_lengths run_manage_properties sysconsole_write_authentication_saml sysconsole_read_environment_web_server sysconsole_read_environment_rate_limiting manage_public_channel_properties create_group_channel sysconsole_read_compliance_data_retention_policy sysconsole_read_environment_high_availability manage_others_slash_commands sysconsole_read_compliance_compliance_export delete_custom_group sysconsole_read_user_management_system_roles purge_elasticsearch_indexes view_team sysconsole_read_environment_performance_monitoring manage_channel_roles playbook_public_make_private remove_saml_public_cert demote_to_guest sysconsole_write_environment_performance_monitoring read_audits sysconsole_write_site_announcement_banner upload_file revoke_user_access_token read_others_bots test_email read_elasticsearch_post_aggregation_job sysconsole_read_compliance_compliance_monitoring join_private_teams delete_post sysconsole_write_site_public_links manage_team edit_custom_group sysconsole_write_experimental_feature_flags sysconsole_write_user_management_system_roles remove_others_reactions manage_license_information sysconsole_read_authentication_signup read_compliance_export_job sysconsole_write_environment_developer remove_saml_idp_cert manage_incoming_webhooks sysconsole_read_site_emoji assign_bot sysconsole_write_integrations_gif sysconsole_read_user_management_users delete_public_channel manage_outgoing_webhooks sysconsole_write_site_posts remove_ldap_private_cert sysconsole_write_site_file_sharing_and_downloads sysconsole_read_integrations_integration_management sysconsole_read_environment_logging test_site_url sysconsole_read_environment_session_lengths read_elasticsearch_post_indexing_job sysconsole_read_billing sysconsole_read_site_notices sysconsole_read_reporting_server_logs sysconsole_write_integrations_bot_accounts sysconsole_write_site_notices create_private_channel read_private_channel_groups run_view read_bots manage_roles test_s3 sysconsole_write_environment_push_notification_server get_logs invite_guest remove_ldap_public_cert sysconsole_read_environment_smtp',1,1); +INSERT INTO `Roles` VALUES ('kb6r9i58x7dxdb3srfohd66sse','system_admin','authentication.roles.global_admin.name','authentication.roles.global_admin.description',0,1662271986948,0,' list_public_teams edit_brand manage_private_channel_properties sysconsole_read_user_management_teams playbook_public_create manage_others_bots invalidate_caches manage_shared_channels sysconsole_write_environment_logging manage_others_outgoing_webhooks sysconsole_read_reporting_team_statistics sysconsole_read_plugins list_team_channels use_group_mentions sysconsole_read_site_users_and_teams sysconsole_write_site_localization get_analytics manage_team_roles sysconsole_read_site_localization edit_post sysconsole_write_user_management_channels test_elasticsearch list_private_teams add_ldap_public_cert join_public_teams manage_slash_commands manage_others_incoming_webhooks manage_public_channel_members sysconsole_read_environment_elasticsearch sysconsole_write_site_customization delete_others_emojis run_manage_members create_emojis sysconsole_write_authentication_email sysconsole_write_compliance_compliance_export add_saml_private_cert create_bot sysconsole_write_environment_rate_limiting add_saml_public_cert edit_other_users sysconsole_write_integrations_integration_management read_user_access_token create_elasticsearch_post_indexing_job sysconsole_write_user_management_users assign_system_admin_role sysconsole_write_user_management_groups sysconsole_read_authentication_guest_access sysconsole_write_about_edition_and_license sysconsole_read_authentication_ldap sysconsole_read_experimental_feature_flags sysconsole_read_integrations_cors sysconsole_read_user_management_groups join_public_channels sysconsole_read_experimental_features test_ldap sysconsole_write_environment_elasticsearch sysconsole_write_reporting_server_logs sysconsole_read_environment_image_proxy sysconsole_read_site_announcement_banner sysconsole_read_reporting_site_statistics sysconsole_write_authentication_mfa sysconsole_read_authentication_openid playbook_public_manage_members delete_emojis sysconsole_write_environment_file_storage sysconsole_write_reporting_site_statistics playbook_private_manage_members import_team sysconsole_write_environment_web_server sysconsole_write_authentication_password read_public_channel_groups create_compliance_export_job sysconsole_read_authentication_password list_users_without_team sysconsole_read_authentication_mfa add_ldap_private_cert create_data_retention_job read_license_information sysconsole_write_authentication_signup sysconsole_read_environment_push_notification_server edit_others_posts download_compliance_export_result create_ldap_sync_job sysconsole_write_authentication_ldap sysconsole_write_plugins read_data_retention_job sysconsole_write_compliance_data_retention_policy sysconsole_read_site_public_links manage_bots manage_system sysconsole_write_compliance_custom_terms_of_service playbook_public_manage_roles playbook_public_manage_properties playbook_private_create sysconsole_read_authentication_email promote_guest get_saml_cert_status add_user_to_team sysconsole_write_site_users_and_teams create_custom_group manage_private_channel_members read_jobs sysconsole_write_experimental_features read_other_users_teams sysconsole_write_reporting_team_statistics sysconsole_read_environment_file_storage sysconsole_read_site_file_sharing_and_downloads playbook_private_make_public playbook_public_view create_user_access_token create_public_channel read_channel sysconsole_read_user_management_channels sysconsole_read_user_management_permissions read_public_channel sysconsole_read_compliance_custom_terms_of_service sysconsole_write_site_emoji sysconsole_read_integrations_gif sysconsole_read_site_customization sysconsole_write_integrations_cors invite_user create_direct_channel sysconsole_write_user_management_teams run_create manage_custom_group_members read_ldap_sync_job sysconsole_read_site_notifications playbook_private_manage_properties sysconsole_read_integrations_bot_accounts convert_public_channel_to_private invalidate_email_invite reload_config get_saml_metadata_from_idp manage_secure_connections delete_private_channel sysconsole_read_about_edition_and_license convert_private_channel_to_public sysconsole_read_environment_developer recycle_database_connections remove_saml_private_cert manage_oauth sysconsole_write_environment_database sysconsole_write_site_notifications sysconsole_write_authentication_guest_access sysconsole_write_compliance_compliance_monitoring sysconsole_write_environment_image_proxy create_post_public manage_jobs remove_user_from_team delete_others_posts create_post_ephemeral playbook_private_view create_elasticsearch_post_aggregation_job remove_reaction add_reaction sysconsole_write_environment_high_availability sysconsole_write_authentication_openid sysconsole_write_user_management_permissions add_saml_idp_cert sysconsole_read_site_posts view_members sysconsole_write_environment_smtp sysconsole_read_authentication_saml create_post use_channel_mentions create_team playbook_private_manage_roles get_public_link sysconsole_write_billing manage_system_wide_oauth sysconsole_read_environment_database sysconsole_write_environment_session_lengths run_manage_properties sysconsole_write_authentication_saml sysconsole_read_environment_web_server sysconsole_read_environment_rate_limiting manage_public_channel_properties create_group_channel sysconsole_read_compliance_data_retention_policy sysconsole_read_environment_high_availability manage_others_slash_commands sysconsole_read_compliance_compliance_export delete_custom_group sysconsole_read_user_management_system_roles purge_elasticsearch_indexes view_team sysconsole_read_environment_performance_monitoring manage_channel_roles playbook_public_make_private remove_saml_public_cert demote_to_guest sysconsole_write_environment_performance_monitoring read_audits sysconsole_write_site_announcement_banner upload_file revoke_user_access_token read_others_bots test_email read_elasticsearch_post_aggregation_job sysconsole_read_compliance_compliance_monitoring join_private_teams delete_post sysconsole_write_site_public_links manage_team edit_custom_group sysconsole_write_experimental_feature_flags sysconsole_write_user_management_system_roles remove_others_reactions manage_license_information sysconsole_read_authentication_signup read_compliance_export_job sysconsole_write_environment_developer remove_saml_idp_cert manage_incoming_webhooks sysconsole_read_site_emoji assign_bot sysconsole_write_integrations_gif sysconsole_read_user_management_users delete_public_channel manage_outgoing_webhooks sysconsole_write_site_posts remove_ldap_private_cert sysconsole_write_site_file_sharing_and_downloads sysconsole_read_integrations_integration_management sysconsole_read_environment_logging test_site_url sysconsole_read_environment_session_lengths read_elasticsearch_post_indexing_job sysconsole_read_billing sysconsole_read_site_notices sysconsole_read_reporting_server_logs sysconsole_write_integrations_bot_accounts sysconsole_write_site_notices create_private_channel read_private_channel_groups run_view read_bots manage_roles test_s3 sysconsole_write_environment_push_notification_server get_logs invite_guest remove_ldap_public_cert sysconsole_read_environment_smtp',1,1); INSERT INTO `Roles` VALUES ('km7kijhdtjbajquwu36uqneyoc','system_post_all','authentication.roles.system_post_all.name','authentication.roles.system_post_all.description',0,1662271986953,0,' create_post use_channel_mentions',0,1); INSERT INTO `Roles` VALUES ('no7s4436sjbzzqjpupg85mszty','custom_group_user','authentication.roles.custom_group_user.name','authentication.roles.custom_group_user.description',1662271985801,1662271986956,0,'',0,0); INSERT INTO `Roles` VALUES ('qo7e17c1m3rezyjqx5iq9dpmxe','system_manager','authentication.roles.system_manager.name','authentication.roles.system_manager.description',0,1662271986960,0,' sysconsole_write_environment_image_proxy sysconsole_read_environment_developer read_ldap_sync_job sysconsole_read_reporting_team_statistics recycle_database_connections get_logs read_private_channel_groups test_elasticsearch sysconsole_read_environment_logging purge_elasticsearch_indexes sysconsole_write_site_posts sysconsole_read_environment_database sysconsole_read_environment_performance_monitoring manage_team sysconsole_read_authentication_password sysconsole_write_site_users_and_teams sysconsole_read_user_management_channels sysconsole_write_environment_rate_limiting sysconsole_write_site_notifications read_license_information edit_brand sysconsole_read_plugins sysconsole_read_environment_high_availability sysconsole_read_environment_file_storage sysconsole_read_environment_elasticsearch sysconsole_write_environment_web_server sysconsole_write_environment_smtp sysconsole_write_environment_performance_monitoring sysconsole_write_environment_session_lengths sysconsole_write_user_management_groups convert_private_channel_to_public manage_private_channel_properties sysconsole_read_site_posts list_private_teams sysconsole_read_authentication_ldap sysconsole_read_authentication_guest_access sysconsole_read_site_emoji sysconsole_write_integrations_integration_management convert_public_channel_to_private manage_private_channel_members read_elasticsearch_post_aggregation_job manage_team_roles sysconsole_write_site_file_sharing_and_downloads read_channel read_public_channel sysconsole_read_authentication_openid add_user_to_team sysconsole_write_environment_developer sysconsole_write_site_localization sysconsole_read_about_edition_and_license test_s3 reload_config sysconsole_write_environment_elasticsearch test_site_url sysconsole_write_site_announcement_banner get_analytics sysconsole_read_environment_push_notification_server sysconsole_read_authentication_signup test_email sysconsole_write_integrations_bot_accounts sysconsole_write_integrations_cors view_team sysconsole_write_integrations_gif sysconsole_read_site_notices sysconsole_read_environment_image_proxy sysconsole_read_integrations_cors sysconsole_write_environment_push_notification_server join_public_teams test_ldap create_elasticsearch_post_aggregation_job sysconsole_read_environment_session_lengths sysconsole_write_environment_file_storage manage_public_channel_members sysconsole_write_site_customization sysconsole_read_site_announcement_banner sysconsole_read_environment_smtp sysconsole_write_user_management_teams delete_public_channel sysconsole_write_environment_logging read_public_channel_groups sysconsole_read_site_users_and_teams sysconsole_read_reporting_site_statistics sysconsole_read_site_localization sysconsole_read_site_customization sysconsole_read_environment_rate_limiting sysconsole_read_environment_web_server sysconsole_write_user_management_permissions sysconsole_read_site_file_sharing_and_downloads sysconsole_write_site_public_links sysconsole_read_site_public_links sysconsole_read_authentication_email read_elasticsearch_post_indexing_job sysconsole_read_authentication_saml remove_user_from_team delete_private_channel sysconsole_write_user_management_channels sysconsole_read_reporting_server_logs sysconsole_read_integrations_bot_accounts sysconsole_read_user_management_teams list_public_teams create_elasticsearch_post_indexing_job sysconsole_write_site_emoji invalidate_caches sysconsole_read_integrations_integration_management sysconsole_write_environment_high_availability sysconsole_read_user_management_permissions join_private_teams manage_channel_roles sysconsole_write_site_notices manage_public_channel_properties sysconsole_write_environment_database sysconsole_read_site_notifications sysconsole_read_user_management_groups sysconsole_read_integrations_gif sysconsole_read_authentication_mfa',0,1); diff --git a/server/channels/testlib/testdata/postgres_migration_warmup.sql b/server/channels/testlib/testdata/postgres_migration_warmup.sql index b58b54e62e4..e82eb2b0e04 100644 --- a/server/channels/testlib/testdata/postgres_migration_warmup.sql +++ b/server/channels/testlib/testdata/postgres_migration_warmup.sql @@ -33,10 +33,10 @@ INSERT INTO public.roles VALUES ('ozgjpnirx7fdjp3i1i8jrg1kwc', 'system_custom_gr INSERT INTO public.roles VALUES ('pfnwpqmbmjrexgqbxdu61wfd3w', 'playbook_member', 'authentication.roles.playbook_member.name', 'authentication.roles.playbook_member.description', 1662230811533, 1662230812034, 0, ' playbook_public_view playbook_public_manage_members playbook_public_manage_properties playbook_private_view playbook_private_manage_members playbook_private_manage_properties run_create', true, true); INSERT INTO public.roles VALUES ('dj5zm9bxbidi9ritmana9t1sxh', 'run_admin', 'authentication.roles.run_admin.name', 'authentication.roles.run_admin.description', 1662230811534, 1662230812035, 0, ' run_manage_members run_manage_properties', true, true); INSERT INTO public.roles VALUES ('abrocgnx8pni7esbrmb4pjxhoe', 'run_member', 'authentication.roles.run_member.name', 'authentication.roles.run_member.description', 1662230811534, 1662230812036, 0, ' run_view', true, true); -INSERT INTO public.roles VALUES ('ha8u9qxwx3dm8mnbq8sfi7ugdc', 'system_admin', 'authentication.roles.global_admin.name', 'authentication.roles.global_admin.description', 0, 1662230812038, 0, ' read_public_channel_groups manage_public_channel_properties create_post_ephemeral sysconsole_write_site_localization sysconsole_write_billing sysconsole_read_site_file_sharing_and_downloads playbook_public_manage_roles sysconsole_read_integrations_gif delete_emojis sysconsole_write_experimental_features sysconsole_write_site_posts add_ldap_private_cert use_group_mentions sysconsole_read_authentication_openid add_user_to_team sysconsole_read_user_management_channels sysconsole_write_environment_high_availability sysconsole_write_site_announcement_banner sysconsole_read_site_notices sysconsole_write_user_management_teams convert_public_channel_to_private sysconsole_read_reporting_server_logs manage_system_wide_oauth revoke_user_access_token invalidate_caches sysconsole_write_environment_push_notification_server sysconsole_read_site_emoji remove_others_reactions sysconsole_write_reporting_server_logs sysconsole_write_user_management_permissions sysconsole_read_site_posts assign_bot sysconsole_write_authentication_password add_saml_private_cert manage_jobs sysconsole_write_environment_developer use_channel_mentions add_ldap_public_cert purge_bleve_indexes playbook_public_manage_properties sysconsole_read_authentication_mfa read_public_channel sysconsole_read_environment_image_proxy import_team sysconsole_read_reporting_team_statistics sysconsole_write_user_management_channels list_private_teams sysconsole_read_user_management_groups join_private_teams sysconsole_read_compliance_data_retention_policy list_public_teams sysconsole_read_site_localization sysconsole_write_authentication_guest_access sysconsole_read_compliance_compliance_monitoring sysconsole_read_environment_developer edit_others_posts sysconsole_read_experimental_bleve read_audits sysconsole_write_authentication_email sysconsole_write_experimental_bleve sysconsole_read_environment_push_notification_server read_elasticsearch_post_aggregation_job remove_ldap_private_cert manage_team manage_bots sysconsole_write_environment_session_lengths sysconsole_write_user_management_users sysconsole_write_environment_file_storage invite_user join_public_channels create_direct_channel sysconsole_read_site_users_and_teams manage_slash_commands playbook_public_view sysconsole_write_compliance_custom_terms_of_service purge_elasticsearch_indexes sysconsole_read_authentication_email test_ldap sysconsole_write_plugins manage_outgoing_webhooks create_bot create_compliance_export_job get_logs create_private_channel get_saml_metadata_from_idp read_elasticsearch_post_indexing_job get_analytics manage_incoming_webhooks sysconsole_read_authentication_saml invite_guest manage_shared_channels create_public_channel sysconsole_write_site_file_sharing_and_downloads sysconsole_read_environment_rate_limiting manage_public_channel_members sysconsole_read_environment_file_storage sysconsole_read_environment_performance_monitoring sysconsole_write_environment_performance_monitoring sysconsole_write_integrations_gif create_post_public playbook_public_manage_members upload_file sysconsole_write_reporting_team_statistics manage_team_roles sysconsole_read_site_notifications delete_public_channel sysconsole_write_compliance_compliance_monitoring create_ldap_sync_job create_data_retention_job sysconsole_write_environment_smtp manage_custom_group_members manage_others_slash_commands read_ldap_sync_job sysconsole_read_integrations_bot_accounts read_others_bots read_bots sysconsole_read_authentication_ldap demote_to_guest remove_saml_public_cert create_post_bleve_indexes_job sysconsole_read_user_management_teams sysconsole_write_about_edition_and_license remove_ldap_public_cert read_channel sysconsole_read_environment_database sysconsole_write_authentication_signup test_s3 sysconsole_read_environment_high_availability manage_roles sysconsole_write_site_notifications run_view sysconsole_write_authentication_saml invalidate_email_invite playbook_private_view read_compliance_export_job list_users_without_team sysconsole_read_compliance_compliance_export sysconsole_write_integrations_cors promote_guest manage_oauth read_data_retention_job sysconsole_write_experimental_feature_flags sysconsole_read_environment_session_lengths manage_license_information sysconsole_write_authentication_ldap assign_system_admin_role create_post read_private_channel_groups add_saml_idp_cert playbook_private_create manage_private_channel_properties sysconsole_read_compliance_custom_terms_of_service sysconsole_read_integrations_integration_management sysconsole_read_billing sysconsole_read_authentication_password delete_private_channel sysconsole_write_site_notices create_elasticsearch_post_indexing_job test_email sysconsole_write_environment_database recycle_database_connections edit_brand sysconsole_write_authentication_mfa remove_user_from_team sysconsole_write_user_management_system_roles add_reaction remove_saml_private_cert sysconsole_read_environment_web_server run_create sysconsole_read_authentication_guest_access sysconsole_read_about_edition_and_license run_manage_properties create_user_access_token manage_others_incoming_webhooks create_elasticsearch_post_aggregation_job sysconsole_write_user_management_groups sysconsole_read_experimental_feature_flags create_team sysconsole_read_environment_elasticsearch join_public_teams sysconsole_read_user_management_users sysconsole_read_integrations_cors sysconsole_read_environment_smtp manage_secure_connections manage_channel_roles edit_other_users delete_others_emojis sysconsole_write_site_users_and_teams add_saml_public_cert sysconsole_read_site_announcement_banner create_custom_group download_compliance_export_result create_group_channel get_saml_cert_status sysconsole_read_site_public_links manage_system create_emojis sysconsole_read_authentication_signup sysconsole_write_environment_image_proxy list_team_channels remove_saml_idp_cert sysconsole_read_plugins sysconsole_read_site_customization sysconsole_write_site_customization playbook_private_manage_roles delete_custom_group delete_others_posts sysconsole_write_compliance_data_retention_policy sysconsole_write_environment_logging test_elasticsearch playbook_public_make_private sysconsole_write_site_public_links edit_post playbook_private_make_public sysconsole_write_environment_elasticsearch test_site_url sysconsole_write_compliance_compliance_export playbook_private_manage_members delete_post reload_config edit_custom_group sysconsole_read_user_management_system_roles sysconsole_write_reporting_site_statistics sysconsole_write_site_emoji read_user_access_token sysconsole_write_environment_rate_limiting view_members sysconsole_write_integrations_bot_accounts manage_others_bots manage_others_outgoing_webhooks sysconsole_read_environment_logging sysconsole_read_experimental_features sysconsole_write_authentication_openid manage_private_channel_members read_jobs sysconsole_write_environment_web_server read_license_information sysconsole_read_user_management_permissions view_team convert_private_channel_to_public sysconsole_read_reporting_site_statistics get_public_link read_other_users_teams sysconsole_write_integrations_integration_management run_manage_members playbook_public_create remove_reaction playbook_private_manage_properties', true, true); +INSERT INTO public.roles VALUES ('ha8u9qxwx3dm8mnbq8sfi7ugdc', 'system_admin', 'authentication.roles.global_admin.name', 'authentication.roles.global_admin.description', 0, 1662230812038, 0, ' read_public_channel_groups manage_public_channel_properties create_post_ephemeral sysconsole_write_site_localization sysconsole_write_billing sysconsole_read_site_file_sharing_and_downloads playbook_public_manage_roles sysconsole_read_integrations_gif delete_emojis sysconsole_write_experimental_features sysconsole_write_site_posts add_ldap_private_cert use_group_mentions sysconsole_read_authentication_openid add_user_to_team sysconsole_read_user_management_channels sysconsole_write_environment_high_availability sysconsole_write_site_announcement_banner sysconsole_read_site_notices sysconsole_write_user_management_teams convert_public_channel_to_private sysconsole_read_reporting_server_logs manage_system_wide_oauth revoke_user_access_token invalidate_caches sysconsole_write_environment_push_notification_server sysconsole_read_site_emoji remove_others_reactions sysconsole_write_reporting_server_logs sysconsole_write_user_management_permissions sysconsole_read_site_posts assign_bot sysconsole_write_authentication_password add_saml_private_cert manage_jobs sysconsole_write_environment_developer use_channel_mentions add_ldap_public_cert playbook_public_manage_properties sysconsole_read_authentication_mfa read_public_channel sysconsole_read_environment_image_proxy import_team sysconsole_read_reporting_team_statistics sysconsole_write_user_management_channels list_private_teams sysconsole_read_user_management_groups join_private_teams sysconsole_read_compliance_data_retention_policy list_public_teams sysconsole_read_site_localization sysconsole_write_authentication_guest_access sysconsole_read_compliance_compliance_monitoring sysconsole_read_environment_developer edit_others_posts read_audits sysconsole_write_authentication_email sysconsole_read_environment_push_notification_server read_elasticsearch_post_aggregation_job remove_ldap_private_cert manage_team manage_bots sysconsole_write_environment_session_lengths sysconsole_write_user_management_users sysconsole_write_environment_file_storage invite_user join_public_channels create_direct_channel sysconsole_read_site_users_and_teams manage_slash_commands playbook_public_view sysconsole_write_compliance_custom_terms_of_service purge_elasticsearch_indexes sysconsole_read_authentication_email test_ldap sysconsole_write_plugins manage_outgoing_webhooks create_bot create_compliance_export_job get_logs create_private_channel get_saml_metadata_from_idp read_elasticsearch_post_indexing_job get_analytics manage_incoming_webhooks sysconsole_read_authentication_saml invite_guest manage_shared_channels create_public_channel sysconsole_write_site_file_sharing_and_downloads sysconsole_read_environment_rate_limiting manage_public_channel_members sysconsole_read_environment_file_storage sysconsole_read_environment_performance_monitoring sysconsole_write_environment_performance_monitoring sysconsole_write_integrations_gif create_post_public playbook_public_manage_members upload_file sysconsole_write_reporting_team_statistics manage_team_roles sysconsole_read_site_notifications delete_public_channel sysconsole_write_compliance_compliance_monitoring create_ldap_sync_job create_data_retention_job sysconsole_write_environment_smtp manage_custom_group_members manage_others_slash_commands read_ldap_sync_job sysconsole_read_integrations_bot_accounts read_others_bots read_bots sysconsole_read_authentication_ldap demote_to_guest remove_saml_public_cert sysconsole_read_user_management_teams sysconsole_write_about_edition_and_license remove_ldap_public_cert read_channel sysconsole_read_environment_database sysconsole_write_authentication_signup test_s3 sysconsole_read_environment_high_availability manage_roles sysconsole_write_site_notifications run_view sysconsole_write_authentication_saml invalidate_email_invite playbook_private_view read_compliance_export_job list_users_without_team sysconsole_read_compliance_compliance_export sysconsole_write_integrations_cors promote_guest manage_oauth read_data_retention_job sysconsole_write_experimental_feature_flags sysconsole_read_environment_session_lengths manage_license_information sysconsole_write_authentication_ldap assign_system_admin_role create_post read_private_channel_groups add_saml_idp_cert playbook_private_create manage_private_channel_properties sysconsole_read_compliance_custom_terms_of_service sysconsole_read_integrations_integration_management sysconsole_read_billing sysconsole_read_authentication_password delete_private_channel sysconsole_write_site_notices create_elasticsearch_post_indexing_job test_email sysconsole_write_environment_database recycle_database_connections edit_brand sysconsole_write_authentication_mfa remove_user_from_team sysconsole_write_user_management_system_roles add_reaction remove_saml_private_cert sysconsole_read_environment_web_server run_create sysconsole_read_authentication_guest_access sysconsole_read_about_edition_and_license run_manage_properties create_user_access_token manage_others_incoming_webhooks create_elasticsearch_post_aggregation_job sysconsole_write_user_management_groups sysconsole_read_experimental_feature_flags create_team sysconsole_read_environment_elasticsearch join_public_teams sysconsole_read_user_management_users sysconsole_read_integrations_cors sysconsole_read_environment_smtp manage_secure_connections manage_channel_roles edit_other_users delete_others_emojis sysconsole_write_site_users_and_teams add_saml_public_cert sysconsole_read_site_announcement_banner create_custom_group download_compliance_export_result create_group_channel get_saml_cert_status sysconsole_read_site_public_links manage_system create_emojis sysconsole_read_authentication_signup sysconsole_write_environment_image_proxy list_team_channels remove_saml_idp_cert sysconsole_read_plugins sysconsole_read_site_customization sysconsole_write_site_customization playbook_private_manage_roles delete_custom_group delete_others_posts sysconsole_write_compliance_data_retention_policy sysconsole_write_environment_logging test_elasticsearch playbook_public_make_private sysconsole_write_site_public_links edit_post playbook_private_make_public sysconsole_write_environment_elasticsearch test_site_url sysconsole_write_compliance_compliance_export playbook_private_manage_members delete_post reload_config edit_custom_group sysconsole_read_user_management_system_roles sysconsole_write_reporting_site_statistics sysconsole_write_site_emoji read_user_access_token sysconsole_write_environment_rate_limiting view_members sysconsole_write_integrations_bot_accounts manage_others_bots manage_others_outgoing_webhooks sysconsole_read_environment_logging sysconsole_read_experimental_features sysconsole_write_authentication_openid manage_private_channel_members read_jobs sysconsole_write_environment_web_server read_license_information sysconsole_read_user_management_permissions view_team convert_private_channel_to_public sysconsole_read_reporting_site_statistics get_public_link read_other_users_teams sysconsole_write_integrations_integration_management run_manage_members playbook_public_create remove_reaction playbook_private_manage_properties', true, true); INSERT INTO public.roles VALUES ('hm1bxei8b3d68e4j95tqnndppw', 'system_manager', 'authentication.roles.system_manager.name', 'authentication.roles.system_manager.description', 0, 1662230812025, 0, ' manage_private_channel_members join_public_teams sysconsole_write_site_announcement_banner sysconsole_write_site_emoji manage_public_channel_members purge_elasticsearch_indexes sysconsole_read_authentication_openid sysconsole_read_about_edition_and_license edit_brand sysconsole_read_reporting_team_statistics sysconsole_read_site_file_sharing_and_downloads sysconsole_read_user_management_teams read_private_channel_groups delete_public_channel sysconsole_read_site_customization sysconsole_write_site_notices sysconsole_read_authentication_email sysconsole_write_environment_file_storage sysconsole_read_user_management_permissions sysconsole_read_reporting_site_statistics test_s3 sysconsole_write_user_management_permissions sysconsole_read_environment_rate_limiting read_license_information sysconsole_read_environment_file_storage sysconsole_write_environment_elasticsearch invalidate_caches sysconsole_read_integrations_cors sysconsole_write_user_management_teams add_user_to_team sysconsole_read_environment_performance_monitoring get_logs sysconsole_write_environment_high_availability sysconsole_read_authentication_signup manage_public_channel_properties sysconsole_write_integrations_integration_management read_elasticsearch_post_indexing_job sysconsole_read_user_management_groups view_team sysconsole_write_environment_rate_limiting sysconsole_read_authentication_guest_access sysconsole_read_environment_elasticsearch manage_team reload_config manage_team_roles test_ldap sysconsole_read_site_public_links sysconsole_read_authentication_saml sysconsole_write_integrations_cors read_public_channel_groups sysconsole_write_site_users_and_teams sysconsole_read_integrations_gif get_analytics create_elasticsearch_post_indexing_job sysconsole_read_authentication_ldap sysconsole_read_site_announcement_banner test_site_url sysconsole_read_site_localization sysconsole_write_environment_push_notification_server sysconsole_write_integrations_bot_accounts sysconsole_write_environment_performance_monitoring sysconsole_write_site_posts sysconsole_read_environment_logging read_elasticsearch_post_aggregation_job sysconsole_write_site_localization sysconsole_write_environment_database sysconsole_read_site_posts sysconsole_write_environment_developer sysconsole_read_site_emoji sysconsole_read_plugins create_elasticsearch_post_aggregation_job manage_channel_roles sysconsole_write_user_management_groups remove_user_from_team read_ldap_sync_job sysconsole_write_site_notifications recycle_database_connections test_email sysconsole_read_site_notifications list_public_teams sysconsole_write_site_customization sysconsole_read_environment_smtp sysconsole_read_authentication_mfa sysconsole_read_integrations_integration_management sysconsole_read_user_management_channels sysconsole_read_reporting_server_logs sysconsole_write_site_public_links test_elasticsearch sysconsole_write_environment_smtp sysconsole_read_environment_push_notification_server sysconsole_write_environment_web_server sysconsole_write_environment_logging sysconsole_read_environment_session_lengths sysconsole_read_site_notices sysconsole_read_environment_high_availability join_private_teams sysconsole_read_authentication_password sysconsole_read_environment_developer delete_private_channel sysconsole_read_integrations_bot_accounts sysconsole_write_environment_session_lengths convert_private_channel_to_public sysconsole_read_environment_database sysconsole_read_environment_image_proxy convert_public_channel_to_private manage_private_channel_properties sysconsole_write_site_file_sharing_and_downloads read_public_channel list_private_teams sysconsole_write_integrations_gif sysconsole_read_environment_web_server sysconsole_read_site_users_and_teams sysconsole_write_user_management_channels read_channel sysconsole_write_environment_image_proxy', false, true); INSERT INTO public.roles VALUES ('f9drbz6cyjdmb8jof6smiqya7h', 'system_user_manager', 'authentication.roles.system_user_manager.name', 'authentication.roles.system_user_manager.description', 0, 1662230812028, 0, ' manage_team_roles sysconsole_read_authentication_saml manage_public_channel_members manage_channel_roles add_user_to_team sysconsole_read_authentication_ldap read_public_channel_groups join_public_teams convert_private_channel_to_public join_private_teams sysconsole_read_user_management_teams list_public_teams sysconsole_read_authentication_email list_private_teams sysconsole_read_authentication_signup read_public_channel sysconsole_read_authentication_mfa sysconsole_read_authentication_guest_access test_ldap manage_private_channel_members sysconsole_read_user_management_permissions read_channel remove_user_from_team delete_public_channel sysconsole_write_user_management_channels delete_private_channel sysconsole_read_authentication_openid sysconsole_write_user_management_teams manage_team sysconsole_read_user_management_groups view_team sysconsole_write_user_management_groups sysconsole_read_user_management_channels manage_public_channel_properties manage_private_channel_properties sysconsole_read_authentication_password read_ldap_sync_job convert_public_channel_to_private read_private_channel_groups', false, true); -INSERT INTO public.roles VALUES ('tkioqq1sgtribqgjbzwop1846c', 'system_read_only_admin', 'authentication.roles.system_read_only_admin.name', 'authentication.roles.system_read_only_admin.description', 0, 1662230812033, 0, ' sysconsole_read_integrations_bot_accounts sysconsole_read_authentication_openid sysconsole_read_user_management_users sysconsole_read_authentication_saml read_ldap_sync_job read_other_users_teams sysconsole_read_user_management_permissions download_compliance_export_result sysconsole_read_environment_smtp sysconsole_read_site_localization read_public_channel read_audits sysconsole_read_compliance_custom_terms_of_service read_data_retention_job sysconsole_read_site_emoji sysconsole_read_compliance_data_retention_policy sysconsole_read_environment_developer sysconsole_read_site_file_sharing_and_downloads sysconsole_read_user_management_channels read_elasticsearch_post_indexing_job sysconsole_read_authentication_mfa sysconsole_read_compliance_compliance_monitoring sysconsole_read_authentication_signup sysconsole_read_authentication_ldap sysconsole_read_authentication_password get_analytics sysconsole_read_site_posts sysconsole_read_environment_performance_monitoring sysconsole_read_compliance_compliance_export sysconsole_read_integrations_integration_management test_ldap sysconsole_read_environment_file_storage sysconsole_read_environment_logging sysconsole_read_user_management_groups sysconsole_read_environment_high_availability sysconsole_read_environment_database sysconsole_read_environment_elasticsearch sysconsole_read_environment_push_notification_server sysconsole_read_site_notices read_compliance_export_job read_license_information sysconsole_read_environment_session_lengths read_private_channel_groups sysconsole_read_integrations_gif read_elasticsearch_post_aggregation_job sysconsole_read_experimental_bleve sysconsole_read_reporting_team_statistics sysconsole_read_about_edition_and_license sysconsole_read_environment_image_proxy sysconsole_read_site_customization sysconsole_read_environment_rate_limiting view_team sysconsole_read_site_announcement_banner sysconsole_read_environment_web_server get_logs sysconsole_read_experimental_feature_flags sysconsole_read_integrations_cors sysconsole_read_authentication_guest_access sysconsole_read_plugins read_channel list_public_teams sysconsole_read_user_management_teams sysconsole_read_reporting_server_logs sysconsole_read_experimental_features sysconsole_read_authentication_email sysconsole_read_site_notifications sysconsole_read_site_users_and_teams sysconsole_read_reporting_site_statistics read_public_channel_groups list_private_teams sysconsole_read_site_public_links', false, true); +INSERT INTO public.roles VALUES ('tkioqq1sgtribqgjbzwop1846c', 'system_read_only_admin', 'authentication.roles.system_read_only_admin.name', 'authentication.roles.system_read_only_admin.description', 0, 1662230812033, 0, ' sysconsole_read_integrations_bot_accounts sysconsole_read_authentication_openid sysconsole_read_user_management_users sysconsole_read_authentication_saml read_ldap_sync_job read_other_users_teams sysconsole_read_user_management_permissions download_compliance_export_result sysconsole_read_environment_smtp sysconsole_read_site_localization read_public_channel read_audits sysconsole_read_compliance_custom_terms_of_service read_data_retention_job sysconsole_read_site_emoji sysconsole_read_compliance_data_retention_policy sysconsole_read_environment_developer sysconsole_read_site_file_sharing_and_downloads sysconsole_read_user_management_channels read_elasticsearch_post_indexing_job sysconsole_read_authentication_mfa sysconsole_read_compliance_compliance_monitoring sysconsole_read_authentication_signup sysconsole_read_authentication_ldap sysconsole_read_authentication_password get_analytics sysconsole_read_site_posts sysconsole_read_environment_performance_monitoring sysconsole_read_compliance_compliance_export sysconsole_read_integrations_integration_management test_ldap sysconsole_read_environment_file_storage sysconsole_read_environment_logging sysconsole_read_user_management_groups sysconsole_read_environment_high_availability sysconsole_read_environment_database sysconsole_read_environment_elasticsearch sysconsole_read_environment_push_notification_server sysconsole_read_site_notices read_compliance_export_job read_license_information sysconsole_read_environment_session_lengths read_private_channel_groups sysconsole_read_integrations_gif read_elasticsearch_post_aggregation_job sysconsole_read_reporting_team_statistics sysconsole_read_about_edition_and_license sysconsole_read_environment_image_proxy sysconsole_read_site_customization sysconsole_read_environment_rate_limiting view_team sysconsole_read_site_announcement_banner sysconsole_read_environment_web_server get_logs sysconsole_read_experimental_feature_flags sysconsole_read_integrations_cors sysconsole_read_authentication_guest_access sysconsole_read_plugins read_channel list_public_teams sysconsole_read_user_management_teams sysconsole_read_reporting_server_logs sysconsole_read_experimental_features sysconsole_read_authentication_email sysconsole_read_site_notifications sysconsole_read_site_users_and_teams sysconsole_read_reporting_site_statistics read_public_channel_groups list_private_teams sysconsole_read_site_public_links', false, true); -- diff --git a/server/go.mod b/server/go.mod index 5270cbd38cc..705ce35da51 100644 --- a/server/go.mod +++ b/server/go.mod @@ -10,7 +10,6 @@ require ( github.com/aws/aws-sdk-go v1.55.7 github.com/bep/imagemeta v0.12.0 github.com/blang/semver/v4 v4.0.0 - github.com/blevesearch/bleve/v2 v2.5.0 github.com/cespare/xxhash/v2 v2.3.0 github.com/dgryski/dgoogauth v0.0.0-20190221195224-5a805980a5f3 github.com/dyatlov/go-opengraph/opengraph v0.0.0-20220524092352-606d7b1e5f8a @@ -82,7 +81,6 @@ require ( filippo.io/edwards25519 v1.1.0 // indirect github.com/JalfResi/justext v0.0.0-20221106200834-be571e3e3052 // indirect github.com/PuerkitoBio/goquery v1.10.3 // indirect - github.com/RoaringBitmap/roaring/v2 v2.4.5 // indirect github.com/STARRY-S/zip v0.2.1 // indirect github.com/advancedlogic/GoOse v0.0.0-20231203033844-ae6b36caf275 // indirect github.com/andybalholm/brotli v1.1.2-0.20250424173009-453214e765f3 // indirect @@ -94,23 +92,6 @@ require ( github.com/beorn7/perks v1.0.1 // indirect github.com/bits-and-blooms/bitset v1.22.0 // indirect github.com/bits-and-blooms/bloom/v3 v3.7.0 // indirect - github.com/blevesearch/bleve_index_api v1.2.8 // indirect - github.com/blevesearch/geo v0.2.2 // indirect - github.com/blevesearch/go-faiss v1.0.25 // indirect - github.com/blevesearch/go-porterstemmer v1.0.3 // indirect - github.com/blevesearch/gtreap v0.1.1 // indirect - github.com/blevesearch/mmap-go v1.0.4 // indirect - github.com/blevesearch/scorch_segment_api/v2 v2.3.10 // indirect - github.com/blevesearch/segment v0.9.1 // indirect - github.com/blevesearch/snowballstem v0.9.0 // indirect - github.com/blevesearch/upsidedown_store_api v1.0.2 // indirect - github.com/blevesearch/vellum v1.1.0 // indirect - github.com/blevesearch/zapx/v11 v11.4.2 // indirect - github.com/blevesearch/zapx/v12 v12.4.2 // indirect - github.com/blevesearch/zapx/v13 v13.4.2 // indirect - github.com/blevesearch/zapx/v14 v14.4.2 // indirect - github.com/blevesearch/zapx/v15 v15.4.2 // indirect - github.com/blevesearch/zapx/v16 v16.2.3 // indirect github.com/bodgit/plumbing v1.3.0 // indirect github.com/bodgit/sevenzip v1.6.0 // indirect github.com/bodgit/windows v1.0.1 // indirect @@ -134,7 +115,6 @@ require ( github.com/go-viper/mapstructure/v2 v2.2.1 // indirect github.com/goccy/go-json v0.10.5 // indirect github.com/golang/protobuf v1.5.4 // indirect - github.com/golang/snappy v1.0.0 // indirect github.com/gomodule/redigo v2.0.0+incompatible // indirect github.com/google/btree v1.1.3 // indirect github.com/google/uuid v1.6.0 // indirect @@ -154,7 +134,6 @@ require ( github.com/isacikgoz/fuzzy v0.2.0 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/jonboulle/clockwork v0.5.0 // indirect - github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/cpuid/v2 v2.2.10 // indirect github.com/klauspost/pgzip v1.2.6 // indirect github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 // indirect @@ -169,9 +148,6 @@ require ( github.com/minio/crc64nvme v1.0.2 // indirect github.com/minio/md5-simd v1.1.2 // indirect github.com/minio/minlz v1.0.0 // indirect - github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect - github.com/modern-go/reflect2 v1.0.2 // indirect - github.com/mschoch/smat v0.2.0 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/ncruces/go-strftime v0.1.9 // indirect github.com/nwaples/rardecode/v2 v2.1.0 // indirect @@ -214,7 +190,6 @@ require ( github.com/ulikunitz/xz v0.5.12 // indirect github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect github.com/wiggin77/srslog v1.0.1 // indirect - go.etcd.io/bbolt v1.4.0 // indirect go.opentelemetry.io/auto/sdk v1.1.0 // indirect go.opentelemetry.io/otel v1.35.0 // indirect go.opentelemetry.io/otel/metric v1.35.0 // indirect diff --git a/server/go.sum b/server/go.sum index d7fc477e75c..9c48b9a989c 100644 --- a/server/go.sum +++ b/server/go.sum @@ -36,8 +36,6 @@ github.com/Masterminds/semver/v3 v3.3.1/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lpr github.com/PuerkitoBio/goquery v1.4.1/go.mod h1:T9ezsOHcCrDCgA8aF1Cqr3sSYbO/xgdy8/R/XiIMAhA= github.com/PuerkitoBio/goquery v1.10.3 h1:pFYcNSqHxBD06Fpj/KsbStFRsgRATgnf3LeXiUkhzPo= github.com/PuerkitoBio/goquery v1.10.3/go.mod h1:tMUX0zDMHXYlAQk6p35XxQMqMweEKB7iK7iLNd4RH4Y= -github.com/RoaringBitmap/roaring/v2 v2.4.5 h1:uGrrMreGjvAtTBobc0g5IrW1D5ldxDQYe2JW2gggRdg= -github.com/RoaringBitmap/roaring/v2 v2.4.5/go.mod h1:FiJcsfkGje/nZBZgCu0ZxCPOKD/hVXDS2dXi7/eUFE0= github.com/STARRY-S/zip v0.2.1 h1:pWBd4tuSGm3wtpoqRZZ2EAwOmcHK6XFf7bU9qcJXyFg= github.com/STARRY-S/zip v0.2.1/go.mod h1:xNvshLODWtC4EJ702g7cTYn13G53o1+X9BWnPFpcWV4= github.com/advancedlogic/GoOse v0.0.0-20231203033844-ae6b36caf275 h1:Kuhf+w+ilOGoXaR4O4nZ6Dp+ZS83LdANUjwyMXsPGX4= @@ -76,49 +74,12 @@ github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6r github.com/bep/imagemeta v0.12.0 h1:ARf+igs5B7pf079LrqRnwzQ/wEB8Q9v4NSDRZO1/F5k= github.com/bep/imagemeta v0.12.0/go.mod h1:23AF6O+4fUi9avjiydpKLStUNtJr5hJB4rarG18JpN8= github.com/bits-and-blooms/bitset v1.10.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= -github.com/bits-and-blooms/bitset v1.12.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= github.com/bits-and-blooms/bitset v1.22.0 h1:Tquv9S8+SGaS3EhyA+up3FXzmkhxPGjQQCkcs2uw7w4= github.com/bits-and-blooms/bitset v1.22.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= github.com/bits-and-blooms/bloom/v3 v3.7.0 h1:VfknkqV4xI+PsaDIsoHueyxVDZrfvMn56jeWUzvzdls= github.com/bits-and-blooms/bloom/v3 v3.7.0/go.mod h1:VKlUSvp0lFIYqxJjzdnSsZEw4iHb1kOL2tfHTgyJBHg= github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= -github.com/blevesearch/bleve/v2 v2.5.0 h1:HzYqBy/5/M9Ul9ESEmXzN/3Jl7YpmWBdHM/+zzv/3k4= -github.com/blevesearch/bleve/v2 v2.5.0/go.mod h1:PcJzTPnEynO15dCf9isxOga7YFRa/cMSsbnRwnszXUk= -github.com/blevesearch/bleve_index_api v1.2.8 h1:Y98Pu5/MdlkRyLM0qDHostYo7i+Vv1cDNhqTeR4Sy6Y= -github.com/blevesearch/bleve_index_api v1.2.8/go.mod h1:rKQDl4u51uwafZxFrPD1R7xFOwKnzZW7s/LSeK4lgo0= -github.com/blevesearch/geo v0.2.2 h1:nNy+1QNErPyv1Dqpq+MUiykqtsDIU7Fn0A985i+ywIo= -github.com/blevesearch/geo v0.2.2/go.mod h1:K56Q33AzXt2YExVHGObtmRSFYZKYGv0JEN5mdacJJR8= -github.com/blevesearch/go-faiss v1.0.25 h1:lel1rkOUGbT1CJ0YgzKwC7k+XH0XVBHnCVWahdCXk4U= -github.com/blevesearch/go-faiss v1.0.25/go.mod h1:OMGQwOaRRYxrmeNdMrXJPvVx8gBnvE5RYrr0BahNnkk= -github.com/blevesearch/go-porterstemmer v1.0.3 h1:GtmsqID0aZdCSNiY8SkuPJ12pD4jI+DdXTAn4YRcHCo= -github.com/blevesearch/go-porterstemmer v1.0.3/go.mod h1:angGc5Ht+k2xhJdZi511LtmxuEf0OVpvUUNrwmM1P7M= -github.com/blevesearch/gtreap v0.1.1 h1:2JWigFrzDMR+42WGIN/V2p0cUvn4UP3C4Q5nmaZGW8Y= -github.com/blevesearch/gtreap v0.1.1/go.mod h1:QaQyDRAT51sotthUWAH4Sj08awFSSWzgYICSZ3w0tYk= -github.com/blevesearch/mmap-go v1.0.4 h1:OVhDhT5B/M1HNPpYPBKIEJaD0F3Si+CrEKULGCDPWmc= -github.com/blevesearch/mmap-go v1.0.4/go.mod h1:EWmEAOmdAS9z/pi/+Toxu99DnsbhG1TIxUoRmJw/pSs= -github.com/blevesearch/scorch_segment_api/v2 v2.3.10 h1:Yqk0XD1mE0fDZAJXTjawJ8If/85JxnLd8v5vG/jWE/s= -github.com/blevesearch/scorch_segment_api/v2 v2.3.10/go.mod h1:Z3e6ChN3qyN35yaQpl00MfI5s8AxUJbpTR/DL8QOQ+8= -github.com/blevesearch/segment v0.9.1 h1:+dThDy+Lvgj5JMxhmOVlgFfkUtZV2kw49xax4+jTfSU= -github.com/blevesearch/segment v0.9.1/go.mod h1:zN21iLm7+GnBHWTao9I+Au/7MBiL8pPFtJBJTsk6kQw= -github.com/blevesearch/snowballstem v0.9.0 h1:lMQ189YspGP6sXvZQ4WZ+MLawfV8wOmPoD/iWeNXm8s= -github.com/blevesearch/snowballstem v0.9.0/go.mod h1:PivSj3JMc8WuaFkTSRDW2SlrulNWPl4ABg1tC/hlgLs= -github.com/blevesearch/upsidedown_store_api v1.0.2 h1:U53Q6YoWEARVLd1OYNc9kvhBMGZzVrdmaozG2MfoB+A= -github.com/blevesearch/upsidedown_store_api v1.0.2/go.mod h1:M01mh3Gpfy56Ps/UXHjEO/knbqyQ1Oamg8If49gRwrQ= -github.com/blevesearch/vellum v1.1.0 h1:CinkGyIsgVlYf8Y2LUQHvdelgXr6PYuvoDIajq6yR9w= -github.com/blevesearch/vellum v1.1.0/go.mod h1:QgwWryE8ThtNPxtgWJof5ndPfx0/YMBh+W2weHKPw8Y= -github.com/blevesearch/zapx/v11 v11.4.2 h1:l46SV+b0gFN+Rw3wUI1YdMWdSAVhskYuvxlcgpQFljs= -github.com/blevesearch/zapx/v11 v11.4.2/go.mod h1:4gdeyy9oGa/lLa6D34R9daXNUvfMPZqUYjPwiLmekwc= -github.com/blevesearch/zapx/v12 v12.4.2 h1:fzRbhllQmEMUuAQ7zBuMvKRlcPA5ESTgWlDEoB9uQNE= -github.com/blevesearch/zapx/v12 v12.4.2/go.mod h1:TdFmr7afSz1hFh/SIBCCZvcLfzYvievIH6aEISCte58= -github.com/blevesearch/zapx/v13 v13.4.2 h1:46PIZCO/ZuKZYgxI8Y7lOJqX3Irkc3N8W82QTK3MVks= -github.com/blevesearch/zapx/v13 v13.4.2/go.mod h1:knK8z2NdQHlb5ot/uj8wuvOq5PhDGjNYQQy0QDnopZk= -github.com/blevesearch/zapx/v14 v14.4.2 h1:2SGHakVKd+TrtEqpfeq8X+So5PShQ5nW6GNxT7fWYz0= -github.com/blevesearch/zapx/v14 v14.4.2/go.mod h1:rz0XNb/OZSMjNorufDGSpFpjoFKhXmppH9Hi7a877D8= -github.com/blevesearch/zapx/v15 v15.4.2 h1:sWxpDE0QQOTjyxYbAVjt3+0ieu8NCE0fDRaFxEsp31k= -github.com/blevesearch/zapx/v15 v15.4.2/go.mod h1:1pssev/59FsuWcgSnTa0OeEpOzmhtmr/0/11H0Z8+Nw= -github.com/blevesearch/zapx/v16 v16.2.3 h1:7Y0r+a3diEvlazsncexq1qoFOcBd64xwMS7aDm4lo1s= -github.com/blevesearch/zapx/v16 v16.2.3/go.mod h1:wVJ+GtURAaRG9KQAMNYyklq0egV+XJlGcXNCE0OFjjA= github.com/bodgit/plumbing v1.3.0 h1:pf9Itz1JOQgn7vEOE7v7nlEfBykYqvUYioC61TwWCFU= github.com/bodgit/plumbing v1.3.0/go.mod h1:JOTb4XiRu5xfnmdnDJo6GmSbSbtSyufrsyZFByMtKEs= github.com/bodgit/sevenzip v1.6.0 h1:a4R0Wu6/P1o1pP/3VV++aEOcyeBxeO/xE2Y9NSTrr6A= @@ -256,8 +217,6 @@ github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= -github.com/golang/snappy v1.0.0 h1:Oy607GVXHs7RtbggtPBnr2RmDArIsAefDwvrdWvRhGs= -github.com/golang/snappy v1.0.0/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/gomodule/redigo v2.0.0+incompatible h1:K/R+8tc58AaqLkqG2Ol3Qk+DR/TlNuhuh457pBFPtt0= github.com/gomodule/redigo v2.0.0+incompatible/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= @@ -370,8 +329,6 @@ github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCV github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= -github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= @@ -465,14 +422,9 @@ github.com/minio/minio-go/v7 v7.0.91/go.mod h1:uvMUcGrpgeSAAI6+sD3818508nUyMULw9 github.com/minio/minlz v1.0.0 h1:Kj7aJZ1//LlTP1DM8Jm7lNKvvJS2m74gyyXXn3+uJWQ= github.com/minio/minlz v1.0.0/go.mod h1:qT0aEB35q79LLornSzeDH75LBf3aH1MV+jB5w9Wasec= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= -github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= -github.com/mschoch/smat v0.2.0 h1:8imxQsjDm8yFEAVBe7azKmKSgzSkZXDuKkSq9374khM= -github.com/mschoch/smat v0.2.0/go.mod h1:kc9mz7DoBKqDyiRL7VZN8KvXQMWeTaVnttLRXOlotKw= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= @@ -709,8 +661,6 @@ github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/yuin/goldmark v1.7.11 h1:ZCxLyDMtz0nT2HFfsYG8WZ47Trip2+JyLysKcMYE5bo= github.com/yuin/goldmark v1.7.11/go.mod h1:ip/1k0VRfGynBgxOz0yCqHrbZXhcjxyuS66Brc7iBKg= -go.etcd.io/bbolt v1.4.0 h1:TU77id3TnN/zKr7CO/uk+fBCwF2jGcMuw2B/FMAzYIk= -go.etcd.io/bbolt v1.4.0/go.mod h1:AsD+OCi/qPN1giOX1aiLAha3o1U8rAz65bvN4j0sRuk= go.opencensus.io v0.18.0/go.mod h1:vKdFvxhtzZ9onBp9VKHK8z/sRpBMnKAsufL7wlDrCOA= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= @@ -1051,7 +1001,6 @@ gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= grpc.go4.org v0.0.0-20170609214715-11d0a25b4919/go.mod h1:77eQGdRu53HpSqPFJFmuJdjuHRquDANNeA4x7B8WQ9o= diff --git a/server/i18n/en.json b/server/i18n/en.json index 8ca697badff..9bf1dc1dd66 100644 --- a/server/i18n/en.json +++ b/server/i18n/en.json @@ -4766,10 +4766,6 @@ "id": "app.channel.get_channels.not_found.app_error", "translation": "No channels were found." }, - { - "id": "app.channel.get_channels_batch_for_indexing.get.app_error", - "translation": "Unable to get the channels batch for indexing." - }, { "id": "app.channel.get_channels_by_ids.app_error", "translation": "Unable to get channels by ids." @@ -6658,10 +6654,6 @@ "id": "app.post.get_direct_posts.app_error", "translation": "Unable to get direct posts." }, - { - "id": "app.post.get_files_batch_for_indexing.get.app_error", - "translation": "Unable to get the files batch for indexing." - }, { "id": "app.post.get_flagged_posts.app_error", "translation": "Unable to get the flagged posts." @@ -6682,10 +6674,6 @@ "id": "app.post.get_posts_around.get.app_error", "translation": "Unable to get the posts for the channel." }, - { - "id": "app.post.get_posts_batch_for_indexing.get.app_error", - "translation": "Unable to get the posts batch for indexing." - }, { "id": "app.post.get_posts_created_at.app_error", "translation": "Unable to get the posts for the channel." @@ -7800,174 +7788,6 @@ "id": "basic_security_check.url.too_long_error", "translation": "URL is too long" }, - { - "id": "bleveengine.already_started.error", - "translation": "Bleve is already started." - }, - { - "id": "bleveengine.create_channel_index.error", - "translation": "Error creating the bleve channel index." - }, - { - "id": "bleveengine.create_file_index.error", - "translation": "Error creating the bleve file index." - }, - { - "id": "bleveengine.create_post_index.error", - "translation": "Error creating the bleve post index." - }, - { - "id": "bleveengine.create_user_index.error", - "translation": "Error creating the bleve user index." - }, - { - "id": "bleveengine.delete_channel.error", - "translation": "Failed to delete the channel." - }, - { - "id": "bleveengine.delete_channel_posts.error", - "translation": "Failed to delete channel posts" - }, - { - "id": "bleveengine.delete_file.error", - "translation": "Failed to delete the file." - }, - { - "id": "bleveengine.delete_files_batch.error", - "translation": "Failed to delete the files." - }, - { - "id": "bleveengine.delete_post.error", - "translation": "Failed to delete the post." - }, - { - "id": "bleveengine.delete_post_files.error", - "translation": "Failed to delete the post files." - }, - { - "id": "bleveengine.delete_user.error", - "translation": "Failed to delete the user." - }, - { - "id": "bleveengine.delete_user_files.error", - "translation": "Failed to delete the user files." - }, - { - "id": "bleveengine.delete_user_posts.error", - "translation": "Failed to delete user posts" - }, - { - "id": "bleveengine.index_channel.error", - "translation": "Failed to index the channel." - }, - { - "id": "bleveengine.index_file.error", - "translation": "Failed to index the file." - }, - { - "id": "bleveengine.index_post.error", - "translation": "Failed to index the post." - }, - { - "id": "bleveengine.index_user.error", - "translation": "Failed to index the user." - }, - { - "id": "bleveengine.indexer.do_job.bulk_index_channels.batch_error", - "translation": "Failed to index channel batch." - }, - { - "id": "bleveengine.indexer.do_job.bulk_index_files.batch_error", - "translation": "Failed to index file batch." - }, - { - "id": "bleveengine.indexer.do_job.bulk_index_posts.batch_error", - "translation": "Failed to index post batch." - }, - { - "id": "bleveengine.indexer.do_job.bulk_index_users.batch_error", - "translation": "Failed to index user batch." - }, - { - "id": "bleveengine.indexer.do_job.engine_inactive", - "translation": "Failed to run Bleve index job: engine is inactive." - }, - { - "id": "bleveengine.indexer.do_job.get_oldest_entity.error", - "translation": "The oldest entity (user, channel or post) could not be retrieved from the database." - }, - { - "id": "bleveengine.indexer.do_job.parse_end_time.error", - "translation": "Bleve indexing worker failed to parse the end time." - }, - { - "id": "bleveengine.indexer.do_job.parse_start_time.error", - "translation": "Bleve indexing worker failed to parse the start time." - }, - { - "id": "bleveengine.indexer.index_batch.nothing_left_to_index.error", - "translation": "Trying to index a new batch when all the entities are completed." - }, - { - "id": "bleveengine.purge_channel_index.error", - "translation": "Failed to purge channel indexes." - }, - { - "id": "bleveengine.purge_file_index.error", - "translation": "Failed to purge file indexes." - }, - { - "id": "bleveengine.purge_list.not_implemented", - "translation": "Purge list feature is not available for Bleve." - }, - { - "id": "bleveengine.purge_post_index.error", - "translation": "Failed to purge post indexes." - }, - { - "id": "bleveengine.purge_user_index.error", - "translation": "Failed to purge user indexes." - }, - { - "id": "bleveengine.search_channels.error", - "translation": "Channel search failed to complete." - }, - { - "id": "bleveengine.search_files.error", - "translation": "File search failed to complete." - }, - { - "id": "bleveengine.search_posts.error", - "translation": "Post search failed to complete." - }, - { - "id": "bleveengine.search_users_in_channel.nuchan.error", - "translation": "User search failed to complete." - }, - { - "id": "bleveengine.search_users_in_channel.uchan.error", - "translation": "User search failed to complete." - }, - { - "id": "bleveengine.search_users_in_team.error", - "translation": "User search failed to complete." - }, - { - "id": "bleveengine.stop_channel_index.error", - "translation": "Failed to close channel index." - }, - { - "id": "bleveengine.stop_file_index.error", - "translation": "Failed to close file index." - }, - { - "id": "bleveengine.stop_post_index.error", - "translation": "Failed to close post index." - }, - { - "id": "bleveengine.stop_user_index.error", - "translation": "Failed to close user index." - }, { "id": "brand.save_brand_image.check_image_limits.app_error", "translation": "Image limits check failed. Resolution is too high." @@ -9180,22 +9000,6 @@ "id": "model.config.is_valid.atmos_camo_image_proxy_url.app_error", "translation": "Invalid RemoteImageProxyURL for atmos/camo. Must be set to your shared key." }, - { - "id": "model.config.is_valid.bleve_search.bulk_indexing_batch_size.app_error", - "translation": "Bleve Bulk Indexing Batch Size must be at least {{.BatchSize}}." - }, - { - "id": "model.config.is_valid.bleve_search.enable_autocomplete.app_error", - "translation": "Bleve EnableIndexing setting must be set to true when Bleve EnableAutocomplete is set to true" - }, - { - "id": "model.config.is_valid.bleve_search.enable_searching.app_error", - "translation": "Bleve EnableIndexing setting must be set to true when Bleve EnableSearching is set to true" - }, - { - "id": "model.config.is_valid.bleve_search.filename.app_error", - "translation": "Bleve IndexingDir setting must be set when Bleve EnableIndexing is set to true" - }, { "id": "model.config.is_valid.cache_type.app_error", "translation": "Cache type must be either lru or redis." @@ -10692,10 +10496,6 @@ "id": "plugin_reattach_request.is_valid.plugin_reattach_config.app_error", "translation": "Missing plugin reattach config" }, - { - "id": "searchengine.bleve.disabled.error", - "translation": "Error purging Bleve indexes: engine is disabled" - }, { "id": "sharedchannel.cannot_deliver_post", "translation": "One or more posts could not be delivered to remote site {{.Remote}} because it is offline. The post(s) will be delivered when the site is online." diff --git a/server/platform/services/searchengine/bleveengine/bleve.go b/server/platform/services/searchengine/bleveengine/bleve.go deleted file mode 100644 index 8733055833f..00000000000 --- a/server/platform/services/searchengine/bleveengine/bleve.go +++ /dev/null @@ -1,337 +0,0 @@ -// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. -// See LICENSE.txt for license information. - -package bleveengine - -import ( - "net/http" - "os" - "path/filepath" - "reflect" - "sync" - "sync/atomic" - "time" - - "github.com/blevesearch/bleve/v2" - "github.com/blevesearch/bleve/v2/analysis/analyzer/keyword" - "github.com/blevesearch/bleve/v2/analysis/analyzer/standard" - "github.com/blevesearch/bleve/v2/mapping" - - "github.com/mattermost/mattermost/server/public/model" - "github.com/mattermost/mattermost/server/public/shared/mlog" - "github.com/mattermost/mattermost/server/public/shared/request" -) - -const ( - EngineName = "bleve" - PostIndex = "posts" - FileIndex = "files" - UserIndex = "users" - ChannelIndex = "channels" -) - -type BleveEngine struct { - PostIndex bleve.Index - FileIndex bleve.Index - UserIndex bleve.Index - ChannelIndex bleve.Index - Mutex sync.RWMutex - ready int32 - cfg *model.Config - indexSync bool -} - -var keywordMapping *mapping.FieldMapping -var standardMapping *mapping.FieldMapping -var dateMapping *mapping.FieldMapping - -func init() { - keywordMapping = bleve.NewTextFieldMapping() - keywordMapping.Analyzer = keyword.Name - - standardMapping = bleve.NewTextFieldMapping() - standardMapping.Analyzer = standard.Name - - dateMapping = bleve.NewNumericFieldMapping() -} - -func getChannelIndexMapping() *mapping.IndexMappingImpl { - channelMapping := bleve.NewDocumentMapping() - channelMapping.AddFieldMappingsAt("Id", keywordMapping) - channelMapping.AddFieldMappingsAt("Type", keywordMapping) - channelMapping.AddFieldMappingsAt("TeamId", keywordMapping) - channelMapping.AddFieldMappingsAt("NameSuggest", keywordMapping) - channelMapping.AddFieldMappingsAt("UserIDs", keywordMapping) - channelMapping.AddFieldMappingsAt("TeamMemberIDs", keywordMapping) - - indexMapping := bleve.NewIndexMapping() - indexMapping.AddDocumentMapping("_default", channelMapping) - - return indexMapping -} - -func getPostIndexMapping() *mapping.IndexMappingImpl { - postMapping := bleve.NewDocumentMapping() - postMapping.AddFieldMappingsAt("Id", keywordMapping) - postMapping.AddFieldMappingsAt("TeamId", keywordMapping) - postMapping.AddFieldMappingsAt("ChannelId", keywordMapping) - postMapping.AddFieldMappingsAt("UserId", keywordMapping) - postMapping.AddFieldMappingsAt("CreateAt", dateMapping) - postMapping.AddFieldMappingsAt("Message", standardMapping) - postMapping.AddFieldMappingsAt("Type", keywordMapping) - postMapping.AddFieldMappingsAt("Hashtags", standardMapping) - postMapping.AddFieldMappingsAt("Attachments", standardMapping) - - indexMapping := bleve.NewIndexMapping() - indexMapping.AddDocumentMapping("_default", postMapping) - - return indexMapping -} - -func getFileIndexMapping() *mapping.IndexMappingImpl { - fileMapping := bleve.NewDocumentMapping() - fileMapping.AddFieldMappingsAt("Id", keywordMapping) - fileMapping.AddFieldMappingsAt("CreatorId", keywordMapping) - fileMapping.AddFieldMappingsAt("ChannelId", keywordMapping) - fileMapping.AddFieldMappingsAt("CreateAt", dateMapping) - fileMapping.AddFieldMappingsAt("Name", standardMapping) - fileMapping.AddFieldMappingsAt("Content", standardMapping) - fileMapping.AddFieldMappingsAt("Extension", keywordMapping) - fileMapping.AddFieldMappingsAt("Content", standardMapping) - - indexMapping := bleve.NewIndexMapping() - indexMapping.AddDocumentMapping("_default", fileMapping) - - return indexMapping -} - -func getUserIndexMapping() *mapping.IndexMappingImpl { - userMapping := bleve.NewDocumentMapping() - userMapping.AddFieldMappingsAt("Id", keywordMapping) - userMapping.AddFieldMappingsAt("SuggestionsWithFullname", keywordMapping) - userMapping.AddFieldMappingsAt("SuggestionsWithoutFullname", keywordMapping) - userMapping.AddFieldMappingsAt("TeamsIds", keywordMapping) - userMapping.AddFieldMappingsAt("ChannelsIds", keywordMapping) - - indexMapping := bleve.NewIndexMapping() - indexMapping.AddDocumentMapping("_default", userMapping) - - return indexMapping -} - -func NewBleveEngine(cfg *model.Config) *BleveEngine { - return &BleveEngine{ - cfg: cfg, - } -} - -func (b *BleveEngine) getIndexDir(indexName string) string { - return filepath.Join(*b.cfg.BleveSettings.IndexDir, indexName+".bleve") -} - -func (b *BleveEngine) createOrOpenIndex(indexName string, mapping *mapping.IndexMappingImpl) (bleve.Index, error) { - indexPath := b.getIndexDir(indexName) - if index, err := bleve.Open(indexPath); err == nil { - return index, nil - } - - index, err := bleve.NewUsing(indexPath, mapping, "scorch", "scorch", map[string]any{ - "forceSegmentType": "zap", - "forceSegmentVersion": 15, - }) - if err != nil { - return nil, err - } - return index, nil -} - -func (b *BleveEngine) openIndexes() *model.AppError { - if atomic.LoadInt32(&b.ready) != 0 { - return model.NewAppError("Bleveengine.Start", "bleveengine.already_started.error", nil, "", http.StatusInternalServerError) - } - - var err error - b.PostIndex, err = b.createOrOpenIndex(PostIndex, getPostIndexMapping()) - if err != nil { - return model.NewAppError("Bleveengine.Start", "bleveengine.create_post_index.error", nil, "", http.StatusInternalServerError).Wrap(err) - } - - b.FileIndex, err = b.createOrOpenIndex(FileIndex, getFileIndexMapping()) - if err != nil { - return model.NewAppError("Bleveengine.Start", "bleveengine.create_file_index.error", nil, "", http.StatusInternalServerError).Wrap(err) - } - - b.UserIndex, err = b.createOrOpenIndex(UserIndex, getUserIndexMapping()) - if err != nil { - return model.NewAppError("Bleveengine.Start", "bleveengine.create_user_index.error", nil, "", http.StatusInternalServerError).Wrap(err) - } - - b.ChannelIndex, err = b.createOrOpenIndex(ChannelIndex, getChannelIndexMapping()) - if err != nil { - return model.NewAppError("Bleveengine.Start", "bleveengine.create_channel_index.error", nil, "", http.StatusInternalServerError).Wrap(err) - } - - atomic.StoreInt32(&b.ready, 1) - return nil -} - -func (b *BleveEngine) Start() *model.AppError { - if !*b.cfg.BleveSettings.EnableIndexing || *b.cfg.BleveSettings.IndexDir == "" { - return nil - } - - b.Mutex.Lock() - defer b.Mutex.Unlock() - - mlog.Info("EXPERIMENTAL: Starting Bleve") - - return b.openIndexes() -} - -func (b *BleveEngine) closeIndexes() *model.AppError { - if b.IsActive() { - if err := b.PostIndex.Close(); err != nil { - return model.NewAppError("Bleveengine.Stop", "bleveengine.stop_post_index.error", nil, "", http.StatusInternalServerError).Wrap(err) - } - - if err := b.FileIndex.Close(); err != nil { - return model.NewAppError("Bleveengine.Stop", "bleveengine.stop_file_index.error", nil, "", http.StatusInternalServerError).Wrap(err) - } - - if err := b.UserIndex.Close(); err != nil { - return model.NewAppError("Bleveengine.Stop", "bleveengine.stop_user_index.error", nil, "", http.StatusInternalServerError).Wrap(err) - } - - if err := b.ChannelIndex.Close(); err != nil { - return model.NewAppError("Bleveengine.Stop", "bleveengine.stop_channel_index.error", nil, "", http.StatusInternalServerError).Wrap(err) - } - } - - atomic.StoreInt32(&b.ready, 0) - return nil -} - -func (b *BleveEngine) Stop() *model.AppError { - b.Mutex.Lock() - defer b.Mutex.Unlock() - - mlog.Info("Stopping Bleve") - - return b.closeIndexes() -} - -func (b *BleveEngine) IsEnabled() bool { - return b.IsIndexingEnabled() -} - -func (b *BleveEngine) IsActive() bool { - return atomic.LoadInt32(&b.ready) == 1 -} - -func (b *BleveEngine) IsIndexingSync() bool { - return b.indexSync -} - -func (b *BleveEngine) RefreshIndexes(_ request.CTX) *model.AppError { - return nil -} - -func (b *BleveEngine) GetVersion() int { - return 0 -} - -func (b *BleveEngine) GetFullVersion() string { - return "0" -} - -func (b *BleveEngine) GetPlugins() []string { - return []string{} -} - -func (b *BleveEngine) GetName() string { - return EngineName -} - -func (b *BleveEngine) TestConfig(rctx request.CTX, cfg *model.Config) *model.AppError { - return nil -} - -func (b *BleveEngine) deleteIndexes() *model.AppError { - if err := os.RemoveAll(b.getIndexDir(PostIndex)); err != nil { - return model.NewAppError("Bleveengine.PurgeIndexes", "bleveengine.purge_post_index.error", nil, "", http.StatusInternalServerError).Wrap(err) - } - if err := os.RemoveAll(b.getIndexDir(UserIndex)); err != nil { - return model.NewAppError("Bleveengine.PurgeIndexes", "bleveengine.purge_user_index.error", nil, "", http.StatusInternalServerError).Wrap(err) - } - if err := os.RemoveAll(b.getIndexDir(ChannelIndex)); err != nil { - return model.NewAppError("Bleveengine.PurgeIndexes", "bleveengine.purge_channel_index.error", nil, "", http.StatusInternalServerError).Wrap(err) - } - if err := os.RemoveAll(b.getIndexDir(FileIndex)); err != nil { - return model.NewAppError("Bleveengine.PurgeIndexes", "bleveengine.purge_file_index.error", nil, "", http.StatusInternalServerError).Wrap(err) - } - return nil -} - -func (b *BleveEngine) PurgeIndexes(rctx request.CTX) *model.AppError { - if *b.cfg.BleveSettings.IndexDir == "" { - return nil - } - - b.Mutex.Lock() - defer b.Mutex.Unlock() - - rctx.Logger().Info("PurgeIndexes Bleve") - if err := b.closeIndexes(); err != nil { - return err - } - - if err := b.deleteIndexes(); err != nil { - return err - } - - return b.openIndexes() -} - -func (b *BleveEngine) PurgeIndexList(rctx request.CTX, indexes []string) *model.AppError { - return model.NewAppError("Bleve.PurgeIndex", "bleveengine.purge_list.not_implemented", nil, "not implemented", http.StatusNotFound) -} - -func (b *BleveEngine) DataRetentionDeleteIndexes(rctx request.CTX, cutoff time.Time) *model.AppError { - return nil -} - -func (b *BleveEngine) IsAutocompletionEnabled() bool { - return *b.cfg.BleveSettings.EnableAutocomplete -} - -func (b *BleveEngine) IsIndexingEnabled() bool { - return *b.cfg.BleveSettings.EnableIndexing -} - -func (b *BleveEngine) IsSearchEnabled() bool { - return *b.cfg.BleveSettings.EnableSearching -} - -func (b *BleveEngine) UpdateConfig(cfg *model.Config) { - b.Mutex.Lock() - defer b.Mutex.Unlock() - - if reflect.DeepEqual(cfg.BleveSettings, b.cfg.BleveSettings) { - return - } - - mlog.Info("UpdateConf Bleve") - - if *cfg.BleveSettings.EnableIndexing != *b.cfg.BleveSettings.EnableIndexing || *cfg.BleveSettings.IndexDir != *b.cfg.BleveSettings.IndexDir { - if err := b.closeIndexes(); err != nil { - mlog.Error("Error closing Bleve indexes to update the config", mlog.Err(err)) - return - } - b.cfg = cfg - if err := b.openIndexes(); err != nil { - mlog.Error("Error opening Bleve indexes after updating the config", mlog.Err(err)) - } - return - } - b.cfg = cfg -} diff --git a/server/platform/services/searchengine/bleveengine/bleve_test.go b/server/platform/services/searchengine/bleveengine/bleve_test.go deleted file mode 100644 index c66f549d732..00000000000 --- a/server/platform/services/searchengine/bleveengine/bleve_test.go +++ /dev/null @@ -1,235 +0,0 @@ -// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. -// See LICENSE.txt for license information. - -package bleveengine - -import ( - "os" - "testing" - - "github.com/blevesearch/bleve/v2" - "github.com/stretchr/testify/require" - "github.com/stretchr/testify/suite" - - "github.com/mattermost/mattermost/server/public/model" - "github.com/mattermost/mattermost/server/public/shared/request" - "github.com/mattermost/mattermost/server/v8/channels/store/searchlayer" - "github.com/mattermost/mattermost/server/v8/channels/store/searchtest" - "github.com/mattermost/mattermost/server/v8/channels/store/sqlstore" - "github.com/mattermost/mattermost/server/v8/channels/store/storetest" - "github.com/mattermost/mattermost/server/v8/channels/testlib" - "github.com/mattermost/mattermost/server/v8/platform/services/searchengine" -) - -type BleveEngineTestSuite struct { - suite.Suite - - SQLSettings *model.SqlSettings - SQLStore *sqlstore.SqlStore - SearchEngine *searchengine.Broker - Store *searchlayer.SearchStore - BleveEngine *BleveEngine - IndexDir string - Context request.CTX -} - -func TestBleveEngineTestSuite(t *testing.T) { - suite.Run(t, &BleveEngineTestSuite{ - Context: request.TestContext(t), - }) -} - -func (s *BleveEngineTestSuite) setupIndexes() { - indexDir, err := os.MkdirTemp("", "mmbleve") - if err != nil { - s.Require().FailNow("Cannot setup bleveengine tests: %s", err.Error()) - } - s.IndexDir = indexDir -} - -func (s *BleveEngineTestSuite) setupStore() { - driverName := os.Getenv("MM_SQLSETTINGS_DRIVERNAME") - if driverName == "" { - driverName = model.DatabaseDriverPostgres - } - s.SQLSettings = storetest.MakeSqlSettings(driverName) - - var err error - s.SQLStore, err = sqlstore.New(*s.SQLSettings, s.Context.Logger(), nil, sqlstore.DisableMorphLogging()) - if err != nil { - s.Require().FailNow("Cannot initialize store: %s", err.Error()) - } - - cfg := &model.Config{} - cfg.SetDefaults() - cfg.BleveSettings.EnableIndexing = model.NewPointer(true) - cfg.BleveSettings.EnableSearching = model.NewPointer(true) - cfg.BleveSettings.EnableAutocomplete = model.NewPointer(true) - cfg.BleveSettings.IndexDir = model.NewPointer(s.IndexDir) - cfg.SqlSettings.DisableDatabaseSearch = model.NewPointer(true) - - s.SearchEngine = searchengine.NewBroker(cfg) - s.Store = searchlayer.NewSearchLayer(&testlib.TestStore{Store: s.SQLStore}, s.SearchEngine, cfg) - - s.BleveEngine = NewBleveEngine(cfg) - s.BleveEngine.indexSync = true - s.SearchEngine.RegisterBleveEngine(s.BleveEngine) - if err := s.BleveEngine.Start(); err != nil { - s.Require().FailNow("Cannot start bleveengine: %s", err.Error()) - } -} - -func (s *BleveEngineTestSuite) SetupSuite() { - s.setupIndexes() - s.setupStore() -} - -func (s *BleveEngineTestSuite) TearDownSuite() { - os.RemoveAll(s.IndexDir) - s.SQLStore.Close() - storetest.CleanupSqlSettings(s.SQLSettings) -} - -func (s *BleveEngineTestSuite) TestBleveSearchStoreTests() { - searchTestEngine := &searchtest.SearchTestEngine{ - Driver: searchtest.EngineBleve, - } - - s.Run("TestSearchChannelStore", func() { - searchtest.TestSearchChannelStore(s.T(), s.Store, searchTestEngine) - }) - - s.Run("TestSearchUserStore", func() { - searchtest.TestSearchUserStore(s.T(), s.Store, searchTestEngine) - }) - - s.Run("TestSearchPostStore", func() { - searchtest.TestSearchPostStore(s.T(), s.Store, searchTestEngine) - }) - - s.Run("TestSearchFileInfoStore", func() { - searchtest.TestSearchFileInfoStore(s.T(), s.Store, searchTestEngine) - }) -} - -func (s *BleveEngineTestSuite) TestDeleteChannelPosts() { - s.Run("Should remove all the posts that belongs to a channel", func() { - s.BleveEngine.PurgeIndexes(s.Context) - teamID := model.NewId() - userID := model.NewId() - channelID := model.NewId() - channelToAvoidID := model.NewId() - for range 10 { - post := createPost(userID, channelID) - appErr := s.SearchEngine.BleveEngine.IndexPost(post, teamID) - require.Nil(s.T(), appErr) - } - postToAvoid := createPost(userID, channelToAvoidID) - appErr := s.SearchEngine.BleveEngine.IndexPost(postToAvoid, teamID) - require.Nil(s.T(), appErr) - - s.SearchEngine.BleveEngine.DeleteChannelPosts(s.Context, channelID) - - doc, err := s.BleveEngine.PostIndex.Document(postToAvoid.Id) - require.NoError(s.T(), err) - require.Equal(s.T(), postToAvoid.Id, doc.ID()) - numberDocs, err := s.BleveEngine.PostIndex.DocCount() - require.NoError(s.T(), err) - require.Equal(s.T(), 1, int(numberDocs)) - }) - - s.Run("Shouldn't do anything if there is not posts for the selected channel", func() { - s.BleveEngine.PurgeIndexes(s.Context) - teamID := model.NewId() - userID := model.NewId() - channelID := model.NewId() - channelToDeleteID := model.NewId() - post := createPost(userID, channelID) - appErr := s.SearchEngine.BleveEngine.IndexPost(post, teamID) - require.Nil(s.T(), appErr) - - s.SearchEngine.BleveEngine.DeleteChannelPosts(s.Context, channelToDeleteID) - - _, err := s.BleveEngine.PostIndex.Document(post.Id) - require.NoError(s.T(), err) - numberDocs, err := s.BleveEngine.PostIndex.DocCount() - require.NoError(s.T(), err) - require.Equal(s.T(), 1, int(numberDocs)) - }) -} - -func (s *BleveEngineTestSuite) TestDeleteUserPosts() { - s.Run("Should remove all the posts that belongs to a user", func() { - s.BleveEngine.PurgeIndexes(s.Context) - teamID := model.NewId() - userID := model.NewId() - userToAvoidID := model.NewId() - channelID := model.NewId() - for range 10 { - post := createPost(userID, channelID) - appErr := s.SearchEngine.BleveEngine.IndexPost(post, teamID) - require.Nil(s.T(), appErr) - } - postToAvoid := createPost(userToAvoidID, channelID) - appErr := s.SearchEngine.BleveEngine.IndexPost(postToAvoid, teamID) - require.Nil(s.T(), appErr) - - s.SearchEngine.BleveEngine.DeleteUserPosts(s.Context, userID) - - doc, err := s.BleveEngine.PostIndex.Document(postToAvoid.Id) - require.NoError(s.T(), err) - require.Equal(s.T(), postToAvoid.Id, doc.ID()) - numberDocs, err := s.BleveEngine.PostIndex.DocCount() - require.NoError(s.T(), err) - require.Equal(s.T(), 1, int(numberDocs)) - }) - - s.Run("Shouldn't do anything if there is not posts for the selected user", func() { - s.BleveEngine.PurgeIndexes(s.Context) - teamID := model.NewId() - userID := model.NewId() - userToDeleteID := model.NewId() - channelID := model.NewId() - post := createPost(userID, channelID) - appErr := s.SearchEngine.BleveEngine.IndexPost(post, teamID) - require.Nil(s.T(), appErr) - - s.SearchEngine.BleveEngine.DeleteUserPosts(s.Context, userToDeleteID) - - _, err := s.BleveEngine.PostIndex.Document(post.Id) - require.NoError(s.T(), err) - numberDocs, err := s.BleveEngine.PostIndex.DocCount() - require.NoError(s.T(), err) - require.Equal(s.T(), 1, int(numberDocs)) - }) -} - -func (s *BleveEngineTestSuite) TestDeletePosts() { - s.BleveEngine.PurgeIndexes(s.Context) - teamID := model.NewId() - userID := model.NewId() - userToAvoidID := model.NewId() - channelID := model.NewId() - for range 10 { - post := createPost(userID, channelID) - appErr := s.SearchEngine.BleveEngine.IndexPost(post, teamID) - require.Nil(s.T(), appErr) - } - postToAvoid := createPost(userToAvoidID, channelID) - appErr := s.SearchEngine.BleveEngine.IndexPost(postToAvoid, teamID) - require.Nil(s.T(), appErr) - - query := bleve.NewTermQuery(userID) - query.SetField("UserId") - search := bleve.NewSearchRequest(query) - count, err := s.BleveEngine.deletePosts(search, 1) - require.NoError(s.T(), err) - require.Equal(s.T(), 10, int(count)) - - doc, err := s.BleveEngine.PostIndex.Document(postToAvoid.Id) - require.NoError(s.T(), err) - require.Equal(s.T(), postToAvoid.Id, doc.ID()) - numberDocs, err := s.BleveEngine.PostIndex.DocCount() - require.NoError(s.T(), err) - require.Equal(s.T(), 1, int(numberDocs)) -} diff --git a/server/platform/services/searchengine/bleveengine/common.go b/server/platform/services/searchengine/bleveengine/common.go deleted file mode 100644 index 0ec62e76dfa..00000000000 --- a/server/platform/services/searchengine/bleveengine/common.go +++ /dev/null @@ -1,163 +0,0 @@ -// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. -// See LICENSE.txt for license information. - -package bleveengine - -import ( - "strings" - - "github.com/mattermost/mattermost/server/public/model" - "github.com/mattermost/mattermost/server/v8/platform/services/searchengine" -) - -type BLVChannel struct { - Id string - Type model.ChannelType - UserIDs []string - TeamId []string - TeamMemberIDs []string - NameSuggest []string -} - -type BLVUser struct { - Id string - SuggestionsWithFullname []string - SuggestionsWithoutFullname []string - TeamsIds []string - ChannelsIds []string -} - -type BLVPost struct { - Id string - TeamId string - ChannelId string - UserId string - CreateAt int64 - Message string - Type string - Hashtags []string - Attachments string -} - -type BLVFile struct { - Id string - CreatorId string - ChannelId string - CreateAt int64 - Name string - Content string - Extension string -} - -func BLVChannelFromChannel(channel *model.Channel, userIDs, teamMemberIDs []string) *BLVChannel { - displayNameInputs := searchengine.GetSuggestionInputsSplitBy(channel.DisplayName, " ") - nameInputs := searchengine.GetSuggestionInputsSplitByMultiple(channel.Name, []string{"-", "_"}) - - return &BLVChannel{ - Id: channel.Id, - Type: channel.Type, - TeamId: []string{channel.TeamId}, - NameSuggest: append(displayNameInputs, nameInputs...), - UserIDs: userIDs, - TeamMemberIDs: teamMemberIDs, - } -} - -func BLVUserFromUserAndTeams(user *model.User, teamsIds, channelsIds []string) *BLVUser { - usernameSuggestions := searchengine.GetSuggestionInputsSplitByMultiple(user.Username, []string{".", "-", "_"}) - - fullnameStrings := []string{} - if user.FirstName != "" { - fullnameStrings = append(fullnameStrings, user.FirstName) - } - if user.LastName != "" { - fullnameStrings = append(fullnameStrings, user.LastName) - } - - fullnameSuggestions := []string{} - if len(fullnameStrings) > 0 { - fullname := strings.Join(fullnameStrings, " ") - fullnameSuggestions = searchengine.GetSuggestionInputsSplitBy(fullname, " ") - } - - nicknameSuggestions := []string{} - if user.Nickname != "" { - nicknameSuggestions = searchengine.GetSuggestionInputsSplitBy(user.Nickname, " ") - } - - usernameAndNicknameSuggestions := append(usernameSuggestions, nicknameSuggestions...) - - return &BLVUser{ - Id: user.Id, - SuggestionsWithFullname: append(usernameAndNicknameSuggestions, fullnameSuggestions...), - SuggestionsWithoutFullname: usernameAndNicknameSuggestions, - TeamsIds: teamsIds, - ChannelsIds: channelsIds, - } -} - -func BLVUserFromUserForIndexing(userForIndexing *model.UserForIndexing) *BLVUser { - user := &model.User{ - Id: userForIndexing.Id, - Username: userForIndexing.Username, - Nickname: userForIndexing.Nickname, - FirstName: userForIndexing.FirstName, - LastName: userForIndexing.LastName, - CreateAt: userForIndexing.CreateAt, - DeleteAt: userForIndexing.DeleteAt, - } - - return BLVUserFromUserAndTeams(user, userForIndexing.TeamsIds, userForIndexing.ChannelsIds) -} - -func BLVPostFromPost(post *model.Post, teamId string) *BLVPost { - p := &model.PostForIndexing{ - TeamId: teamId, - } - post.ShallowCopy(&p.Post) - return BLVPostFromPostForIndexing(p) -} - -func BLVPostFromPostForIndexing(post *model.PostForIndexing) *BLVPost { - return &BLVPost{ - Id: post.Id, - TeamId: post.TeamId, - ChannelId: post.ChannelId, - UserId: post.UserId, - CreateAt: post.CreateAt, - Message: post.Message, - Type: post.Type, - Hashtags: strings.Fields(post.Hashtags), - } -} - -func splitFilenameWords(name string) string { - result := name - result = strings.ReplaceAll(result, "-", " ") - result = strings.ReplaceAll(result, ".", " ") - return result -} - -func BLVFileFromFileInfo(fileInfo *model.FileInfo, channelId string) *BLVFile { - return &BLVFile{ - Id: fileInfo.Id, - ChannelId: channelId, - CreatorId: fileInfo.CreatorId, - CreateAt: fileInfo.CreateAt, - Content: fileInfo.Content, - Extension: fileInfo.Extension, - Name: fileInfo.Name + " " + splitFilenameWords(fileInfo.Name), - } -} - -func BLVFileFromFileForIndexing(file *model.FileForIndexing) *BLVFile { - return &BLVFile{ - Id: file.Id, - ChannelId: file.ChannelId, - CreatorId: file.CreatorId, - CreateAt: file.CreateAt, - Content: file.Content, - Extension: file.Extension, - Name: file.Name + " " + splitFilenameWords(file.Name), - } -} diff --git a/server/platform/services/searchengine/bleveengine/indexer/indexing_job.go b/server/platform/services/searchengine/bleveengine/indexer/indexing_job.go deleted file mode 100644 index 519c179aeae..00000000000 --- a/server/platform/services/searchengine/bleveengine/indexer/indexing_job.go +++ /dev/null @@ -1,638 +0,0 @@ -// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. -// See LICENSE.txt for license information. - -package indexer - -import ( - "context" - "net/http" - "strconv" - "sync" - "time" - - "github.com/mattermost/mattermost/server/public/model" - "github.com/mattermost/mattermost/server/public/shared/mlog" - "github.com/mattermost/mattermost/server/public/shared/request" - "github.com/mattermost/mattermost/server/v8/channels/jobs" - "github.com/mattermost/mattermost/server/v8/platform/services/searchengine/bleveengine" -) - -const ( - timeBetweenBatches = 100 * time.Millisecond - - estimatedPostCount = 10000000 - estimatedFilesCount = 100000 - estimatedChannelCount = 100000 - estimatedUserCount = 10000 -) - -type BleveIndexerWorker struct { - name string - // stateMut protects stopCh and helps enforce - // ordering in case subsequent Run or Stop calls are made. - stateMut sync.Mutex - stopCh chan struct{} - stoppedCh chan bool - jobs chan model.Job - jobServer *jobs.JobServer - logger mlog.LoggerIFace - engine *bleveengine.BleveEngine - stopped bool -} - -func MakeWorker(jobServer *jobs.JobServer, engine *bleveengine.BleveEngine) *BleveIndexerWorker { - if engine == nil { - return nil - } - const workerName = "BleveIndexer" - return &BleveIndexerWorker{ - name: workerName, - stoppedCh: make(chan bool, 1), - jobs: make(chan model.Job), - jobServer: jobServer, - logger: jobServer.Logger().With(mlog.String("worker_name", workerName)), - engine: engine, - stopped: true, - } -} - -type IndexingProgress struct { - Now time.Time - StartAtTime int64 - EndAtTime int64 - LastEntityTime int64 - - TotalPostsCount int64 - DonePostsCount int64 - DonePosts bool - LastPostID string - - TotalFilesCount int64 - DoneFilesCount int64 - DoneFiles bool - LastFileID string - - TotalChannelsCount int64 - DoneChannelsCount int64 - DoneChannels bool - LastChannelID string - - TotalUsersCount int64 - DoneUsersCount int64 - DoneUsers bool - LastUserID string -} - -func (ip *IndexingProgress) CurrentProgress() int64 { - return (ip.DonePostsCount + ip.DoneChannelsCount + ip.DoneUsersCount + ip.DoneFilesCount) * 100 / (ip.TotalPostsCount + ip.TotalChannelsCount + ip.TotalUsersCount + ip.TotalFilesCount) -} - -func (ip *IndexingProgress) IsDone() bool { - return ip.DonePosts && ip.DoneChannels && ip.DoneUsers && ip.DoneFiles -} - -func (worker *BleveIndexerWorker) JobChannel() chan<- model.Job { - return worker.jobs -} - -func (worker *BleveIndexerWorker) IsEnabled(cfg *model.Config) bool { - return true -} - -func (worker *BleveIndexerWorker) Run() { - worker.stateMut.Lock() - // We have to re-assign the stop channel again, because - // it might happen that the job was restarted due to a config change. - if worker.stopped { - worker.stopped = false - worker.stopCh = make(chan struct{}) - } else { - worker.stateMut.Unlock() - return - } - // Run is called from a separate goroutine and doesn't return. - // So we cannot Unlock in a defer clause. - worker.stateMut.Unlock() - - worker.logger.Debug("Worker Started") - - defer func() { - worker.logger.Debug("Worker: Finished") - worker.stoppedCh <- true - }() - - for { - select { - case <-worker.stopCh: - worker.logger.Debug("Worker: Received stop signal") - return - case job := <-worker.jobs: - worker.DoJob(&job) - } - } -} - -func (worker *BleveIndexerWorker) Stop() { - worker.stateMut.Lock() - defer worker.stateMut.Unlock() - - // Set to close, and if already closed before, then return. - if worker.stopped { - return - } - worker.stopped = true - worker.logger.Debug("Worker Stopping") - close(worker.stopCh) - <-worker.stoppedCh -} - -func (worker *BleveIndexerWorker) DoJob(job *model.Job) { - logger := worker.logger.With(jobs.JobLoggerFields(job)...) - logger.Debug("Worker: Received a new candidate job.") - - var appErr *model.AppError - job, appErr = worker.jobServer.ClaimJob(job) - if appErr != nil { - logger.Warn("Worker: Error occurred while trying to claim job", mlog.Err(appErr)) - return - } else if job == nil { - return - } - - logger.Info("Worker: Indexing job claimed by worker") - - if !worker.engine.IsActive() { - appError := model.NewAppError("BleveIndexerWorker", "bleveengine.indexer.do_job.engine_inactive", nil, "", http.StatusInternalServerError) - if err := worker.jobServer.SetJobError(job, appError); err != nil { - logger.Error("Worker: Failed to run job as ") - } - return - } - - progress := IndexingProgress{ - Now: time.Now(), - DonePosts: false, - DoneChannels: false, - DoneUsers: false, - DoneFiles: false, - StartAtTime: 0, - EndAtTime: model.GetMillis(), - } - - // Extract the start and end times, if they are set. - if startString, ok := job.Data["start_time"]; ok { - startInt, err := strconv.ParseInt(startString, 10, 64) - if err != nil { - logger.Error("Worker: Failed to parse start_time for job", mlog.String("start_time", startString), mlog.Err(err)) - appError := model.NewAppError("BleveIndexerWorker", "bleveengine.indexer.do_job.parse_start_time.error", nil, "", http.StatusInternalServerError).Wrap(err) - if err := worker.jobServer.SetJobError(job, appError); err != nil { - logger.Error("Worker: Failed to set job error", mlog.Err(err), mlog.NamedErr("set_error", appError)) - } - return - } - progress.StartAtTime = startInt - } else { - // Set start time to oldest entity in the database. - // A user or a channel may be created before any post. - oldestEntityCreationTime, err := worker.jobServer.Store.Post().GetOldestEntityCreationTime() - if err != nil { - logger.Error("Worker: Failed to fetch oldest entity for job.", mlog.String("start_time", startString), mlog.Err(err)) - appError := model.NewAppError("BleveIndexerWorker", "bleveengine.indexer.do_job.get_oldest_entity.error", nil, "", http.StatusInternalServerError).Wrap(err) - if err := worker.jobServer.SetJobError(job, appError); err != nil { - logger.Error("Worker: Failed to set job error", mlog.Err(err), mlog.NamedErr("set_error", appError)) - } - return - } - progress.StartAtTime = oldestEntityCreationTime - } - progress.LastEntityTime = progress.StartAtTime - - if endString, ok := job.Data["end_time"]; ok { - endInt, err := strconv.ParseInt(endString, 10, 64) - if err != nil { - logger.Error("Worker: Failed to parse end_time for job", mlog.String("end_time", endString), mlog.Err(err)) - appError := model.NewAppError("BleveIndexerWorker", "bleveengine.indexer.do_job.parse_end_time.error", nil, "", http.StatusInternalServerError).Wrap(err) - if err := worker.jobServer.SetJobError(job, appError); err != nil { - logger.Error("Worker: Failed to set job errorv", mlog.Err(err), mlog.NamedErr("set_error", appError)) - } - return - } - progress.EndAtTime = endInt - } - - if id, ok := job.Data["start_post_id"]; ok { - progress.LastPostID = id - } - if id, ok := job.Data["start_channel_id"]; ok { - progress.LastChannelID = id - } - if id, ok := job.Data["start_user_id"]; ok { - progress.LastUserID = id - } - if id, ok := job.Data["start_file_id"]; ok { - progress.LastFileID = id - } - - // Counting all posts may fail or timeout when the posts table is large. If this happens, log a warning, but carry - // on with the indexing job anyway. The only issue is that the progress % reporting will be inaccurate. - if count, err := worker.jobServer.Store.Post().AnalyticsPostCount(&model.PostCountOptions{}); err != nil { - logger.Warn("Worker: Failed to fetch total post count for job. An estimated value will be used for progress reporting.", mlog.Err(err)) - progress.TotalPostsCount = estimatedPostCount - } else { - progress.TotalPostsCount = count - } - - // Same possible fail as above can happen when counting channels - if count, err := worker.jobServer.Store.Channel().AnalyticsTypeCount("", ""); err != nil { - logger.Warn("Worker: Failed to fetch total channel count for job. An estimated value will be used for progress reporting.", mlog.Err(err)) - progress.TotalChannelsCount = estimatedChannelCount - } else { - progress.TotalChannelsCount = count - } - - // Same possible fail as above can happen when counting users - if count, err := worker.jobServer.Store.User().Count(model.UserCountOptions{ - IncludeBotAccounts: true, // This actually doesn't join with the bots table - // since ExcludeRegularUsers is set to false - }); err != nil { - logger.Warn("Worker: Failed to fetch total user count for job. An estimated value will be used for progress reporting.", mlog.Err(err)) - progress.TotalUsersCount = estimatedUserCount - } else { - progress.TotalUsersCount = count - } - - // Counting all files may fail or timeout when the file_info table is large. If this happens, log a warning, but carry - // on with the indexing job anyway. The only issue is that the progress % reporting will be inaccurate. - if count, err := worker.jobServer.Store.FileInfo().CountAll(); err != nil { - logger.Warn("Worker: Failed to fetch total file info count for job. An estimated value will be used for progress reporting.", mlog.Err(err)) - progress.TotalFilesCount = estimatedFilesCount - } else { - progress.TotalFilesCount = count - } - - var cancelContext request.CTX = request.EmptyContext(worker.logger) - cancelCtx, cancelCancelWatcher := context.WithCancel(context.Background()) - cancelWatcherChan := make(chan struct{}, 1) - cancelContext = cancelContext.WithContext(cancelCtx) - go worker.jobServer.CancellationWatcher(cancelContext, job.Id, cancelWatcherChan) - defer cancelCancelWatcher() - - for { - select { - case <-cancelWatcherChan: - logger.Info("Worker: Indexing job has been canceled via CancellationWatcher") - if err := worker.jobServer.SetJobCanceled(job); err != nil { - logger.Error("Worker: Failed to mark job as cancelled", mlog.Err(err)) - } - return - - case <-worker.stopCh: - logger.Info("Worker: Indexing has been canceled via Worker Stop") - if err := worker.jobServer.SetJobCanceled(job); err != nil { - logger.Error("Worker: Failed to mark job as canceled", mlog.Err(err)) - } - return - - case <-time.After(timeBetweenBatches): - var err *model.AppError - if progress, err = worker.IndexBatch(logger, progress); err != nil { - logger.Error("Worker: Failed to index batch for job", mlog.Err(err)) - if err2 := worker.jobServer.SetJobError(job, err); err2 != nil { - logger.Error("Worker: Failed to set job error", mlog.Err(err2), mlog.NamedErr("set_error", err)) - } - return - } - - // Storing the batch progress in metadata. - if job.Data == nil { - job.Data = make(model.StringMap) - } - - job.Data["start_time"] = strconv.FormatInt(progress.LastEntityTime, 10) - job.Data["start_post_id"] = progress.LastPostID - job.Data["start_channel_id"] = progress.LastChannelID - job.Data["start_user_id"] = progress.LastUserID - job.Data["start_file_id"] = progress.LastFileID - job.Data["original_start_time"] = strconv.FormatInt(progress.StartAtTime, 10) - job.Data["end_time"] = strconv.FormatInt(progress.EndAtTime, 10) - - if err := worker.jobServer.SetJobProgress(job, progress.CurrentProgress()); err != nil { - logger.Error("Worker: Failed to set progress for job", mlog.Err(err)) - if err2 := worker.jobServer.SetJobError(job, err); err2 != nil { - logger.Error("Worker: Failed to set error for job", mlog.Err(err2), mlog.NamedErr("set_error", err)) - } - return - } - - if progress.IsDone() { - if err := worker.jobServer.SetJobSuccess(job); err != nil { - logger.Error("Worker: Failed to set success for job", mlog.Err(err)) - if err2 := worker.jobServer.SetJobError(job, err); err2 != nil { - logger.Error("Worker: Failed to set error for job", mlog.Err(err2), mlog.NamedErr("set_error", err)) - } - } - logger.Info("Worker: Indexing job finished successfully") - return - } - } - } -} - -func (worker *BleveIndexerWorker) IndexBatch(logger mlog.LoggerIFace, progress IndexingProgress) (IndexingProgress, *model.AppError) { - if !progress.DonePosts { - return worker.IndexPostsBatch(logger, progress) - } - if !progress.DoneChannels { - return worker.IndexChannelsBatch(logger, progress) - } - if !progress.DoneUsers { - return worker.IndexUsersBatch(logger, progress) - } - if !progress.DoneFiles { - return worker.IndexFilesBatch(logger, progress) - } - return progress, model.NewAppError("BleveIndexerWorker", "bleveengine.indexer.index_batch.nothing_left_to_index.error", nil, "", http.StatusInternalServerError) -} - -func (worker *BleveIndexerWorker) IndexPostsBatch(logger mlog.LoggerIFace, progress IndexingProgress) (IndexingProgress, *model.AppError) { - var posts []*model.PostForIndexing - - tries := 0 - for posts == nil { - var err error - posts, err = worker.jobServer.Store.Post().GetPostsBatchForIndexing(progress.LastEntityTime, progress.LastPostID, *worker.jobServer.Config().BleveSettings.BatchSize) - if err != nil { - if tries >= 10 { - return progress, model.NewAppError("IndexPostsBatch", "app.post.get_posts_batch_for_indexing.get.app_error", nil, "", http.StatusInternalServerError).Wrap(err) - } - logger.Warn("Failed to get posts batch for indexing. Retrying.", mlog.Err(err)) - - // Wait a bit before trying again. - time.Sleep(15 * time.Second) - } - - tries++ - } - - // Handle zero messages. - if len(posts) == 0 { - progress.DonePosts = true - progress.LastEntityTime = progress.StartAtTime - return progress, nil - } - - lastPost, err := worker.BulkIndexPosts(posts, progress) - if err != nil { - return progress, err - } - - // Our exit condition is when the last post's createAt reaches the initial endAtTime - // set during job creation. - if progress.EndAtTime <= lastPost.CreateAt { - progress.DonePosts = true - progress.LastEntityTime = progress.StartAtTime - } else { - progress.LastEntityTime = lastPost.CreateAt - } - - progress.LastPostID = lastPost.Id - progress.DonePostsCount += int64(len(posts)) - - return progress, nil -} - -func (worker *BleveIndexerWorker) BulkIndexPosts(posts []*model.PostForIndexing, progress IndexingProgress) (*model.Post, *model.AppError) { - batch := worker.engine.PostIndex.NewBatch() - - for _, post := range posts { - if post.DeleteAt == 0 { - searchPost := bleveengine.BLVPostFromPostForIndexing(post) - batch.Index(searchPost.Id, searchPost) - } else { - batch.Delete(post.Id) - } - } - - worker.engine.Mutex.RLock() - defer worker.engine.Mutex.RUnlock() - - if err := worker.engine.PostIndex.Batch(batch); err != nil { - return nil, model.NewAppError("BleveIndexerWorker.BulkIndexPosts", "bleveengine.indexer.do_job.bulk_index_posts.batch_error", nil, "", http.StatusInternalServerError).Wrap(err) - } - return &posts[len(posts)-1].Post, nil -} - -func (worker *BleveIndexerWorker) IndexFilesBatch(logger mlog.LoggerIFace, progress IndexingProgress) (IndexingProgress, *model.AppError) { - var files []*model.FileForIndexing - - tries := 0 - for files == nil { - var err error - files, err = worker.jobServer.Store.FileInfo().GetFilesBatchForIndexing(progress.LastEntityTime, progress.LastFileID, true, *worker.jobServer.Config().BleveSettings.BatchSize) - if err != nil { - if tries >= 10 { - return progress, model.NewAppError("IndexFilesBatch", "app.post.get_files_batch_for_indexing.get.app_error", nil, "", http.StatusInternalServerError).Wrap(err) - } - logger.Warn("Failed to get files batch for indexing. Retrying.", mlog.Err(err)) - - // Wait a bit before trying again. - time.Sleep(15 * time.Second) - } - - tries++ - } - - if len(files) == 0 { - progress.DoneFiles = true - progress.LastEntityTime = progress.StartAtTime - return progress, nil - } - - lastFile, err := worker.BulkIndexFiles(files, progress) - if err != nil { - return progress, err - } - - // Our exit condition is when the last file's createAt reaches the initial endAtTime - // set during job creation. - if progress.EndAtTime <= lastFile.CreateAt { - progress.DoneFiles = true - progress.LastEntityTime = progress.StartAtTime - } else { - progress.LastEntityTime = lastFile.CreateAt - } - - progress.LastFileID = lastFile.Id - progress.DoneFilesCount += int64(len(files)) - - return progress, nil -} - -func (worker *BleveIndexerWorker) BulkIndexFiles(files []*model.FileForIndexing, progress IndexingProgress) (*model.FileInfo, *model.AppError) { - batch := worker.engine.FileIndex.NewBatch() - - for _, file := range files { - if file.ShouldIndex() { - searchFile := bleveengine.BLVFileFromFileForIndexing(file) - batch.Index(searchFile.Id, searchFile) - } else { - batch.Delete(file.Id) - } - } - - worker.engine.Mutex.RLock() - defer worker.engine.Mutex.RUnlock() - - if err := worker.engine.FileIndex.Batch(batch); err != nil { - return nil, model.NewAppError("BleveIndexerWorker.BulkIndexPosts", "bleveengine.indexer.do_job.bulk_index_files.batch_error", nil, "", http.StatusInternalServerError).Wrap(err) - } - return &files[len(files)-1].FileInfo, nil -} - -func (worker *BleveIndexerWorker) IndexChannelsBatch(logger mlog.LoggerIFace, progress IndexingProgress) (IndexingProgress, *model.AppError) { - var channels []*model.Channel - - tries := 0 - for channels == nil { - var nErr error - channels, nErr = worker.jobServer.Store.Channel().GetChannelsBatchForIndexing(progress.LastEntityTime, progress.LastChannelID, *worker.jobServer.Config().BleveSettings.BatchSize) - if nErr != nil { - if tries >= 10 { - return progress, model.NewAppError("BleveIndexerWorker.IndexChannelsBatch", "app.channel.get_channels_batch_for_indexing.get.app_error", nil, "", http.StatusInternalServerError).Wrap(nErr) - } - - logger.Warn("Failed to get channels batch for indexing. Retrying.", mlog.Err(nErr)) - - // Wait a bit before trying again. - time.Sleep(15 * time.Second) - } - tries++ - } - - if len(channels) == 0 { - progress.DoneChannels = true - progress.LastEntityTime = progress.StartAtTime - return progress, nil - } - - lastChannel, err := worker.BulkIndexChannels(logger, channels, progress) - if err != nil { - return progress, err - } - - // Our exit condition is when the last channel's createAt reaches the initial endAtTime - // set during job creation. - if progress.EndAtTime <= lastChannel.CreateAt { - progress.DoneChannels = true - progress.LastEntityTime = progress.StartAtTime - } else { - progress.LastEntityTime = lastChannel.CreateAt - } - - progress.LastChannelID = lastChannel.Id - progress.DoneChannelsCount += int64(len(channels)) - - return progress, nil -} - -func (worker *BleveIndexerWorker) BulkIndexChannels(logger mlog.LoggerIFace, channels []*model.Channel, progress IndexingProgress) (*model.Channel, *model.AppError) { - batch := worker.engine.ChannelIndex.NewBatch() - - for _, channel := range channels { - if channel.DeleteAt == 0 { - var userIDs []string - var err error - if channel.Type == model.ChannelTypePrivate { - userIDs, err = worker.jobServer.Store.Channel().GetAllChannelMemberIdsByChannelId(channel.Id) - if err != nil { - return nil, model.NewAppError("BleveIndexerWorker.BulkIndexChannels", "bleveengine.indexer.do_job.bulk_index_channels.batch_error", nil, "", http.StatusInternalServerError).Wrap(err) - } - } - - // Get teamMember ids from channelid - teamMemberIDs, err := worker.jobServer.Store.Channel().GetTeamMembersForChannel(channel.Id) - if err != nil { - return nil, model.NewAppError("BleveIndexerWorker.BulkIndexChannels", "bleveengine.indexer.do_job.bulk_index_channels.batch_error", nil, "", http.StatusInternalServerError).Wrap(err) - } - - searchChannel := bleveengine.BLVChannelFromChannel(channel, userIDs, teamMemberIDs) - batch.Index(searchChannel.Id, searchChannel) - } else { - batch.Delete(channel.Id) - } - } - - worker.engine.Mutex.RLock() - defer worker.engine.Mutex.RUnlock() - - if err := worker.engine.ChannelIndex.Batch(batch); err != nil { - return nil, model.NewAppError("BleveIndexerWorker.BulkIndexChannels", "bleveengine.indexer.do_job.bulk_index_channels.batch_error", nil, "", http.StatusInternalServerError).Wrap(err) - } - return channels[len(channels)-1], nil -} - -func (worker *BleveIndexerWorker) IndexUsersBatch(logger mlog.LoggerIFace, progress IndexingProgress) (IndexingProgress, *model.AppError) { - var users []*model.UserForIndexing - - tries := 0 - for users == nil { - if usersBatch, err := worker.jobServer.Store.User().GetUsersBatchForIndexing(progress.LastEntityTime, progress.LastUserID, *worker.jobServer.Config().BleveSettings.BatchSize); err != nil { - if tries >= 10 { - return progress, model.NewAppError("IndexUsersBatch", "app.user.get_users_batch_for_indexing.get_users.app_error", nil, "", http.StatusInternalServerError).Wrap(err) - } - logger.Warn("Failed to get users batch for indexing. Retrying.", mlog.Err(err)) - - // Wait a bit before trying again. - time.Sleep(15 * time.Second) - } else { - users = usersBatch - } - - tries++ - } - - if len(users) == 0 { - progress.DoneUsers = true - progress.LastEntityTime = progress.StartAtTime - return progress, nil - } - - lastUser, err := worker.BulkIndexUsers(logger, users, progress) - if err != nil { - return progress, err - } - - // Our exit condition is when the last user's createAt reaches the initial endAtTime - // set during job creation. - if progress.EndAtTime <= lastUser.CreateAt { - progress.DoneUsers = true - progress.LastEntityTime = progress.StartAtTime - } else { - progress.LastEntityTime = lastUser.CreateAt - } - progress.LastUserID = lastUser.Id - progress.DoneUsersCount += int64(len(users)) - - return progress, nil -} - -func (worker *BleveIndexerWorker) BulkIndexUsers(logger mlog.LoggerIFace, users []*model.UserForIndexing, progress IndexingProgress) (*model.UserForIndexing, *model.AppError) { - batch := worker.engine.UserIndex.NewBatch() - - for _, user := range users { - if user.DeleteAt == 0 { - searchUser := bleveengine.BLVUserFromUserForIndexing(user) - batch.Index(searchUser.Id, searchUser) - } else { - batch.Delete(user.Id) - } - } - - worker.engine.Mutex.RLock() - defer worker.engine.Mutex.RUnlock() - - if err := worker.engine.UserIndex.Batch(batch); err != nil { - return nil, model.NewAppError("BleveIndexerWorker.BulkIndexUsers", "bleveengine.indexer.do_job.bulk_index_users.batch_error", nil, "", http.StatusInternalServerError).Wrap(err) - } - return users[len(users)-1], nil -} diff --git a/server/platform/services/searchengine/bleveengine/indexer/indexing_job_test.go b/server/platform/services/searchengine/bleveengine/indexer/indexing_job_test.go deleted file mode 100644 index 02ee77fc8c6..00000000000 --- a/server/platform/services/searchengine/bleveengine/indexer/indexing_job_test.go +++ /dev/null @@ -1,75 +0,0 @@ -// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. -// See LICENSE.txt for license information. - -package indexer - -import ( - "errors" - "os" - "testing" - - "github.com/stretchr/testify/mock" - "github.com/stretchr/testify/require" - - "github.com/mattermost/mattermost/server/public/model" - "github.com/mattermost/mattermost/server/public/shared/mlog" - "github.com/mattermost/mattermost/server/v8/channels/jobs" - "github.com/mattermost/mattermost/server/v8/channels/store/storetest" - "github.com/mattermost/mattermost/server/v8/channels/utils/testutils" - "github.com/mattermost/mattermost/server/v8/platform/services/searchengine/bleveengine" -) - -func TestBleveIndexer(t *testing.T) { - mockStore := &storetest.Store{} - defer mockStore.AssertExpectations(t) - - t.Run("Call GetOldestEntityCreationTime for the first indexing call", func(t *testing.T) { - job := &model.Job{ - Id: model.NewId(), - CreateAt: model.GetMillis(), - Status: model.JobStatusPending, - Type: model.JobTypeBlevePostIndexing, - } - retJob := *job - retJob.Status = model.JobStatusInProgress - - mockStore.JobStore. - On("UpdateStatusOptimistically", job.Id, model.JobStatusPending, model.JobStatusInProgress). - Return(&retJob, nil) - mockStore.JobStore.On("UpdateOptimistically", mock.AnythingOfType("*model.Job"), model.JobStatusInProgress).Return(true, nil) - mockStore.PostStore.On("GetOldestEntityCreationTime").Return(int64(1), errors.New("")) // intentionally return error to return from function - - tempDir, err := os.MkdirTemp("", "setupConfigFile") - require.NoError(t, err) - - t.Cleanup(func() { - os.RemoveAll(tempDir) - }) - - cfg := &model.Config{ - BleveSettings: model.BleveSettings{ - EnableIndexing: model.NewPointer(true), - IndexDir: model.NewPointer(tempDir), - }, - } - - jobServer := &jobs.JobServer{ - Store: mockStore, - ConfigService: &testutils.StaticConfigService{ - Cfg: cfg, - }, - } - - bleveEngine := bleveengine.NewBleveEngine(cfg) - aErr := bleveEngine.Start() - require.Nil(t, aErr) - - worker := &BleveIndexerWorker{ - jobServer: jobServer, - engine: bleveEngine, - logger: mlog.CreateConsoleTestLogger(t), - } - - worker.DoJob(job) - }) -} diff --git a/server/platform/services/searchengine/bleveengine/search.go b/server/platform/services/searchengine/bleveengine/search.go deleted file mode 100644 index 2e4be983db0..00000000000 --- a/server/platform/services/searchengine/bleveengine/search.go +++ /dev/null @@ -1,880 +0,0 @@ -// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. -// See LICENSE.txt for license information. - -package bleveengine - -import ( - "net/http" - "strings" - - "github.com/blevesearch/bleve/v2" - "github.com/blevesearch/bleve/v2/search/query" - - "github.com/mattermost/mattermost/server/public/model" - "github.com/mattermost/mattermost/server/public/shared/mlog" - "github.com/mattermost/mattermost/server/public/shared/request" -) - -const DeletePostsBatchSize = 500 -const DeleteFilesBatchSize = 500 - -func (b *BleveEngine) IndexPost(post *model.Post, teamId string) *model.AppError { - b.Mutex.RLock() - defer b.Mutex.RUnlock() - - blvPost := BLVPostFromPost(post, teamId) - if err := b.PostIndex.Index(blvPost.Id, blvPost); err != nil { - return model.NewAppError("Bleveengine.IndexPost", "bleveengine.index_post.error", nil, "", http.StatusInternalServerError).Wrap(err) - } - return nil -} - -func (b *BleveEngine) SearchPosts(channels model.ChannelList, searchParams []*model.SearchParams, page, perPage int) ([]string, model.PostSearchMatches, *model.AppError) { - channelQueries := []query.Query{} - for _, channel := range channels { - channelIdQ := bleve.NewTermQuery(channel.Id) - channelIdQ.SetField("ChannelId") - channelQueries = append(channelQueries, channelIdQ) - } - channelDisjunctionQ := bleve.NewDisjunctionQuery(channelQueries...) - - var termQueries []query.Query - var notTermQueries []query.Query - var filters []query.Query - var notFilters []query.Query - - typeQ := bleve.NewTermQuery("") - typeQ.SetField("Type") - filters = append(filters, typeQ) - - for i, params := range searchParams { - var termOperator query.MatchQueryOperator = query.MatchQueryOperatorAnd - if searchParams[0].OrTerms { - termOperator = query.MatchQueryOperatorOr - } - - // Date, channels and FromUsers filters come in all - // searchParams iteration, and as they are global to the - // query, we only need to process them once - if i == 0 { - if len(params.InChannels) > 0 { - inChannels := []query.Query{} - for _, channelId := range params.InChannels { - channelQ := bleve.NewTermQuery(channelId) - channelQ.SetField("ChannelId") - inChannels = append(inChannels, channelQ) - } - filters = append(filters, bleve.NewDisjunctionQuery(inChannels...)) - } - - if len(params.ExcludedChannels) > 0 { - excludedChannels := []query.Query{} - for _, channelId := range params.ExcludedChannels { - channelQ := bleve.NewTermQuery(channelId) - channelQ.SetField("ChannelId") - excludedChannels = append(excludedChannels, channelQ) - } - notFilters = append(notFilters, bleve.NewDisjunctionQuery(excludedChannels...)) - } - - if len(params.FromUsers) > 0 { - fromUsers := []query.Query{} - for _, userId := range params.FromUsers { - userQ := bleve.NewTermQuery(userId) - userQ.SetField("UserId") - fromUsers = append(fromUsers, userQ) - } - filters = append(filters, bleve.NewDisjunctionQuery(fromUsers...)) - } - - if len(params.ExcludedUsers) > 0 { - excludedUsers := []query.Query{} - for _, userId := range params.ExcludedUsers { - userQ := bleve.NewTermQuery(userId) - userQ.SetField("UserId") - excludedUsers = append(excludedUsers, userQ) - } - notFilters = append(notFilters, bleve.NewDisjunctionQuery(excludedUsers...)) - } - - if params.OnDate != "" { - before, after := params.GetOnDateMillis() - beforeFloat64 := float64(before) - afterFloat64 := float64(after) - onDateQ := bleve.NewNumericRangeQuery(&beforeFloat64, &afterFloat64) - onDateQ.SetField("CreateAt") - filters = append(filters, onDateQ) - } else { - if params.AfterDate != "" || params.BeforeDate != "" { - var rangeMin, rangeMax *float64 - if params.AfterDate != "" { - minf := float64(params.GetAfterDateMillis()) - rangeMin = &minf - } - - if params.BeforeDate != "" { - maxf := float64(params.GetBeforeDateMillis()) - rangeMax = &maxf - } - - dateQ := bleve.NewNumericRangeQuery(rangeMin, rangeMax) - dateQ.SetField("CreateAt") - filters = append(filters, dateQ) - } - - if params.ExcludedAfterDate != "" { - minf := float64(params.GetExcludedAfterDateMillis()) - dateQ := bleve.NewNumericRangeQuery(&minf, nil) - dateQ.SetField("CreateAt") - notFilters = append(notFilters, dateQ) - } - - if params.ExcludedBeforeDate != "" { - maxf := float64(params.GetExcludedBeforeDateMillis()) - dateQ := bleve.NewNumericRangeQuery(nil, &maxf) - dateQ.SetField("CreateAt") - notFilters = append(notFilters, dateQ) - } - - if params.ExcludedDate != "" { - before, after := params.GetExcludedDateMillis() - beforef := float64(before) - afterf := float64(after) - onDateQ := bleve.NewNumericRangeQuery(&beforef, &afterf) - onDateQ.SetField("CreateAt") - notFilters = append(notFilters, onDateQ) - } - } - } - - if params.IsHashtag { - if params.Terms != "" { - hashtagQ := bleve.NewMatchQuery(params.Terms) - hashtagQ.SetField("Hashtags") - hashtagQ.SetOperator(termOperator) - termQueries = append(termQueries, hashtagQ) - } else if params.ExcludedTerms != "" { - hashtagQ := bleve.NewMatchQuery(params.ExcludedTerms) - hashtagQ.SetField("Hashtags") - hashtagQ.SetOperator(termOperator) - notTermQueries = append(notTermQueries, hashtagQ) - } - } else { - if params.Terms != "" { - terms := []string{} - for term := range strings.SplitSeq(params.Terms, " ") { - if strings.HasSuffix(term, "*") { - messageQ := bleve.NewWildcardQuery(term) - messageQ.SetField("Message") - termQueries = append(termQueries, messageQ) - } else { - terms = append(terms, term) - } - } - - if len(terms) > 0 { - messageQ := bleve.NewMatchQuery(strings.Join(terms, " ")) - messageQ.SetField("Message") - messageQ.SetOperator(termOperator) - termQueries = append(termQueries, messageQ) - } - } - - if params.ExcludedTerms != "" { - messageQ := bleve.NewMatchQuery(params.ExcludedTerms) - messageQ.SetField("Message") - messageQ.SetOperator(termOperator) - notTermQueries = append(notTermQueries, messageQ) - } - } - } - - allTermsQ := bleve.NewBooleanQuery() - allTermsQ.AddMustNot(notTermQueries...) - if searchParams[0].OrTerms { - allTermsQ.AddShould(termQueries...) - } else { - allTermsQ.AddMust(termQueries...) - } - - query := bleve.NewBooleanQuery() - query.AddMust(channelDisjunctionQ) - - if len(termQueries) > 0 || len(notTermQueries) > 0 { - query.AddMust(allTermsQ) - } - - if len(filters) > 0 { - query.AddMust(bleve.NewConjunctionQuery(filters...)) - } - if len(notFilters) > 0 { - query.AddMustNot(notFilters...) - } - - search := bleve.NewSearchRequestOptions(query, perPage, page*perPage, false) - search.SortBy([]string{"-CreateAt"}) - results, err := b.PostIndex.Search(search) - if err != nil { - return nil, nil, model.NewAppError("Bleveengine.SearchPosts", "bleveengine.search_posts.error", nil, "", http.StatusInternalServerError).Wrap(err) - } - - postIds := []string{} - matches := model.PostSearchMatches{} - - for _, r := range results.Hits { - postIds = append(postIds, r.ID) - } - - return postIds, matches, nil -} - -func (b *BleveEngine) deletePosts(searchRequest *bleve.SearchRequest, batchSize int) (int64, error) { - resultsCount := int64(0) - - for { - // As we are deleting the posts after fetching them, we need to keep - // From fixed always to 0 - searchRequest.From = 0 - searchRequest.Size = batchSize - results, err := b.PostIndex.Search(searchRequest) - if err != nil { - return -1, err - } - batch := b.PostIndex.NewBatch() - for _, post := range results.Hits { - batch.Delete(post.ID) - } - if err := b.PostIndex.Batch(batch); err != nil { - return -1, err - } - resultsCount += int64(results.Hits.Len()) - if results.Hits.Len() < batchSize { - break - } - } - - return resultsCount, nil -} - -func (b *BleveEngine) DeleteChannelPosts(rctx request.CTX, channelID string) *model.AppError { - b.Mutex.RLock() - defer b.Mutex.RUnlock() - - query := bleve.NewTermQuery(channelID) - query.SetField("ChannelId") - search := bleve.NewSearchRequest(query) - deleted, err := b.deletePosts(search, DeletePostsBatchSize) - if err != nil { - return model.NewAppError("Bleveengine.DeleteChannelPosts", - "bleveengine.delete_channel_posts.error", nil, - err.Error(), http.StatusInternalServerError) - } - - rctx.Logger().Info("Posts for channel deleted", mlog.String("channel_id", channelID), mlog.Int("deleted", deleted)) - - return nil -} - -func (b *BleveEngine) DeleteUserPosts(rctx request.CTX, userID string) *model.AppError { - b.Mutex.RLock() - defer b.Mutex.RUnlock() - - query := bleve.NewTermQuery(userID) - query.SetField("UserId") - search := bleve.NewSearchRequest(query) - deleted, err := b.deletePosts(search, DeletePostsBatchSize) - if err != nil { - return model.NewAppError("Bleveengine.DeleteUserPosts", - "bleveengine.delete_user_posts.error", nil, - err.Error(), http.StatusInternalServerError) - } - - rctx.Logger().Info("Posts for user deleted", mlog.String("user_id", userID), mlog.Int("deleted", deleted)) - - return nil -} - -func (b *BleveEngine) DeletePost(post *model.Post) *model.AppError { - b.Mutex.RLock() - defer b.Mutex.RUnlock() - - if err := b.PostIndex.Delete(post.Id); err != nil { - return model.NewAppError("Bleveengine.DeletePost", "bleveengine.delete_post.error", nil, "", http.StatusInternalServerError).Wrap(err) - } - return nil -} - -func (b *BleveEngine) IndexChannel(_ request.CTX, channel *model.Channel, userIDs, teamMemberIDs []string) *model.AppError { - b.Mutex.RLock() - defer b.Mutex.RUnlock() - - blvChannel := BLVChannelFromChannel(channel, userIDs, teamMemberIDs) - if err := b.ChannelIndex.Index(blvChannel.Id, blvChannel); err != nil { - return model.NewAppError("Bleveengine.IndexChannel", "bleveengine.index_channel.error", nil, "", http.StatusInternalServerError).Wrap(err) - } - return nil -} - -func (b *BleveEngine) SearchChannels(teamId, userID, term string, isGuest, _ bool) ([]string, *model.AppError) { - // This query essentially boils down to (if teamID is passed): - // match teamID == <> - // AND - // match term == <> - // AND - // match (channelType != 'P' || (<> in userIDs && channelType == 'P')) - - // (or if teamID is not passed) - // <> in teamMemberIds - // AND - // match term == <> - // AND - // match (channelType != 'P' || (<> in userIDs && channelType == 'P')) - - // (or if isGuest is true) - // <> in teamMemberIds - // AND - // match term == <> - // AND - // match (<> in userIDs) - - queries := []query.Query{} - if teamId != "" { - teamIdQ := bleve.NewTermQuery(teamId) - teamIdQ.SetField("TeamId") - queries = append(queries, teamIdQ) - } else { - teamMemberQ := bleve.NewTermQuery(userID) - teamMemberQ.SetField("TeamMemberIDs") - queries = append(queries, teamMemberQ) - } - - if isGuest { - userQ := bleve.NewBooleanQuery() - userIDQ := bleve.NewTermQuery(userID) - userIDQ.SetField("UserIDs") - userQ.AddMust(userIDQ) - queries = append(queries, userIDQ) - } else { - boolNotPrivate := bleve.NewBooleanQuery() - privateQ := bleve.NewTermQuery(string(model.ChannelTypePrivate)) - privateQ.SetField("Type") - boolNotPrivate.AddMustNot(privateQ) - - userQ := bleve.NewBooleanQuery() - userIDQ := bleve.NewTermQuery(userID) - userIDQ.SetField("UserIDs") - userQ.AddMust(userIDQ) - userQ.AddMust(privateQ) - - channelTypeQ := bleve.NewDisjunctionQuery() - channelTypeQ.AddQuery(boolNotPrivate) - channelTypeQ.AddQuery(userQ) // userID && 'p' - queries = append(queries, channelTypeQ) - } - - if term != "" { - nameSuggestQ := bleve.NewPrefixQuery(strings.ToLower(term)) - nameSuggestQ.SetField("NameSuggest") - queries = append(queries, nameSuggestQ) - } - - query := bleve.NewSearchRequest(bleve.NewConjunctionQuery(queries...)) - query.Size = model.ChannelSearchDefaultLimit - results, err := b.ChannelIndex.Search(query) - if err != nil { - return nil, model.NewAppError("Bleveengine.SearchChannels", "bleveengine.search_channels.error", nil, "", http.StatusInternalServerError).Wrap(err) - } - - channelIds := []string{} - for _, result := range results.Hits { - channelIds = append(channelIds, result.ID) - } - - return channelIds, nil -} - -func (b *BleveEngine) DeleteChannel(channel *model.Channel) *model.AppError { - b.Mutex.RLock() - defer b.Mutex.RUnlock() - - if err := b.ChannelIndex.Delete(channel.Id); err != nil { - return model.NewAppError("Bleveengine.DeleteChannel", "bleveengine.delete_channel.error", nil, "", http.StatusInternalServerError).Wrap(err) - } - return nil -} - -func (b *BleveEngine) IndexUser(_ request.CTX, user *model.User, teamsIds, channelsIds []string) *model.AppError { - b.Mutex.RLock() - defer b.Mutex.RUnlock() - - blvUser := BLVUserFromUserAndTeams(user, teamsIds, channelsIds) - if err := b.UserIndex.Index(blvUser.Id, blvUser); err != nil { - return model.NewAppError("Bleveengine.IndexUser", "bleveengine.index_user.error", nil, "", http.StatusInternalServerError).Wrap(err) - } - return nil -} - -func (b *BleveEngine) SearchUsersInChannel(teamId, channelId string, restrictedToChannels []string, term string, options *model.UserSearchOptions) ([]string, []string, *model.AppError) { - if restrictedToChannels != nil && len(restrictedToChannels) == 0 { - return []string{}, []string{}, nil - } - - // users in channel - var queries []query.Query - if term != "" { - termQ := bleve.NewPrefixQuery(strings.ToLower(term)) - if options.AllowFullNames { - termQ.SetField("SuggestionsWithFullname") - } else { - termQ.SetField("SuggestionsWithoutFullname") - } - queries = append(queries, termQ) - } - - channelIdQ := bleve.NewTermQuery(channelId) - channelIdQ.SetField("ChannelsIds") - queries = append(queries, channelIdQ) - - query := bleve.NewConjunctionQuery(queries...) - - uchanSearch := bleve.NewSearchRequest(query) - uchanSearch.Size = options.Limit - uchan, err := b.UserIndex.Search(uchanSearch) - if err != nil { - return nil, nil, model.NewAppError("Bleveengine.SearchUsersInChannel", "bleveengine.search_users_in_channel.uchan.error", nil, "", http.StatusInternalServerError).Wrap(err) - } - - // users not in channel - boolQ := bleve.NewBooleanQuery() - - if term != "" { - termQ := bleve.NewPrefixQuery(strings.ToLower(term)) - if options.AllowFullNames { - termQ.SetField("SuggestionsWithFullname") - } else { - termQ.SetField("SuggestionsWithoutFullname") - } - boolQ.AddMust(termQ) - } - - teamIdQ := bleve.NewTermQuery(teamId) - teamIdQ.SetField("TeamsIds") - boolQ.AddMust(teamIdQ) - - outsideChannelIdQ := bleve.NewTermQuery(channelId) - outsideChannelIdQ.SetField("ChannelsIds") - boolQ.AddMustNot(outsideChannelIdQ) - - if len(restrictedToChannels) > 0 { - restrictedChannelsQ := bleve.NewDisjunctionQuery() - for _, channelId := range restrictedToChannels { - restrictedChannelQ := bleve.NewTermQuery(channelId) - restrictedChannelsQ.AddQuery(restrictedChannelQ) - } - boolQ.AddMust(restrictedChannelsQ) - } - - nuchanSearch := bleve.NewSearchRequest(boolQ) - nuchanSearch.Size = options.Limit - nuchan, err := b.UserIndex.Search(nuchanSearch) - if err != nil { - return nil, nil, model.NewAppError("Bleveengine.SearchUsersInChannel", "bleveengine.search_users_in_channel.nuchan.error", nil, "", http.StatusInternalServerError).Wrap(err) - } - - uchanIds := []string{} - for _, result := range uchan.Hits { - uchanIds = append(uchanIds, result.ID) - } - - nuchanIds := []string{} - for _, result := range nuchan.Hits { - nuchanIds = append(nuchanIds, result.ID) - } - - return uchanIds, nuchanIds, nil -} - -func (b *BleveEngine) SearchUsersInTeam(teamId string, restrictedToChannels []string, term string, options *model.UserSearchOptions) ([]string, *model.AppError) { - if restrictedToChannels != nil && len(restrictedToChannels) == 0 { - return []string{}, nil - } - - var rootQ query.Query - if term == "" && teamId == "" && restrictedToChannels == nil { - rootQ = bleve.NewMatchAllQuery() - } else { - boolQ := bleve.NewBooleanQuery() - - if term != "" { - termQ := bleve.NewPrefixQuery(strings.ToLower(term)) - if options.AllowFullNames { - termQ.SetField("SuggestionsWithFullname") - } else { - termQ.SetField("SuggestionsWithoutFullname") - } - boolQ.AddMust(termQ) - } - - if len(restrictedToChannels) > 0 { - // restricted channels are already filtered by team, so we - // can search only those matches - restrictedChannelsQ := []query.Query{} - for _, channelId := range restrictedToChannels { - channelIdQ := bleve.NewTermQuery(channelId) - channelIdQ.SetField("ChannelsIds") - restrictedChannelsQ = append(restrictedChannelsQ, channelIdQ) - } - boolQ.AddMust(bleve.NewDisjunctionQuery(restrictedChannelsQ...)) - } else { - // this means that we only need to restrict by team - if teamId != "" { - teamIdQ := bleve.NewTermQuery(teamId) - teamIdQ.SetField("TeamsIds") - boolQ.AddMust(teamIdQ) - } - } - - rootQ = boolQ - } - - search := bleve.NewSearchRequest(rootQ) - search.Size = options.Limit - results, err := b.UserIndex.Search(search) - if err != nil { - return nil, model.NewAppError("Bleveengine.SearchUsersInTeam", "bleveengine.search_users_in_team.error", nil, "", http.StatusInternalServerError).Wrap(err) - } - - usersIds := []string{} - for _, r := range results.Hits { - usersIds = append(usersIds, r.ID) - } - - return usersIds, nil -} - -func (b *BleveEngine) DeleteUser(user *model.User) *model.AppError { - b.Mutex.RLock() - defer b.Mutex.RUnlock() - - if err := b.UserIndex.Delete(user.Id); err != nil { - return model.NewAppError("Bleveengine.DeleteUser", "bleveengine.delete_user.error", nil, "", http.StatusInternalServerError).Wrap(err) - } - return nil -} - -func (b *BleveEngine) IndexFile(file *model.FileInfo, channelId string) *model.AppError { - b.Mutex.RLock() - defer b.Mutex.RUnlock() - - blvFile := BLVFileFromFileInfo(file, channelId) - if err := b.FileIndex.Index(blvFile.Id, blvFile); err != nil { - return model.NewAppError("Bleveengine.IndexFile", "bleveengine.index_file.error", nil, "", http.StatusInternalServerError).Wrap(err) - } - return nil -} - -func (b *BleveEngine) SearchFiles(channels model.ChannelList, searchParams []*model.SearchParams, page, perPage int) ([]string, *model.AppError) { - channelQueries := []query.Query{} - for _, channel := range channels { - channelIdQ := bleve.NewTermQuery(channel.Id) - channelIdQ.SetField("ChannelId") - channelQueries = append(channelQueries, channelIdQ) - } - channelDisjunctionQ := bleve.NewDisjunctionQuery(channelQueries...) - - var termQueries []query.Query - var notTermQueries []query.Query - var filters []query.Query - var notFilters []query.Query - - for i, params := range searchParams { - var termOperator query.MatchQueryOperator = query.MatchQueryOperatorAnd - if searchParams[0].OrTerms { - termOperator = query.MatchQueryOperatorOr - } - - // Date, channels and FromUsers filters come in all - // searchParams iteration, and as they are global to the - // query, we only need to process them once - if i == 0 { - if len(params.InChannels) > 0 { - inChannels := []query.Query{} - for _, channelId := range params.InChannels { - channelQ := bleve.NewTermQuery(channelId) - channelQ.SetField("ChannelId") - inChannels = append(inChannels, channelQ) - } - filters = append(filters, bleve.NewDisjunctionQuery(inChannels...)) - } - - if len(params.ExcludedChannels) > 0 { - excludedChannels := []query.Query{} - for _, channelId := range params.ExcludedChannels { - channelQ := bleve.NewTermQuery(channelId) - channelQ.SetField("ChannelId") - excludedChannels = append(excludedChannels, channelQ) - } - notFilters = append(notFilters, bleve.NewDisjunctionQuery(excludedChannels...)) - } - - if len(params.FromUsers) > 0 { - fromUsers := []query.Query{} - for _, userId := range params.FromUsers { - userQ := bleve.NewTermQuery(userId) - userQ.SetField("CreatorId") - fromUsers = append(fromUsers, userQ) - } - filters = append(filters, bleve.NewDisjunctionQuery(fromUsers...)) - } - - if len(params.ExcludedUsers) > 0 { - excludedUsers := []query.Query{} - for _, userId := range params.ExcludedUsers { - userQ := bleve.NewTermQuery(userId) - userQ.SetField("CreatorId") - excludedUsers = append(excludedUsers, userQ) - } - notFilters = append(notFilters, bleve.NewDisjunctionQuery(excludedUsers...)) - } - - if len(params.Extensions) > 0 { - extensions := []query.Query{} - for _, extension := range params.Extensions { - extensionQ := bleve.NewTermQuery(extension) - extensionQ.SetField("Extension") - extensions = append(extensions, extensionQ) - } - filters = append(filters, bleve.NewDisjunctionQuery(extensions...)) - } - - if len(params.ExcludedExtensions) > 0 { - excludedExtensions := []query.Query{} - for _, extension := range params.ExcludedExtensions { - extensionQ := bleve.NewTermQuery(extension) - extensionQ.SetField("Extension") - excludedExtensions = append(excludedExtensions, extensionQ) - } - notFilters = append(notFilters, bleve.NewDisjunctionQuery(excludedExtensions...)) - } - - if params.OnDate != "" { - before, after := params.GetOnDateMillis() - beforeFloat64 := float64(before) - afterFloat64 := float64(after) - onDateQ := bleve.NewNumericRangeQuery(&beforeFloat64, &afterFloat64) - onDateQ.SetField("CreateAt") - filters = append(filters, onDateQ) - } else { - if params.AfterDate != "" || params.BeforeDate != "" { - var rangeMin, rangeMax *float64 - if params.AfterDate != "" { - minf := float64(params.GetAfterDateMillis()) - rangeMin = &minf - } - - if params.BeforeDate != "" { - maxf := float64(params.GetBeforeDateMillis()) - rangeMax = &maxf - } - - dateQ := bleve.NewNumericRangeQuery(rangeMin, rangeMax) - dateQ.SetField("CreateAt") - filters = append(filters, dateQ) - } - - if params.ExcludedAfterDate != "" { - minf := float64(params.GetExcludedAfterDateMillis()) - dateQ := bleve.NewNumericRangeQuery(&minf, nil) - dateQ.SetField("CreateAt") - notFilters = append(notFilters, dateQ) - } - - if params.ExcludedBeforeDate != "" { - maxf := float64(params.GetExcludedBeforeDateMillis()) - dateQ := bleve.NewNumericRangeQuery(nil, &maxf) - dateQ.SetField("CreateAt") - notFilters = append(notFilters, dateQ) - } - - if params.ExcludedDate != "" { - before, after := params.GetExcludedDateMillis() - beforef := float64(before) - afterf := float64(after) - onDateQ := bleve.NewNumericRangeQuery(&beforef, &afterf) - onDateQ.SetField("CreateAt") - notFilters = append(notFilters, onDateQ) - } - } - } - - if params.Terms != "" { - terms := []string{} - for term := range strings.SplitSeq(params.Terms, " ") { - if strings.HasSuffix(term, "*") { - nameQ := bleve.NewWildcardQuery(term) - nameQ.SetField("Name") - contentQ := bleve.NewWildcardQuery(term) - contentQ.SetField("Content") - termQueries = append(termQueries, bleve.NewDisjunctionQuery(nameQ, contentQ)) - } else { - terms = append(terms, term) - } - } - - if len(terms) > 0 { - nameQ := bleve.NewMatchQuery(strings.Join(terms, " ")) - nameQ.SetField("Name") - nameQ.SetOperator(termOperator) - contentQ := bleve.NewMatchQuery(strings.Join(terms, " ")) - contentQ.SetField("Content") - contentQ.SetOperator(termOperator) - termQueries = append(termQueries, bleve.NewDisjunctionQuery(nameQ, contentQ)) - } - } - - if params.ExcludedTerms != "" { - nameQ := bleve.NewMatchQuery(params.ExcludedTerms) - nameQ.SetField("Name") - nameQ.SetOperator(termOperator) - contentQ := bleve.NewMatchQuery(params.ExcludedTerms) - contentQ.SetField("Content") - contentQ.SetOperator(termOperator) - notTermQueries = append(notTermQueries, bleve.NewDisjunctionQuery(nameQ, contentQ)) - } - } - - allTermsQ := bleve.NewBooleanQuery() - allTermsQ.AddMustNot(notTermQueries...) - if searchParams[0].OrTerms { - allTermsQ.AddShould(termQueries...) - } else { - allTermsQ.AddMust(termQueries...) - } - - query := bleve.NewBooleanQuery() - query.AddMust(channelDisjunctionQ) - - if len(termQueries) > 0 || len(notTermQueries) > 0 { - query.AddMust(allTermsQ) - } - - if len(filters) > 0 { - query.AddMust(bleve.NewConjunctionQuery(filters...)) - } - if len(notFilters) > 0 { - query.AddMustNot(notFilters...) - } - - search := bleve.NewSearchRequestOptions(query, perPage, page*perPage, false) - search.SortBy([]string{"-CreateAt"}) - results, err := b.FileIndex.Search(search) - if err != nil { - return nil, model.NewAppError("Bleveengine.SearchFiles", "bleveengine.search_files.error", nil, "", http.StatusInternalServerError).Wrap(err) - } - - fileIds := []string{} - - for _, r := range results.Hits { - fileIds = append(fileIds, r.ID) - } - - return fileIds, nil -} - -func (b *BleveEngine) DeleteFile(fileID string) *model.AppError { - b.Mutex.RLock() - defer b.Mutex.RUnlock() - - if err := b.FileIndex.Delete(fileID); err != nil { - return model.NewAppError("Bleveengine.DeleteFile", "bleveengine.delete_file.error", nil, "", http.StatusInternalServerError).Wrap(err) - } - return nil -} - -func (b *BleveEngine) deleteFiles(searchRequest *bleve.SearchRequest, batchSize int) (int64, error) { - resultsCount := int64(0) - - for { - // As we are deleting the files after fetching them, we need to keep - // From fixed always to 0 - searchRequest.From = 0 - searchRequest.Size = batchSize - results, err := b.FileIndex.Search(searchRequest) - if err != nil { - return -1, err - } - batch := b.FileIndex.NewBatch() - for _, file := range results.Hits { - batch.Delete(file.ID) - } - if err := b.FileIndex.Batch(batch); err != nil { - return -1, err - } - resultsCount += int64(results.Hits.Len()) - if results.Hits.Len() < batchSize { - break - } - } - - return resultsCount, nil -} - -func (b *BleveEngine) DeleteUserFiles(rctx request.CTX, userID string) *model.AppError { - b.Mutex.RLock() - defer b.Mutex.RUnlock() - - query := bleve.NewTermQuery(userID) - query.SetField("CreatorId") - search := bleve.NewSearchRequest(query) - deleted, err := b.deleteFiles(search, DeleteFilesBatchSize) - if err != nil { - return model.NewAppError("Bleveengine.DeleteUserFiles", - "bleveengine.delete_user_files.error", nil, - err.Error(), http.StatusInternalServerError) - } - - rctx.Logger().Info("Files for user deleted", mlog.String("user_id", userID), mlog.Int("deleted", deleted)) - - return nil -} - -func (b *BleveEngine) DeletePostFiles(rctx request.CTX, postID string) *model.AppError { - b.Mutex.RLock() - defer b.Mutex.RUnlock() - - query := bleve.NewTermQuery(postID) - query.SetField("PostId") - search := bleve.NewSearchRequest(query) - deleted, err := b.deleteFiles(search, DeleteFilesBatchSize) - if err != nil { - return model.NewAppError("Bleveengine.DeletePostFiles", - "bleveengine.delete_post_files.error", nil, - err.Error(), http.StatusInternalServerError) - } - - rctx.Logger().Info("Files for post deleted", mlog.String("post_id", postID), mlog.Int("deleted", deleted)) - - return nil -} - -func (b *BleveEngine) DeleteFilesBatch(rctx request.CTX, endTime, limit int64) *model.AppError { - b.Mutex.RLock() - defer b.Mutex.RUnlock() - - endTimeFloat := float64(endTime) - query := bleve.NewNumericRangeQuery(nil, &endTimeFloat) - query.SetField("CreateAt") - search := bleve.NewSearchRequestOptions(query, int(limit), 0, false) - search.SortBy([]string{"-CreateAt"}) - - deleted, err := b.deleteFiles(search, DeleteFilesBatchSize) - if err != nil { - return model.NewAppError("Bleveengine.DeleteFilesBatch", - "bleveengine.delete_files_batch.error", nil, - err.Error(), http.StatusInternalServerError) - } - - rctx.Logger().Info("Files in batch deleted", mlog.Int("endTime", endTime), mlog.Int("limit", limit), mlog.Int("deleted", deleted)) - - return nil -} diff --git a/server/platform/services/searchengine/bleveengine/testlib.go b/server/platform/services/searchengine/bleveengine/testlib.go deleted file mode 100644 index b6475a66ff0..00000000000 --- a/server/platform/services/searchengine/bleveengine/testlib.go +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. -// See LICENSE.txt for license information. - -package bleveengine - -import ( - "strconv" - - "github.com/mattermost/mattermost/server/public/model" -) - -func createPost(userId string, channelId string) *model.Post { - post := &model.Post{ - Message: model.NewRandomString(15), - ChannelId: channelId, - PendingPostId: model.NewId() + ":" + strconv.FormatInt(model.GetMillis(), 10), - UserId: userId, - CreateAt: 1000000, - } - post.PreSave() - - return post -} diff --git a/server/platform/services/searchengine/searchengine.go b/server/platform/services/searchengine/searchengine.go index 67e52462ef7..2117d760ba5 100644 --- a/server/platform/services/searchengine/searchengine.go +++ b/server/platform/services/searchengine/searchengine.go @@ -17,14 +17,9 @@ func (seb *Broker) RegisterElasticsearchEngine(es SearchEngineInterface) { seb.ElasticsearchEngine = es } -func (seb *Broker) RegisterBleveEngine(be SearchEngineInterface) { - seb.BleveEngine = be -} - type Broker struct { cfg *model.Config ElasticsearchEngine SearchEngineInterface - BleveEngine SearchEngineInterface } func (seb *Broker) UpdateConfig(cfg *model.Config) *model.AppError { @@ -33,10 +28,6 @@ func (seb *Broker) UpdateConfig(cfg *model.Config) *model.AppError { seb.ElasticsearchEngine.UpdateConfig(cfg) } - if seb.BleveEngine != nil { - seb.BleveEngine.UpdateConfig(cfg) - } - return nil } @@ -45,9 +36,6 @@ func (seb *Broker) GetActiveEngines() []SearchEngineInterface { if seb.ElasticsearchEngine != nil && seb.ElasticsearchEngine.IsActive() { engines = append(engines, seb.ElasticsearchEngine) } - if seb.BleveEngine != nil && seb.BleveEngine.IsActive() && seb.BleveEngine.IsIndexingEnabled() { - engines = append(engines, seb.BleveEngine) - } return engines } diff --git a/server/platform/services/searchengine/searchengine_test.go b/server/platform/services/searchengine/searchengine_test.go index 422385fe252..58375a96802 100644 --- a/server/platform/services/searchengine/searchengine_test.go +++ b/server/platform/services/searchengine/searchengine_test.go @@ -21,21 +21,12 @@ func TestActiveEngine(t *testing.T) { esMock.On("IsActive").Return(true) esMock.On("GetName").Return("elasticsearch") - bleveMock := &mocks.SearchEngineInterface{} - bleveMock.On("IsActive").Return(true) - bleveMock.On("IsIndexingEnabled").Return(true) - bleveMock.On("GetName").Return("bleve") - assert.Equal(t, "database", b.ActiveEngine()) b.ElasticsearchEngine = esMock assert.Equal(t, "elasticsearch", b.ActiveEngine()) b.ElasticsearchEngine = nil - b.BleveEngine = bleveMock - assert.Equal(t, "bleve", b.ActiveEngine()) - - b.BleveEngine = nil *b.cfg.SqlSettings.DisableDatabaseSearch = true assert.Equal(t, "none", b.ActiveEngine()) diff --git a/server/platform/services/telemetry/telemetry.go b/server/platform/services/telemetry/telemetry.go index a33c144cd54..3c3de538574 100644 --- a/server/platform/services/telemetry/telemetry.go +++ b/server/platform/services/telemetry/telemetry.go @@ -74,7 +74,6 @@ const ( TrackConfigDisplay = "config_display" TrackConfigGuestAccounts = "config_guest_accounts" TrackConfigImageProxy = "config_image_proxy" - TrackConfigBleve = "config_bleve" TrackConfigExport = "config_export" TrackConfigWrangler = "config_wrangler" TrackConfigConnectedWorkspaces = "config_connected_workspaces" @@ -945,13 +944,6 @@ func (ts *TelemetryService) trackConfig() { "isdefault_remote_image_proxy_options": isDefault(*cfg.ImageProxySettings.RemoteImageProxyOptions, ""), } - configs[TrackConfigBleve] = map[string]any{ - "enable_indexing": *cfg.BleveSettings.EnableIndexing, - "enable_searching": *cfg.BleveSettings.EnableSearching, - "enable_autocomplete": *cfg.BleveSettings.EnableAutocomplete, - "bulk_indexing_batch_size": *cfg.BleveSettings.BatchSize, - } - configs[TrackConfigExport] = map[string]any{ "retention_days": *cfg.ExportSettings.RetentionDays, } diff --git a/server/public/model/client4.go b/server/public/model/client4.go index 15ee118c1c7..e43bf0afae6 100644 --- a/server/public/model/client4.go +++ b/server/public/model/client4.go @@ -442,10 +442,6 @@ func (c *Client4) elasticsearchRoute() string { return "/elasticsearch" } -func (c *Client4) bleveRoute() string { - return "/bleve" -} - func (c *Client4) commandsRoute() string { return "/commands" } @@ -6500,18 +6496,6 @@ func (c *Client4) PurgeElasticsearchIndexes(ctx context.Context) (*Response, err return BuildResponse(r), nil } -// Bleve Section - -// PurgeBleveIndexes immediately deletes all Bleve indexes. -func (c *Client4) PurgeBleveIndexes(ctx context.Context) (*Response, error) { - r, err := c.DoAPIPost(ctx, c.bleveRoute()+"/purge_indexes", "") - if err != nil { - return BuildResponse(r), err - } - defer closeBody(r) - return BuildResponse(r), nil -} - // Data Retention Section // GetDataRetentionPolicy will get the current global data retention policy details. diff --git a/server/public/model/config.go b/server/public/model/config.go index acd91fa4060..158f2f140d3 100644 --- a/server/public/model/config.go +++ b/server/public/model/config.go @@ -224,9 +224,6 @@ const ( ElasticsearchSettingsESBackend = "elasticsearch" ElasticsearchSettingsOSBackend = "opensearch" - BleveSettingsDefaultIndexDir = "" - BleveSettingsDefaultBatchSize = 10000 - DataRetentionSettingsDefaultMessageRetentionDays = 365 DataRetentionSettingsDefaultMessageRetentionHours = 0 DataRetentionSettingsDefaultFileRetentionDays = 365 @@ -3146,37 +3143,6 @@ func (s *ElasticsearchSettings) SetDefaults() { } } -type BleveSettings struct { - IndexDir *string `access:"experimental_bleve"` // telemetry: none - EnableIndexing *bool `access:"experimental_bleve"` - EnableSearching *bool `access:"experimental_bleve"` - EnableAutocomplete *bool `access:"experimental_bleve"` - BulkIndexingTimeWindowSeconds *int `json:",omitempty"` // telemetry: none - BatchSize *int `access:"experimental_bleve"` -} - -func (bs *BleveSettings) SetDefaults() { - if bs.IndexDir == nil { - bs.IndexDir = NewPointer(BleveSettingsDefaultIndexDir) - } - - if bs.EnableIndexing == nil { - bs.EnableIndexing = NewPointer(false) - } - - if bs.EnableSearching == nil { - bs.EnableSearching = NewPointer(false) - } - - if bs.EnableAutocomplete == nil { - bs.EnableAutocomplete = NewPointer(false) - } - - if bs.BatchSize == nil { - bs.BatchSize = NewPointer(BleveSettingsDefaultBatchSize) - } -} - type DataRetentionSettings struct { EnableMessageDeletion *bool `access:"compliance_data_retention_policy"` EnableFileDeletion *bool `access:"compliance_data_retention_policy"` @@ -3883,7 +3849,6 @@ type Config struct { ExperimentalSettings ExperimentalSettings AnalyticsSettings AnalyticsSettings ElasticsearchSettings ElasticsearchSettings - BleveSettings BleveSettings DataRetentionSettings DataRetentionSettings MessageExportSettings MessageExportSettings JobSettings JobSettings @@ -3995,7 +3960,6 @@ func (o *Config) SetDefaults() { o.ComplianceSettings.SetDefaults() o.LocalizationSettings.SetDefaults() o.ElasticsearchSettings.SetDefaults() - o.BleveSettings.SetDefaults() o.NativeAppSettings.SetDefaults() o.DataRetentionSettings.SetDefaults() o.RateLimitSettings.SetDefaults() @@ -4085,10 +4049,6 @@ func (o *Config) IsValid() *AppError { return appErr } - if appErr := o.BleveSettings.isValid(); appErr != nil { - return appErr - } - if appErr := o.DataRetentionSettings.isValid(); appErr != nil { return appErr } @@ -4647,27 +4607,6 @@ func (s *ElasticsearchSettings) isValid() *AppError { return nil } -func (bs *BleveSettings) isValid() *AppError { - if *bs.EnableIndexing { - if *bs.IndexDir == "" { - return NewAppError("Config.IsValid", "model.config.is_valid.bleve_search.filename.app_error", nil, "", http.StatusBadRequest) - } - } else { - if *bs.EnableSearching { - return NewAppError("Config.IsValid", "model.config.is_valid.bleve_search.enable_searching.app_error", nil, "", http.StatusBadRequest) - } - if *bs.EnableAutocomplete { - return NewAppError("Config.IsValid", "model.config.is_valid.bleve_search.enable_autocomplete.app_error", nil, "", http.StatusBadRequest) - } - } - minBatchSize := 1 - if *bs.BatchSize < minBatchSize { - return NewAppError("Config.IsValid", "model.config.is_valid.bleve_search.bulk_indexing_batch_size.app_error", map[string]any{"BatchSize": minBatchSize}, "", http.StatusBadRequest) - } - - return nil -} - func (s *DataRetentionSettings) isValid() *AppError { if s.MessageRetentionDays == nil || *s.MessageRetentionDays < 0 { return NewAppError("Config.IsValid", "model.config.is_valid.data_retention.message_retention_days_too_low.app_error", nil, "", http.StatusBadRequest) diff --git a/server/public/model/config_test.go b/server/public/model/config_test.go index 1498aed285c..7eaec77644f 100644 --- a/server/public/model/config_test.go +++ b/server/public/model/config_test.go @@ -34,8 +34,7 @@ func TestConfigDefaults(t *testing.T) { if v.Type().Kind() == reflect.Ptr { // Ignoring these 2 settings. // TODO: remove them completely in v8.0. - if name == "config.BleveSettings.BulkIndexingTimeWindowSeconds" || - name == "config.ElasticsearchSettings.BulkIndexingTimeWindowSeconds" || + if name == "config.ElasticsearchSettings.BulkIndexingTimeWindowSeconds" || name == "config.ClusterSettings.EnableExperimentalGossipEncryption" { return } diff --git a/server/public/model/job.go b/server/public/model/job.go index fd106d9c78f..53e7adf7c9e 100644 --- a/server/public/model/job.go +++ b/server/public/model/job.go @@ -15,7 +15,6 @@ const ( JobTypeCLIMessageExport = "cli_message_export" JobTypeElasticsearchPostIndexing = "elasticsearch_post_indexing" JobTypeElasticsearchPostAggregation = "elasticsearch_post_aggregation" - JobTypeBlevePostIndexing = "bleve_post_indexing" JobTypeLdapSync = "ldap_sync" JobTypeMigrations = "migrations" JobTypePlugins = "plugins" @@ -60,7 +59,6 @@ var AllJobTypes = [...]string{ JobTypeMessageExport, JobTypeElasticsearchPostIndexing, JobTypeElasticsearchPostAggregation, - JobTypeBlevePostIndexing, JobTypeLdapSync, JobTypeMigrations, JobTypePlugins, diff --git a/server/public/model/permission.go b/server/public/model/permission.go index 49a4c4d33fe..e973222a679 100644 --- a/server/public/model/permission.go +++ b/server/public/model/permission.go @@ -841,6 +841,7 @@ func initializePermissions() { PermissionScopeSystem, } + // DEPRECATED PermissionPurgeBleveIndexes = &Permission{ "purge_bleve_indexes", "", @@ -848,6 +849,7 @@ func initializePermissions() { PermissionScopeSystem, } + // DEPRECATED PermissionCreatePostBleveIndexesJob = &Permission{ "create_post_bleve_indexes_job", "", @@ -855,6 +857,7 @@ func initializePermissions() { PermissionScopeSystem, } + // DEPRECATED PermissionManagePostBleveIndexesJob = &Permission{ "manage_post_bleve_indexes_job", "", @@ -2095,12 +2098,15 @@ func initializePermissions() { "", PermissionScopeSystem, } + + // DEPRECATED PermissionSysconsoleReadExperimentalBleve = &Permission{ "sysconsole_read_experimental_bleve", "", "", PermissionScopeSystem, } + // DEPRECATED PermissionSysconsoleWriteExperimentalBleve = &Permission{ "sysconsole_write_experimental_bleve", "", @@ -2331,7 +2337,6 @@ func initializePermissions() { PermissionSysconsoleReadComplianceCustomTermsOfService, PermissionSysconsoleReadExperimentalFeatures, PermissionSysconsoleReadExperimentalFeatureFlags, - PermissionSysconsoleReadExperimentalBleve, PermissionSysconsoleReadProductsBoards, PermissionSysconsoleReadIPFilters, } @@ -2391,7 +2396,6 @@ func initializePermissions() { PermissionSysconsoleWriteComplianceCustomTermsOfService, PermissionSysconsoleWriteExperimentalFeatures, PermissionSysconsoleWriteExperimentalFeatureFlags, - PermissionSysconsoleWriteExperimentalBleve, PermissionSysconsoleWriteProductsBoards, PermissionSysconsoleWriteIPFilters, } @@ -2451,9 +2455,6 @@ func initializePermissions() { PermissionManageElasticsearchPostAggregationJob, PermissionReadElasticsearchPostIndexingJob, PermissionReadElasticsearchPostAggregationJob, - PermissionPurgeBleveIndexes, - PermissionCreatePostBleveIndexesJob, - PermissionManagePostBleveIndexesJob, PermissionCreateLdapSyncJob, PermissionManageLdapSyncJob, PermissionReadLdapSyncJob, @@ -2578,6 +2579,11 @@ func initializePermissions() { PermissionSysconsoleWriteIntegrations, PermissionSysconsoleReadCompliance, PermissionSysconsoleWriteCompliance, + PermissionPurgeBleveIndexes, + PermissionCreatePostBleveIndexesJob, + PermissionManagePostBleveIndexesJob, + PermissionSysconsoleReadExperimentalBleve, + PermissionSysconsoleWriteExperimentalBleve, } PlaybookScopedPermissions := []*Permission{ diff --git a/server/public/model/role.go b/server/public/model/role.go index cebfe49f39f..f9436192eec 100644 --- a/server/public/model/role.go +++ b/server/public/model/role.go @@ -163,11 +163,6 @@ func init() { PermissionSysconsoleReadComplianceComplianceMonitoring.Id: { PermissionReadAudits, }, - PermissionSysconsoleWriteExperimentalBleve.Id: { - PermissionCreatePostBleveIndexesJob, - PermissionPurgeBleveIndexes, - PermissionManagePostBleveIndexesJob, - }, PermissionSysconsoleWriteAuthenticationLdap.Id: { PermissionCreateLdapSyncJob, PermissionManageLdapSyncJob, @@ -265,7 +260,6 @@ func init() { PermissionSysconsoleReadComplianceCustomTermsOfService.Id, PermissionSysconsoleReadExperimentalFeatures.Id, PermissionSysconsoleReadExperimentalFeatureFlags.Id, - PermissionSysconsoleReadExperimentalBleve.Id, PermissionSysconsoleReadProductsBoards.Id, } diff --git a/server/public/model/support_packet.go b/server/public/model/support_packet.go index 9ae38f6d4c8..2b7dbf144a6 100644 --- a/server/public/model/support_packet.go +++ b/server/public/model/support_packet.go @@ -98,7 +98,6 @@ type SupportPacketJobList struct { MessageExportJobs []*Job `yaml:"message_export_jobs"` ElasticPostIndexingJobs []*Job `yaml:"elastic_post_indexing_jobs"` ElasticPostAggregationJobs []*Job `yaml:"elastic_post_aggregation_jobs"` - BlevePostIndexingJobs []*Job `yaml:"bleve_post_indexin_jobs"` MigrationJobs []*Job `yaml:"migration_jobs"` } diff --git a/server/public/model/utils_test.go b/server/public/model/utils_test.go index 960632a68a9..bb13c92b39a 100644 --- a/server/public/model/utils_test.go +++ b/server/public/model/utils_test.go @@ -1036,8 +1036,7 @@ func checkNowhereNil(t *testing.T, name string, value any) bool { case reflect.Ptr: // Ignoring these 2 settings. // TODO: remove them completely in v8.0. - if name == "config.BleveSettings.BulkIndexingTimeWindowSeconds" || - name == "config.ElasticsearchSettings.BulkIndexingTimeWindowSeconds" || + if name == "config.ElasticsearchSettings.BulkIndexingTimeWindowSeconds" || name == "config.ClusterSettings.EnableExperimentalGossipEncryption" { return true } diff --git a/webapp/channels/src/actions/admin_actions.jsx b/webapp/channels/src/actions/admin_actions.jsx index 3119db1aa7f..44f0c723243 100644 --- a/webapp/channels/src/actions/admin_actions.jsx +++ b/webapp/channels/src/actions/admin_actions.jsx @@ -405,20 +405,6 @@ export async function rebuildChannelsIndex(success, error) { success(); } -export async function blevePurgeIndexes(success, error) { - const purgeBleveIndexes = bindClientFunc({ - clientFunc: Client4.purgeBleveIndexes, - params: [], - }); - - const {data, error: err} = await dispatch(purgeBleveIndexes); - if (data && success) { - success(data); - } else if (err && error) { - error({id: err.server_error_id, ...err}); - } -} - export function setNavigationBlocked(blocked) { return { type: ActionTypes.SET_NAVIGATION_BLOCKED, diff --git a/webapp/channels/src/components/admin_console/__snapshots__/bleve_settings.test.tsx.snap b/webapp/channels/src/components/admin_console/__snapshots__/bleve_settings.test.tsx.snap deleted file mode 100644 index 415922b02d7..00000000000 --- a/webapp/channels/src/components/admin_console/__snapshots__/bleve_settings.test.tsx.snap +++ /dev/null @@ -1,387 +0,0 @@ -// Jest Snapshot v1, https://goo.gl/fbAQLP - -exports[`components/BleveSettings should match snapshot, disabled 1`] = ` -
-
- - - - - Learn more about Bleve in our documentation." - id="admin.bleve.enableIndexingDescription" - values={ - Object { - "link": [Function], - } - } - /> - } - id="enableIndexing" - label={ - - } - onChange={[Function]} - setByEnv={false} - value={false} - /> - - } - id="indexDir" - label={ - - } - onChange={[Function]} - setByEnv={false} - value="" - /> - - } - > -
- - } - createJobHelpText={ - - } - disabled={true} - getExtraInfoText={[Function]} - jobType="bleve_post_indexing" - /> -
- - - } - disabled={true} - errorMessage={ - Object { - "defaultMessage": "Failed to purge indexes: {error}", - "id": "admin.bleve.purgeIndexesButton.error", - } - } - helpText={ - - } - id="purgeIndexesSection" - includeDetailedError={false} - label={ - - } - requestAction={[MockFunction]} - saveNeeded={false} - showSuccessMessage={true} - successMessage={ - Object { - "defaultMessage": "Indexes purged successfully.", - "id": "admin.bleve.purgeIndexesButton.success", - } - } - /> - - } - id="enableSearching" - label={ - - } - onChange={[Function]} - setByEnv={false} - value={false} - /> - - } - id="enableAutocomplete" - label={ - - } - onChange={[Function]} - setByEnv={false} - value={false} - /> - -
- - } - /> - -
- -
- -
-
-
-`; - -exports[`components/BleveSettings should match snapshot, enabled 1`] = ` -
-
- - - - - Learn more about Bleve in our documentation." - id="admin.bleve.enableIndexingDescription" - values={ - Object { - "link": [Function], - } - } - /> - } - id="enableIndexing" - label={ - - } - onChange={[Function]} - setByEnv={false} - value={true} - /> - - } - id="indexDir" - label={ - - } - onChange={[Function]} - setByEnv={false} - value="bleve.idx" - /> - - } - > -
- - } - createJobHelpText={ - - } - disabled={false} - getExtraInfoText={[Function]} - jobType="bleve_post_indexing" - /> -
- - - } - disabled={false} - errorMessage={ - Object { - "defaultMessage": "Failed to purge indexes: {error}", - "id": "admin.bleve.purgeIndexesButton.error", - } - } - helpText={ - - } - id="purgeIndexesSection" - includeDetailedError={false} - label={ - - } - requestAction={[MockFunction]} - saveNeeded={false} - showSuccessMessage={true} - successMessage={ - Object { - "defaultMessage": "Indexes purged successfully.", - "id": "admin.bleve.purgeIndexesButton.success", - } - } - /> - - } - id="enableSearching" - label={ - - } - onChange={[Function]} - setByEnv={false} - value={false} - /> - - } - id="enableAutocomplete" - label={ - - } - onChange={[Function]} - setByEnv={false} - value={false} - /> - -
- - } - /> - -
- -
- -
-
-
-`; diff --git a/webapp/channels/src/components/admin_console/__snapshots__/database_settings.test.tsx.snap b/webapp/channels/src/components/admin_console/__snapshots__/database_settings.test.tsx.snap index cc746a8425e..90b38a01e35 100644 --- a/webapp/channels/src/components/admin_console/__snapshots__/database_settings.test.tsx.snap +++ b/webapp/channels/src/components/admin_console/__snapshots__/database_settings.test.tsx.snap @@ -387,7 +387,7 @@ exports[`components/DatabaseSettings should match snapshot 1`] = ` className="help-text" > diff --git a/webapp/channels/src/components/admin_console/admin_definition.tsx b/webapp/channels/src/components/admin_console/admin_definition.tsx index 7b5b20144bd..529815fd277 100644 --- a/webapp/channels/src/components/admin_console/admin_definition.tsx +++ b/webapp/channels/src/components/admin_console/admin_definition.tsx @@ -57,7 +57,6 @@ import BillingHistory, {searchableStrings as billingHistorySearchableStrings} fr import BillingSubscriptions, {searchableStrings as billingSubscriptionSearchableStrings} from './billing/billing_subscriptions'; import CompanyInfo, {searchableStrings as billingCompanyInfoSearchableStrings} from './billing/company_info'; import CompanyInfoEdit from './billing/company_info_edit'; -import BleveSettings, {searchableStrings as bleveSearchableStrings} from './bleve_settings'; import BrandImageSetting from './brand_image_setting/brand_image_setting'; import ClientSideUserIdsSetting from './client_side_userids_setting'; import ClusterSettings, {searchableStrings as clusterSearchableStrings} from './cluster_settings'; @@ -6276,20 +6275,6 @@ const AdminDefinition: AdminDefinitionType = { component: FeatureFlags, }, }, - bleve: { - url: 'experimental/blevesearch', - title: defineMessage({id: 'admin.sidebar.blevesearch', defaultMessage: 'Bleve'}), - isHidden: it.any( - it.configIsTrue('ExperimentalSettings', 'RestrictSystemAdmin'), - it.not(it.userHasReadPermissionOnResource(RESOURCE_KEYS.EXPERIMENTAL.BLEVE)), - ), - isDisabled: it.not(it.userHasWritePermissionOnResource(RESOURCE_KEYS.EXPERIMENTAL.BLEVE)), - searchableStrings: bleveSearchableStrings, - schema: { - id: 'BleveSettings', - component: BleveSettings, - }, - }, }, }, }; diff --git a/webapp/channels/src/components/admin_console/admin_sidebar/__snapshots__/admin_sidebar.test.tsx.snap b/webapp/channels/src/components/admin_console/admin_sidebar/__snapshots__/admin_sidebar.test.tsx.snap index 4a3aef70e9c..908aa155098 100644 --- a/webapp/channels/src/components/admin_console/admin_sidebar/__snapshots__/admin_sidebar.test.tsx.snap +++ b/webapp/channels/src/components/admin_console/admin_sidebar/__snapshots__/admin_sidebar.test.tsx.snap @@ -857,17 +857,6 @@ exports[`components/AdminSidebar should match snapshot 1`] = ` /> } /> - - } - /> @@ -1947,17 +1936,6 @@ exports[`components/AdminSidebar should match snapshot with license with enterpr /> } /> - - } - /> @@ -3024,17 +3002,6 @@ exports[`components/AdminSidebar should match snapshot with license with enterpr /> } /> - - } - /> @@ -4012,17 +3979,6 @@ exports[`components/AdminSidebar should match snapshot with license with profess /> } /> - - } - /> @@ -4717,17 +4673,6 @@ exports[`components/AdminSidebar should match snapshot with workspace optimizati /> } /> - - } - /> @@ -5477,17 +5422,6 @@ exports[`components/AdminSidebar should match snapshot, not prevent the console /> } /> - - } - /> @@ -6182,17 +6116,6 @@ exports[`components/AdminSidebar should match snapshot, render plugins without a /> } /> - - } - /> @@ -7084,17 +7007,6 @@ exports[`components/AdminSidebar should match snapshot, with license (with all f /> } /> - - } - /> @@ -8072,17 +7984,6 @@ exports[`components/AdminSidebar should match snapshot, with license (without an /> } /> - - } - /> diff --git a/webapp/channels/src/components/admin_console/bleve_settings.test.tsx b/webapp/channels/src/components/admin_console/bleve_settings.test.tsx deleted file mode 100644 index 2240084fd32..00000000000 --- a/webapp/channels/src/components/admin_console/bleve_settings.test.tsx +++ /dev/null @@ -1,51 +0,0 @@ -// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. -// See LICENSE.txt for license information. - -import {shallow} from 'enzyme'; -import React from 'react'; - -import type {AdminConfig} from '@mattermost/types/config'; - -import BleveSettings from 'components/admin_console/bleve_settings'; - -jest.mock('actions/admin_actions.jsx', () => { - return { - blevePurgeIndexes: jest.fn(), - }; -}); - -describe('components/BleveSettings', () => { - test('should match snapshot, disabled', () => { - const config = { - BleveSettings: { - IndexDir: '', - EnableIndexing: false, - EnableSearching: false, - EnableAutocomplete: false, - }, - } as AdminConfig; - const wrapper = shallow( - , - ); - expect(wrapper).toMatchSnapshot(); - }); - - test('should match snapshot, enabled', () => { - const config = { - BleveSettings: { - IndexDir: 'bleve.idx', - EnableIndexing: true, - EnableSearching: false, - EnableAutocomplete: false, - }, - } as AdminConfig; - const wrapper = shallow( - , - ); - expect(wrapper).toMatchSnapshot(); - }); -}); diff --git a/webapp/channels/src/components/admin_console/bleve_settings.tsx b/webapp/channels/src/components/admin_console/bleve_settings.tsx deleted file mode 100644 index ba5ac5ebcaa..00000000000 --- a/webapp/channels/src/components/admin_console/bleve_settings.tsx +++ /dev/null @@ -1,243 +0,0 @@ -// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. -// See LICENSE.txt for license information. - -import React from 'react'; -import {FormattedMessage, defineMessage, defineMessages} from 'react-intl'; - -import type {AdminConfig} from '@mattermost/types/config'; -import type {Job} from '@mattermost/types/jobs'; - -import {blevePurgeIndexes} from 'actions/admin_actions.jsx'; - -import ExternalLink from 'components/external_link'; - -import {JobStatuses, JobTypes} from 'utils/constants'; - -import BooleanSetting from './boolean_setting'; -import JobsTable from './jobs'; -import OLDAdminSettings from './old_admin_settings'; -import type {BaseProps, BaseState} from './old_admin_settings'; -import RequestButton from './request_button/request_button'; -import SettingSet from './setting_set'; -import SettingsGroup from './settings_group'; -import TextSetting from './text_setting'; - -type Props = BaseProps & { - config: AdminConfig; -}; - -type State = BaseState & { - indexDir: string; - enableIndexing: boolean; - enableSearching: boolean; - enableAutocomplete: boolean; - canSave: boolean; - canPurgeAndIndex: boolean; -}; - -const messages = defineMessages({ - title: {id: 'admin.bleve.title', defaultMessage: 'Bleve'}, - enableIndexingTitle: {id: 'admin.bleve.enableIndexingTitle', defaultMessage: 'Enable Bleve Indexing:'}, - enableIndexingDescription: {id: 'admin.bleve.enableIndexingDescription', defaultMessage: 'When true, indexing of new posts occurs automatically. Search queries will use database search until "Enable Bleve for search queries" is enabled. Learn more about Bleve in our documentation.'}, - bulkIndexingTitle: {id: 'admin.bleve.bulkIndexingTitle', defaultMessage: 'Bulk Indexing:'}, - createJob_help: {id: 'admin.bleve.createJob.help', defaultMessage: 'All users, channels and posts in the database will be indexed from oldest to newest. Bleve is available during indexing but search results may be incomplete until the indexing job is complete.'}, - purgeIndexesHelpText: {id: 'admin.bleve.purgeIndexesHelpText', defaultMessage: 'Purging will entirely remove the content of the Bleve index directory. Search results may be incomplete until a bulk index of the existing database is rebuilt.'}, - purgeIndexesButton: {id: 'admin.bleve.purgeIndexesButton', defaultMessage: 'Purge Index'}, - purgeIndexesButton_label: {id: 'admin.bleve.purgeIndexesButton.label', defaultMessage: 'Purge Indexes:'}, - enableSearchingTitle: {id: 'admin.bleve.enableSearchingTitle', defaultMessage: 'Enable Bleve for search queries:'}, - enableSearchingDescription: {id: 'admin.bleve.enableSearchingDescription', defaultMessage: 'When true, Bleve will be used for all search queries using the latest index. Search results may be incomplete until a bulk index of the existing post database is finished. When false, database search is used.'}, -}); - -export const searchableStrings = [ - messages.title, - messages.enableIndexingTitle, - messages.enableIndexingDescription, - messages.bulkIndexingTitle, - messages.createJob_help, - messages.purgeIndexesHelpText, - messages.purgeIndexesButton, - messages.purgeIndexesButton_label, - messages.enableSearchingTitle, - messages.enableSearchingDescription, -]; - -export default class BleveSettings extends OLDAdminSettings { - getConfigFromState = (config: Props['config']) => { - if (config && config.BleveSettings) { - config.BleveSettings.IndexDir = this.state.indexDir; - config.BleveSettings.EnableIndexing = this.state.enableIndexing; - config.BleveSettings.EnableSearching = this.state.enableSearching; - config.BleveSettings.EnableAutocomplete = this.state.enableAutocomplete; - } - return config; - }; - - getStateFromConfig(config: Props['config']) { - return { - enableIndexing: config.BleveSettings.EnableIndexing, - indexDir: config.BleveSettings.IndexDir, - enableSearching: config.BleveSettings.EnableSearching, - enableAutocomplete: config.BleveSettings.EnableAutocomplete, - canSave: true, - canPurgeAndIndex: config.BleveSettings.EnableIndexing, - }; - } - - handleSettingChanged = (id: string, value: boolean) => { - if (id === 'enableIndexing') { - if (value === false) { - this.setState({ - enableSearching: false, - enableAutocomplete: false, - }); - } - } - - if (id !== 'enableSearching' && id !== 'enableAutocomplete') { - this.setState({ - canPurgeAndIndex: false, - }); - } - - this.handleChange(id, value); - }; - - handleSaved = () => { - this.setState({ - canPurgeAndIndex: this.state.enableIndexing && this.state.indexDir !== '', - }); - }; - - canSave = () => { - return this.state.canSave; - }; - - getExtraInfo(job: Job) { - if (job.status === JobStatuses.IN_PROGRESS) { - return ( - - ); - } - - return <>; - } - - renderTitle() { - return (); - } - - renderSettings = () => { - return ( - - } - helpText={ - ( - - {chunks} - - ), - }} - /> - } - value={this.state.enableIndexing} - onChange={this.handleSettingChanged} - setByEnv={this.isSetByEnv('BleveSettings.EnableIndexing')} - disabled={this.props.isDisabled} - /> - - } - helpText={ - - } - value={this.state.indexDir} - onChange={this.handleSettingChanged} - setByEnv={this.isSetByEnv('BleveSettings.IndexDir')} - disabled={this.props.isDisabled} - /> - } - > -
- - } - createJobHelpText={} - getExtraInfoText={this.getExtraInfo} - /> -
- - } - buttonText={} - successMessage={defineMessage({ - id: 'admin.bleve.purgeIndexesButton.success', - defaultMessage: 'Indexes purged successfully.', - })} - errorMessage={defineMessage({ - id: 'admin.bleve.purgeIndexesButton.error', - defaultMessage: 'Failed to purge indexes: {error}', - })} - disabled={!this.state.canPurgeAndIndex || this.props.isDisabled} - label={} - /> - } - helpText={} - value={this.state.enableSearching} - disabled={!this.state.enableIndexing || this.props.isDisabled} - onChange={this.handleSettingChanged} - setByEnv={this.isSetByEnv('BleveSettings.EnableSearching')} - /> - - } - helpText={ - - } - value={this.state.enableAutocomplete} - disabled={!this.state.enableIndexing || this.props.isDisabled} - onChange={this.handleSettingChanged} - setByEnv={this.isSetByEnv('BleveSettings.EnableAutocomplete')} - /> - - ); - }; -} diff --git a/webapp/channels/src/components/admin_console/database_settings.tsx b/webapp/channels/src/components/admin_console/database_settings.tsx index 25564658e0b..d7b1067951f 100644 --- a/webapp/channels/src/components/admin_console/database_settings.tsx +++ b/webapp/channels/src/components/admin_console/database_settings.tsx @@ -419,7 +419,7 @@ export default class DatabaseSettings extends OLDAdminSettings {
diff --git a/webapp/channels/src/components/admin_console/system_roles/system_role/__snapshots__/system_role_permissions.test.tsx.snap b/webapp/channels/src/components/admin_console/system_roles/system_role/__snapshots__/system_role_permissions.test.tsx.snap index e3428b3994c..89a7be3cd29 100644 --- a/webapp/channels/src/components/admin_console/system_roles/system_role/__snapshots__/system_role_permissions.test.tsx.snap +++ b/webapp/channels/src/components/admin_console/system_roles/system_role/__snapshots__/system_role_permissions.test.tsx.snap @@ -390,9 +390,6 @@ exports[`admin_console/system_role_permissions should match snapshot 1`] = ` Object { "name": "experimental_feature_flags", }, - Object { - "name": "experimental_bleve", - }, ], } } @@ -705,9 +702,6 @@ exports[`admin_console/system_role_permissions should match snapshot with isLice Object { "name": "experimental_feature_flags", }, - Object { - "name": "experimental_bleve", - }, ], } } diff --git a/webapp/channels/src/components/admin_console/system_roles/system_role/strings.tsx b/webapp/channels/src/components/admin_console/system_roles/system_role/strings.tsx index c26fd44f9ed..c172dd7aa45 100644 --- a/webapp/channels/src/components/admin_console/system_roles/system_role/strings.tsx +++ b/webapp/channels/src/components/admin_console/system_roles/system_role/strings.tsx @@ -434,10 +434,4 @@ export const sectionStrings: Record> = defaultMessage: 'Feature Flags', }, }), - experimental_bleve: defineMessages({ - name: { - id: 'admin.permissions.sysconsole_section_experimental_bleve.name', - defaultMessage: 'Bleve', - }, - }), }; diff --git a/webapp/channels/src/components/admin_console/system_roles/system_role/system_role_permissions.tsx b/webapp/channels/src/components/admin_console/system_roles/system_role/system_role_permissions.tsx index 79b9717fe1f..11c0208ec82 100644 --- a/webapp/channels/src/components/admin_console/system_roles/system_role/system_role_permissions.tsx +++ b/webapp/channels/src/components/admin_console/system_roles/system_role/system_role_permissions.tsx @@ -146,7 +146,6 @@ const sectionsList: SystemSection[] = [ subsections: [ {name: 'experimental_features'}, {name: 'experimental_feature_flags'}, - {name: 'experimental_bleve'}, ], }, ]; diff --git a/webapp/channels/src/i18n/en.json b/webapp/channels/src/i18n/en.json index 13543b2a8e0..6aa45955b9f 100644 --- a/webapp/channels/src/i18n/en.json +++ b/webapp/channels/src/i18n/en.json @@ -519,24 +519,6 @@ "admin.billing.subscriptions.billing_summary.noBillingHistory.link": "See how billing works", "admin.billing.subscriptions.billing_summary.noBillingHistory.title": "No billing history yet", "admin.billing.subscriptions.billing_summary.upcomingInvoice.has_more_line_items": "And {count} more items", - "admin.bleve.bulkIndexingTitle": "Bulk Indexing:", - "admin.bleve.createJob.help": "All users, channels and posts in the database will be indexed from oldest to newest. Bleve is available during indexing but search results may be incomplete until the indexing job is complete.", - "admin.bleve.createJob.title": "Index Now", - "admin.bleve.enableAutocompleteDescription": "When true, Bleve will be used for all autocompletion queries on users and channels using the latest index. Autocompletion results may be incomplete until a bulk index of the existing users and channels database is finished. When false, database autocomplete is used.", - "admin.bleve.enableAutocompleteTitle": "Enable Bleve for autocomplete queries:", - "admin.bleve.enableIndexingDescription": "When true, indexing of new posts occurs automatically. Search queries will use database search until \"Enable Bleve for search queries\" is enabled. {documentationLink}", - "admin.bleve.enableIndexingTitle": "Enable Bleve Indexing:", - "admin.bleve.enableSearchingDescription": "When true, Bleve will be used for all search queries using the latest index. Search results may be incomplete until a bulk index of the existing post database is finished. When false, database search is used.", - "admin.bleve.enableSearchingTitle": "Enable Bleve for search queries:", - "admin.bleve.indexDirDescription": "Directory path to use for store bleve indexes.", - "admin.bleve.indexDirTitle": "Index Directory:", - "admin.bleve.percentComplete": "{percent}% Complete", - "admin.bleve.purgeIndexesButton": "Purge Index", - "admin.bleve.purgeIndexesButton.error": "Failed to purge indexes: {error}", - "admin.bleve.purgeIndexesButton.label": "Purge Indexes:", - "admin.bleve.purgeIndexesButton.success": "Indexes purged successfully.", - "admin.bleve.purgeIndexesHelpText": "Purging will entirely remove the content of the Bleve index directory. Search results may be incomplete until a bulk index of the existing database is rebuilt.", - "admin.bleve.title": "Bleve", "admin.cacheSettings.cacheType.lru": "LRU", "admin.cacheSettings.cacheType.redis": "Redis", "admin.cacheSettings.cacheTypeDesc": "The type of the cache backend. E.g.: \"redis\" or \"lru\"", @@ -2104,7 +2086,6 @@ "admin.permissions.sysconsole_section_environment_web_server.name": "Web Server", "admin.permissions.sysconsole_section_environment.description": "Review server environment configuration such as URLs, database and performance.", "admin.permissions.sysconsole_section_environment.name": "Environment", - "admin.permissions.sysconsole_section_experimental_bleve.name": "Bleve", "admin.permissions.sysconsole_section_experimental_feature_flags.name": "Feature Flags", "admin.permissions.sysconsole_section_experimental_features.name": "Features", "admin.permissions.sysconsole_section_experimental.description": "Review the settings of experimental features", @@ -2683,7 +2664,6 @@ "admin.sidebar.authentication": "Authentication", "admin.sidebar.billing": "Billing & Account", "admin.sidebar.billing_history": "Billing History", - "admin.sidebar.blevesearch": "Bleve", "admin.sidebar.channels": "Channels", "admin.sidebar.company_info": "Company Information", "admin.sidebar.compliance": "Compliance", diff --git a/webapp/platform/types/src/config.ts b/webapp/platform/types/src/config.ts index 0876e88654d..09d13f5689f 100644 --- a/webapp/platform/types/src/config.ts +++ b/webapp/platform/types/src/config.ts @@ -882,14 +882,6 @@ export type ElasticsearchSettings = { IgnoredPurgeIndexes: string; }; -export type BleveSettings = { - IndexDir: string; - EnableIndexing: boolean; - EnableSearching: boolean; - EnableAutocomplete: boolean; - BatchSize: number; -}; - export type DataRetentionSettings = { EnableMessageDeletion: boolean; EnableFileDeletion: boolean; @@ -1059,7 +1051,6 @@ export type AdminConfig = { AnalyticsSettings: AnalyticsSettings; CacheSettings: CacheSettings; ElasticsearchSettings: ElasticsearchSettings; - BleveSettings: BleveSettings; DataRetentionSettings: DataRetentionSettings; MessageExportSettings: MessageExportSettings; JobSettings: JobSettings;