From 91de3d23837c2ec9293c73cb570d08aa86ccba1d Mon Sep 17 00:00:00 2001 From: sabril <5334504+saturninoabril@users.noreply.github.com> Date: Fri, 8 May 2026 10:15:49 +0800 Subject: [PATCH] SEC-10179 Integrate test system IO for Playwright and Cypress (#36376) * change retry to 1, fixed and disabled failed tests * add v2 templates for Cypress and Playwright E2E tests with test system io integration * add commenting to pr * identify more playwrights to fix separately * disable deletion-report.spec for separate fix --------- Co-authored-by: Mattermost Build --- .../e2e-tests-cypress-template-v2.yml | 400 ++++++++++++++++++ .github/workflows/e2e-tests-cypress.yml | 42 +- .../e2e-tests-playwright-template-v2.yml | 357 ++++++++++++++++ .github/workflows/e2e-tests-playwright.yml | 38 +- e2e-tests/cypress/cypress.config.ts | 4 +- .../messaging/message_deletion_spec.js | 2 +- e2e-tests/playwright/playwright.config.ts | 2 +- .../autotranslation/autotranslation.spec.ts | 12 +- .../deletion-report/deletion-report.spec.ts | 2 +- .../team_settings_policy_editor.spec.ts | 2 +- .../demo_plugin_hook_toggle.spec.ts | 2 +- .../user_attributes_admin_editing.spec.ts | 8 +- .../user_attributes/user_attributes.spec.ts | 22 +- 13 files changed, 861 insertions(+), 32 deletions(-) create mode 100644 .github/workflows/e2e-tests-cypress-template-v2.yml create mode 100644 .github/workflows/e2e-tests-playwright-template-v2.yml diff --git a/.github/workflows/e2e-tests-cypress-template-v2.yml b/.github/workflows/e2e-tests-cypress-template-v2.yml new file mode 100644 index 00000000000..c772c4b8411 --- /dev/null +++ b/.github/workflows/e2e-tests-cypress-template-v2.yml @@ -0,0 +1,400 @@ +--- +name: E2E Tests - Cypress Template (v2 - test system io dispatch) + +# Delegates Cypress spec dispatch + reporting to test system io. +# Authenticates via GitHub Actions OIDC; calling job MUST grant +# `id-token: write`. + +on: + workflow_call: + inputs: + test_type: + description: "Type of test run (smoke or full)" + type: string + required: true + workers: + description: "Number of parallel test system io dispatch workers" + type: number + required: false + default: 40 + enabled_docker_services: + description: "Space-separated list of docker services to enable" + type: string + required: false + default: "postgres inbucket minio openldap elasticsearch keycloak" + + commit_sha: + type: string + required: true + branch: + type: string + required: true + build_id: + type: string + required: true + server_image_tag: + description: "Server image tag (e.g., master or short SHA)" + type: string + required: true + server: + type: string + required: false + default: onprem + server_edition: + description: "Server edition: enterprise (default), fips, or team" + type: string + required: false + default: enterprise + server_image_repo: + description: "Docker registry: mattermostdevelopment (default) or mattermost" + type: string + required: false + default: mattermostdevelopment + server_image_aliases: + description: "Comma-separated alias tags for description" + type: string + required: false + + enable_reporting: + type: boolean + required: false + default: false + report_type: + type: string + required: false + ref_branch: + type: string + required: false + pr_number: + type: string + required: false + context_name: + description: "GitHub commit status context name" + type: string + required: true + + cypress_stage: + description: "Comma-separated `// Stage:` tags; spec must share at least one. Empty disables filter." + type: string + required: false + default: "@prod" + cypress_include_group: + description: "Comma-separated `// Group:` tags; spec must share at least one. Empty disables filter." + type: string + required: false + default: "" + cypress_exclude_group: + description: "Comma-separated `// Group:` tags; spec dropped if it shares any." + type: string + required: false + default: "@te_only,@cloud_only,@high_availability" + cypress_skip_on: + description: "Comma-separated active-env tag(s); spec dropped if its `// Skip:` line shares any." + type: string + required: false + default: "@headless" + cypress_sort_first: + description: "Comma-separated `// Group:` tags; matching specs dispatch first." + type: string + required: false + default: "@compliance_export,@elasticsearch,@ldap_group,@ldap" + cypress_sort_last: + description: "Comma-separated `// Group:` tags; matching specs dispatch last." + type: string + required: false + default: "@saml,@keycloak,@plugin,@plugins_uninstall,@mfa,@license_removal" + retest_on_fail: + description: "Re-dispatch failed dispatch units once (whole-spec retry, on top of cypress.config retries)" + type: boolean + required: false + default: true + + secrets: + MM_LICENSE: + required: false + AUTOMATION_DASHBOARD_URL: + required: false + AUTOMATION_DASHBOARD_TOKEN: + required: false + PUSH_NOTIFICATION_SERVER: + required: false + REPORT_WEBHOOK_URL: + required: false + CWS_URL: + required: false + CWS_EXTRA_HTTP_HEADERS: + required: false + +# Callers must grant: contents: read, statuses: write, id-token: write +permissions: + contents: read + statuses: write + id-token: write + pull-requests: write + +env: + SERVER_IMAGE: "${{ inputs.server_image_repo }}/${{ inputs.server_edition == 'fips' && 'mattermost-enterprise-fips-edition' || inputs.server_edition == 'team' && 'mattermost-team-edition' || 'mattermost-enterprise-edition' }}:${{ inputs.server_image_tag }}" + +jobs: + update-initial-status: + runs-on: ubuntu-24.04 + permissions: + contents: read + statuses: write + steps: + - name: ci/set-initial-status + uses: mattermost/actions/delivery/update-commit-status@f324ac89b05cc3511cb06e60642ac2fb829f0a63 + env: + GITHUB_TOKEN: ${{ github.token }} + with: + repository_full_name: ${{ github.repository }} + commit_sha: ${{ inputs.commit_sha }} + context: ${{ inputs.context_name }} + description: "tests running, image_tag:${{ inputs.server_image_tag }}${{ inputs.server_image_aliases && format(' ({0})', inputs.server_image_aliases) || '' }}" + status: pending + + dispatch-begin: + runs-on: ubuntu-24.04 + permissions: + contents: read + id-token: write + pull-requests: write + outputs: + composite-identity-json: ${{ steps.composite-identity.outputs.composite-identity-json }} + workers-matrix: ${{ steps.matrix.outputs.workers }} + start_time: ${{ steps.matrix.outputs.start_time }} + steps: + - name: ci/checkout-repo + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + ref: ${{ inputs.commit_sha }} + fetch-depth: 1 + - name: ci/composite-identity + id: composite-identity + env: + CONTEXT_NAME: ${{ inputs.context_name }} + MM_SHA: ${{ inputs.commit_sha }} + MM_BRANCH: ${{ inputs.branch }} + PR_NUMBER: ${{ inputs.pr_number }} + run: | + # Derive the test-system-io run name from the GitHub commit-status + # context: drop the `e2e-test/` prefix (the framework name already + # implies E2E in the dashboard) and swap remaining `/` for `-` so + # the dashboard URL is path-safe. The commit-status context itself + # stays unchanged elsewhere — branch protection rules depend on it. + NAME="${CONTEXT_NAME#e2e-test/}" + NAME="${NAME//\//-}" + # gh_pr_number is optional; include it only when present. + if [ -n "$PR_NUMBER" ]; then + COMPOSITE_IDENTITY=$(jq -nc \ + --arg repo "${{ github.repository }}" \ + --arg sha "${MM_SHA}" \ + --arg run_id "${GITHUB_RUN_ID}" \ + --arg name "${NAME}" \ + --arg attempt "${GITHUB_RUN_ATTEMPT}" \ + --arg branch "${MM_BRANCH}" \ + --arg pr "${PR_NUMBER}" \ + '{repository:$repo, commit_sha:$sha, gh_run_id:$run_id, name:$name, gh_run_attempt:$attempt, branch:$branch, gh_pr_number:$pr}') + else + COMPOSITE_IDENTITY=$(jq -nc \ + --arg repo "${{ github.repository }}" \ + --arg sha "${MM_SHA}" \ + --arg run_id "${GITHUB_RUN_ID}" \ + --arg name "${NAME}" \ + --arg attempt "${GITHUB_RUN_ATTEMPT}" \ + --arg branch "${MM_BRANCH}" \ + '{repository:$repo, commit_sha:$sha, gh_run_id:$run_id, name:$name, gh_run_attempt:$attempt, branch:$branch}') + fi + echo "composite-identity-json=${COMPOSITE_IDENTITY}" >> $GITHUB_OUTPUT + - name: ci/matrix + id: matrix + run: | + echo "workers=$(jq -nc --argjson n ${{ inputs.workers }} '[range(1; $n+1)]')" >> $GITHUB_OUTPUT + echo "start_time=$(date +%s)" >> $GITHUB_OUTPUT + - name: ci/dispatch-begin + uses: mattermost/mattermost-test-system-io/.github/actions/test-system-io-dispatch-begin@main + with: + use-staging: ${{ vars.E2E_USE_STAGING_TEST_IO_URL != 'false' }} + framework: cypress + repo-dir: ${{ github.workspace }} + composite-identity: ${{ steps.composite-identity.outputs.composite-identity-json }} + total-reports-expected: ${{ inputs.workers }} + retest-on-fail: ${{ inputs.retest_on_fail }} + cypress-stage: ${{ inputs.cypress_stage }} + cypress-include-group: ${{ inputs.cypress_include_group }} + cypress-exclude-group: ${{ inputs.cypress_exclude_group }} + cypress-skip-on: ${{ inputs.cypress_skip_on }} + cypress-sort-first: ${{ inputs.cypress_sort_first }} + cypress-sort-last: ${{ inputs.cypress_sort_last }} + post-pr-comment: 'true' + github-token: ${{ secrets.GITHUB_TOKEN }} + test-type: ${{ inputs.test_type }} + server-edition: ${{ inputs.server_edition }} + + workers: + name: dispatch-run-${{ matrix.worker_index }} + runs-on: ubuntu-24.04 + timeout-minutes: 30 + needs: dispatch-begin + permissions: + contents: read + id-token: write + strategy: + fail-fast: false + matrix: + worker_index: ${{ fromJSON(needs.dispatch-begin.outputs.workers-matrix) }} + env: + COMPOSITE_IDENTITY: ${{ needs.dispatch-begin.outputs.composite-identity-json }} + SERVER: "${{ inputs.server }}" + MM_LICENSE: "${{ secrets.MM_LICENSE }}" + ENABLED_DOCKER_SERVICES: "${{ inputs.enabled_docker_services }}" + TEST: cypress + # The dispatch adapter invokes `npx cypress run` directly (no + # cross-env), so pin TZ here — several specs assume UTC. + TZ: Etc/UTC + BRANCH: "${{ inputs.branch }}" + BUILD_ID: "${{ inputs.build_id }}" + CI_BASE_URL: "${{ inputs.test_type }}-test-${{ matrix.worker_index }}" + CYPRESS_pushNotificationServer: "${{ secrets.PUSH_NOTIFICATION_SERVER }}" + CWS_URL: "${{ secrets.CWS_URL }}" + CWS_EXTRA_HTTP_HEADERS: "${{ secrets.CWS_EXTRA_HTTP_HEADERS }}" + steps: + - name: ci/checkout-repo + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + ref: ${{ inputs.commit_sha }} + fetch-depth: 0 + - name: ci/setup-node + uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0 + with: + node-version-file: ".nvmrc" + cache: npm + cache-dependency-path: "e2e-tests/cypress/package-lock.json" + - name: ci/get-webapp-node-modules + working-directory: webapp + run: make node_modules + - name: ci/cloud-init + working-directory: e2e-tests + run: make cloud-init + - name: ci/start-server + working-directory: e2e-tests + run: make start-server + # `npm ci` in the host context replaces the container-built native + # binaries with host-built ones, since the dispatch adapter spawns + # `npx cypress run` on the host. + - name: ci/prepare-cypress + working-directory: e2e-tests/cypress + run: npm ci + - name: ci/dispatch-run + uses: mattermost/mattermost-test-system-io/.github/actions/test-system-io-dispatch-run@main + with: + use-staging: ${{ vars.E2E_USE_STAGING_TEST_IO_URL != 'false' }} + framework: cypress + composite-identity: ${{ needs.dispatch-begin.outputs.composite-identity-json }} + repo-dir: ${{ github.workspace }} + artifacts-root: ${{ github.workspace }}/worker-artifacts + github-token: ${{ secrets.GITHUB_TOKEN }} + gh-job-name: dispatch-run-${{ matrix.worker_index }} + - name: ci/cloud-teardown + if: always() + working-directory: e2e-tests + run: make cloud-teardown + - name: ci/upload-debug-artifacts + if: always() + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + with: + name: cypress-${{ inputs.test_type }}-${{ inputs.server_edition }}-debug-${{ matrix.worker_index }} + path: | + e2e-tests/cypress/logs/ + e2e-tests/cypress/results/ + e2e-tests/cypress/tests/screenshots/ + worker-artifacts/ + retention-days: 5 + if-no-files-found: ignore + + report: + runs-on: ubuntu-24.04 + needs: [dispatch-begin, workers] + if: always() + permissions: + contents: read + id-token: write + pull-requests: write + outputs: + commit_status_description: ${{ steps.summary.outputs.commit_status_description }} + webhook_payload: ${{ steps.summary.outputs.webhook_payload }} + steps: + - name: ci/checkout-repo + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - name: ci/run-summary + id: summary + continue-on-error: true + uses: mattermost/mattermost-test-system-io/.github/actions/test-system-io-summary@main + with: + use-staging: ${{ vars.E2E_USE_STAGING_TEST_IO_URL != 'false' }} + composite-identity: ${{ needs.dispatch-begin.outputs.composite-identity-json }} + framework: cypress + test-type: ${{ inputs.test_type }} + report-type: ${{ inputs.report_type }} + image-tag: ${{ inputs.server_image_tag }} + image-aliases: ${{ inputs.server_image_aliases }} + server-image: ${{ env.SERVER_IMAGE }} + pr-number: ${{ inputs.pr_number }} + ref-branch: ${{ inputs.ref_branch }} + server-edition: ${{ inputs.server_edition }} + post-pr-comment: 'true' + github-token: ${{ secrets.GITHUB_TOKEN }} + - name: ci/publish-webhook + if: inputs.enable_reporting && env.REPORT_WEBHOOK_URL != '' + env: + REPORT_WEBHOOK_URL: ${{ secrets.REPORT_WEBHOOK_URL }} + PAYLOAD: ${{ steps.summary.outputs.webhook_payload }} + run: | + curl -X POST -H "Content-Type: application/json" -d "$PAYLOAD" "$REPORT_WEBHOOK_URL" + - name: ci/assert-results + env: + SUMMARY_OUTCOME: ${{ steps.summary.outcome }} + run: | + [ "$SUMMARY_OUTCOME" = "success" ] + + update-success-status: + runs-on: ubuntu-24.04 + permissions: + contents: read + statuses: write + if: always() && needs.report.result == 'success' + needs: + - dispatch-begin + - report + steps: + - uses: mattermost/actions/delivery/update-commit-status@f324ac89b05cc3511cb06e60642ac2fb829f0a63 + env: + GITHUB_TOKEN: ${{ github.token }} + with: + repository_full_name: ${{ github.repository }} + commit_sha: ${{ inputs.commit_sha }} + context: ${{ inputs.context_name }} + description: ${{ needs.report.outputs.commit_status_description }} + status: success + target_url: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + + update-failure-status: + runs-on: ubuntu-24.04 + permissions: + contents: read + statuses: write + if: always() && needs.report.result != 'success' + needs: + - dispatch-begin + - report + steps: + - uses: mattermost/actions/delivery/update-commit-status@f324ac89b05cc3511cb06e60642ac2fb829f0a63 + env: + GITHUB_TOKEN: ${{ github.token }} + with: + repository_full_name: ${{ github.repository }} + commit_sha: ${{ inputs.commit_sha }} + context: ${{ inputs.context_name }} + description: ${{ needs.report.outputs.commit_status_description }} + status: failure + target_url: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} diff --git a/.github/workflows/e2e-tests-cypress.yml b/.github/workflows/e2e-tests-cypress.yml index 0fa715e5427..fe638d0a51d 100644 --- a/.github/workflows/e2e-tests-cypress.yml +++ b/.github/workflows/e2e-tests-cypress.yml @@ -166,10 +166,15 @@ jobs: -f target_url="https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" echo "Posted success for ${CONTEXT_NAME}" - cypress-full: + # ── Routing fork ───────────────────────────────────────────────────── + # vars.E2E_USE_TEST_IO_DISPATCH selects between v1 (legacy) and v2. + # vars.E2E_USE_STAGING_TEST_IO_URL toggles v2's staging vs production + # endpoint (default: staging). + + cypress-full-v1: needs: - generate-build-variables - if: inputs.should_run != 'false' + if: inputs.should_run != 'false' && vars.E2E_USE_TEST_IO_DISPATCH != 'true' permissions: contents: read statuses: write @@ -200,3 +205,36 @@ jobs: REPORT_WEBHOOK_URL: ${{ secrets.REPORT_WEBHOOK_URL }} CWS_URL: ${{ secrets.CWS_URL }} CWS_EXTRA_HTTP_HEADERS: ${{ secrets.CWS_EXTRA_HTTP_HEADERS }} + + cypress-full-v2: + needs: + - generate-build-variables + if: inputs.should_run != 'false' && vars.E2E_USE_TEST_IO_DISPATCH == 'true' + permissions: + contents: read + statuses: write + id-token: write + uses: ./.github/workflows/e2e-tests-cypress-template-v2.yml + with: + test_type: full + workers: 40 + enabled_docker_services: "postgres inbucket minio openldap elasticsearch keycloak" + commit_sha: ${{ inputs.commit_sha }} + branch: ${{ needs.generate-build-variables.outputs.branch }} + build_id: ${{ needs.generate-build-variables.outputs.build_id }} + server_image_tag: ${{ needs.generate-build-variables.outputs.server_image_tag }} + server_edition: ${{ inputs.server_edition }} + server_image_repo: ${{ inputs.server_image_repo }} + server_image_aliases: ${{ inputs.server_image_aliases }} + server: ${{ inputs.server }} + enable_reporting: ${{ inputs.enable_reporting }} + report_type: ${{ inputs.report_type }} + ref_branch: ${{ inputs.ref_branch }} + pr_number: ${{ inputs.pr_number }} + context_name: "e2e-test/cypress-full/${{ inputs.server_edition || 'enterprise' }}${{ needs.generate-build-variables.outputs.context_suffix }}" + secrets: + MM_LICENSE: ${{ secrets.MM_LICENSE }} + PUSH_NOTIFICATION_SERVER: ${{ secrets.PUSH_NOTIFICATION_SERVER }} + REPORT_WEBHOOK_URL: ${{ secrets.REPORT_WEBHOOK_URL }} + CWS_URL: ${{ secrets.CWS_URL }} + CWS_EXTRA_HTTP_HEADERS: ${{ secrets.CWS_EXTRA_HTTP_HEADERS }} diff --git a/.github/workflows/e2e-tests-playwright-template-v2.yml b/.github/workflows/e2e-tests-playwright-template-v2.yml new file mode 100644 index 00000000000..ca13516251c --- /dev/null +++ b/.github/workflows/e2e-tests-playwright-template-v2.yml @@ -0,0 +1,357 @@ +--- +name: E2E Tests - Playwright Template (v2 - test system io dispatch) + +# Delegates Playwright spec dispatch + reporting to test system io. +# Authenticates via GitHub Actions OIDC; calling job MUST grant +# `id-token: write`. + +on: + workflow_call: + inputs: + workers: + description: "Number of parallel test system io dispatch workers" + type: number + required: false + default: 8 + enabled_docker_services: + description: "Space-separated list of docker services to enable" + type: string + required: false + default: "postgres inbucket" + + commit_sha: + type: string + required: true + branch: + type: string + required: true + build_id: + type: string + required: true + server_image_tag: + description: "Server image tag (e.g., master or short SHA)" + type: string + required: true + server: + type: string + required: false + default: onprem + server_edition: + description: "Server edition: enterprise (default), fips, or team" + type: string + required: false + default: enterprise + server_image_repo: + description: "Docker registry: mattermostdevelopment (default) or mattermost" + type: string + required: false + default: mattermostdevelopment + server_image_aliases: + description: "Comma-separated alias tags for description" + type: string + required: false + + enable_reporting: + type: boolean + required: false + default: false + report_type: + type: string + required: false + ref_branch: + type: string + required: false + pr_number: + type: string + required: false + context_name: + description: "GitHub commit status context name" + type: string + required: true + + playwright_project: + description: "Playwright project name (passed to dispatch-begin metadata and dispatch-run --project=)." + type: string + required: false + default: chrome + playwright_retries: + description: "Playwright --retries=N (per-spec, in-process retry of flaky tests)" + type: number + required: false + default: 1 + retest_on_fail: + description: "Re-dispatch failed dispatch units once (whole-spec retry, on top of Playwright --retries)" + type: boolean + required: false + default: true + + secrets: + MM_LICENSE: + required: false + REPORT_WEBHOOK_URL: + required: false + +# Callers must grant: contents: read, statuses: write, id-token: write +permissions: + contents: read + statuses: write + id-token: write + pull-requests: write + +env: + SERVER_IMAGE: "${{ inputs.server_image_repo }}/${{ inputs.server_edition == 'fips' && 'mattermost-enterprise-fips-edition' || inputs.server_edition == 'team' && 'mattermost-team-edition' || 'mattermost-enterprise-edition' }}:${{ inputs.server_image_tag }}" + +jobs: + update-initial-status: + runs-on: ubuntu-24.04 + permissions: + contents: read + statuses: write + steps: + - name: ci/set-initial-status + uses: mattermost/actions/delivery/update-commit-status@f324ac89b05cc3511cb06e60642ac2fb829f0a63 + env: + GITHUB_TOKEN: ${{ github.token }} + with: + repository_full_name: ${{ github.repository }} + commit_sha: ${{ inputs.commit_sha }} + context: ${{ inputs.context_name }} + description: "tests running, image_tag:${{ inputs.server_image_tag }}${{ inputs.server_image_aliases && format(' ({0})', inputs.server_image_aliases) || '' }}" + status: pending + + dispatch-begin: + runs-on: ubuntu-24.04 + permissions: + contents: read + id-token: write + pull-requests: write + outputs: + composite-identity-json: ${{ steps.composite-identity.outputs.composite-identity-json }} + workers-matrix: ${{ steps.matrix.outputs.workers }} + start_time: ${{ steps.matrix.outputs.start_time }} + steps: + - name: ci/checkout-repo + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + ref: ${{ inputs.commit_sha }} + fetch-depth: 1 + - name: ci/composite-identity + id: composite-identity + env: + CONTEXT_NAME: ${{ inputs.context_name }} + MM_SHA: ${{ inputs.commit_sha }} + MM_BRANCH: ${{ inputs.branch }} + PR_NUMBER: ${{ inputs.pr_number }} + run: | + # Derive the test-system-io run name from the GitHub commit-status + # context: drop the `e2e-test/` prefix (the framework name already + # implies E2E in the dashboard) and swap remaining `/` for `-` so + # the dashboard URL is path-safe. The commit-status context itself + # stays unchanged elsewhere — branch protection rules depend on it. + NAME="${CONTEXT_NAME#e2e-test/}" + NAME="${NAME//\//-}" + if [ -n "$PR_NUMBER" ]; then + COMPOSITE_IDENTITY=$(jq -nc \ + --arg repo "${{ github.repository }}" \ + --arg sha "${MM_SHA}" \ + --arg run_id "${GITHUB_RUN_ID}" \ + --arg name "${NAME}" \ + --arg attempt "${GITHUB_RUN_ATTEMPT}" \ + --arg branch "${MM_BRANCH}" \ + --arg pr "${PR_NUMBER}" \ + '{repository:$repo, commit_sha:$sha, gh_run_id:$run_id, name:$name, gh_run_attempt:$attempt, branch:$branch, gh_pr_number:$pr}') + else + COMPOSITE_IDENTITY=$(jq -nc \ + --arg repo "${{ github.repository }}" \ + --arg sha "${MM_SHA}" \ + --arg run_id "${GITHUB_RUN_ID}" \ + --arg name "${NAME}" \ + --arg attempt "${GITHUB_RUN_ATTEMPT}" \ + --arg branch "${MM_BRANCH}" \ + '{repository:$repo, commit_sha:$sha, gh_run_id:$run_id, name:$name, gh_run_attempt:$attempt, branch:$branch}') + fi + echo "composite-identity-json=${COMPOSITE_IDENTITY}" >> $GITHUB_OUTPUT + - name: ci/matrix + id: matrix + run: | + echo "workers=$(jq -nc --argjson n ${{ inputs.workers }} '[range(1; $n+1)]')" >> $GITHUB_OUTPUT + echo "start_time=$(date +%s)" >> $GITHUB_OUTPUT + - name: ci/dispatch-begin + uses: mattermost/mattermost-test-system-io/.github/actions/test-system-io-dispatch-begin@main + with: + use-staging: ${{ vars.E2E_USE_STAGING_TEST_IO_URL != 'false' }} + framework: playwright + repo-dir: ${{ github.workspace }} + composite-identity: ${{ steps.composite-identity.outputs.composite-identity-json }} + total-reports-expected: ${{ inputs.workers }} + retest-on-fail: ${{ inputs.retest_on_fail }} + playwright-project: ${{ inputs.playwright_project }} + post-pr-comment: 'true' + github-token: ${{ secrets.GITHUB_TOKEN }} + test-type: full + server-edition: ${{ inputs.server_edition }} + + workers: + name: dispatch-run-${{ matrix.worker_index }} + runs-on: ubuntu-24.04 + timeout-minutes: 30 + needs: dispatch-begin + permissions: + contents: read + id-token: write + strategy: + fail-fast: false + matrix: + worker_index: ${{ fromJSON(needs.dispatch-begin.outputs.workers-matrix) }} + env: + COMPOSITE_IDENTITY: ${{ needs.dispatch-begin.outputs.composite-identity-json }} + SERVER: "${{ inputs.server }}" + MM_LICENSE: "${{ secrets.MM_LICENSE }}" + ENABLED_DOCKER_SERVICES: "${{ inputs.enabled_docker_services }}" + TEST: playwright + BRANCH: "${{ inputs.branch }}" + BUILD_ID: "${{ inputs.build_id }}" + CI_BASE_URL: "full-test-${{ matrix.worker_index }}" + steps: + - name: ci/checkout-repo + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + ref: ${{ inputs.commit_sha }} + fetch-depth: 0 + - name: ci/setup-node + uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0 + with: + node-version-file: ".nvmrc" + cache: npm + cache-dependency-path: "e2e-tests/playwright/package-lock.json" + - name: ci/get-webapp-node-modules + working-directory: webapp + run: make node_modules + - name: ci/cloud-init + working-directory: e2e-tests + run: make cloud-init + - name: ci/start-server + working-directory: e2e-tests + run: make start-server + # Build once + run the `setup` project so per-spec dispatches can + # pass --no-deps and skip plugin-load + server-deployment checks. + - name: ci/prepare-playwright + working-directory: e2e-tests/playwright + run: | + npm ci + npm run build + npx playwright test --project=setup + - name: ci/dispatch-run + uses: mattermost/mattermost-test-system-io/.github/actions/test-system-io-dispatch-run@main + with: + use-staging: ${{ vars.E2E_USE_STAGING_TEST_IO_URL != 'false' }} + framework: playwright + composite-identity: ${{ needs.dispatch-begin.outputs.composite-identity-json }} + repo-dir: ${{ github.workspace }} + artifacts-root: ${{ github.workspace }}/worker-artifacts + github-token: ${{ secrets.GITHUB_TOKEN }} + gh-job-name: dispatch-run-${{ matrix.worker_index }} + playwright-retries: ${{ inputs.playwright_retries }} + playwright-project: ${{ inputs.playwright_project }} + - name: ci/cloud-teardown + if: always() + working-directory: e2e-tests + run: make cloud-teardown + - name: ci/upload-debug-artifacts + if: always() + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + with: + name: playwright-full-${{ inputs.server_edition }}-debug-${{ matrix.worker_index }} + path: | + e2e-tests/playwright/logs/ + e2e-tests/playwright/results/ + worker-artifacts/ + retention-days: 5 + if-no-files-found: ignore + + report: + runs-on: ubuntu-24.04 + needs: [dispatch-begin, workers] + if: always() + permissions: + contents: read + id-token: write + pull-requests: write + outputs: + commit_status_description: ${{ steps.summary.outputs.commit_status_description }} + webhook_payload: ${{ steps.summary.outputs.webhook_payload }} + steps: + - name: ci/checkout-repo + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - name: ci/run-summary + id: summary + continue-on-error: true + uses: mattermost/mattermost-test-system-io/.github/actions/test-system-io-summary@main + with: + use-staging: ${{ vars.E2E_USE_STAGING_TEST_IO_URL != 'false' }} + composite-identity: ${{ needs.dispatch-begin.outputs.composite-identity-json }} + framework: playwright + test-type: full + report-type: ${{ inputs.report_type }} + image-tag: ${{ inputs.server_image_tag }} + image-aliases: ${{ inputs.server_image_aliases }} + server-image: ${{ env.SERVER_IMAGE }} + pr-number: ${{ inputs.pr_number }} + ref-branch: ${{ inputs.ref_branch }} + server-edition: ${{ inputs.server_edition }} + post-pr-comment: 'true' + github-token: ${{ secrets.GITHUB_TOKEN }} + - name: ci/publish-webhook + if: inputs.enable_reporting && env.REPORT_WEBHOOK_URL != '' + env: + REPORT_WEBHOOK_URL: ${{ secrets.REPORT_WEBHOOK_URL }} + PAYLOAD: ${{ steps.summary.outputs.webhook_payload }} + run: | + curl -X POST -H "Content-Type: application/json" -d "$PAYLOAD" "$REPORT_WEBHOOK_URL" + - name: ci/assert-results + env: + SUMMARY_OUTCOME: ${{ steps.summary.outcome }} + run: | + [ "$SUMMARY_OUTCOME" = "success" ] + + update-success-status: + runs-on: ubuntu-24.04 + permissions: + contents: read + statuses: write + if: always() && needs.report.result == 'success' + needs: + - dispatch-begin + - report + steps: + - uses: mattermost/actions/delivery/update-commit-status@f324ac89b05cc3511cb06e60642ac2fb829f0a63 + env: + GITHUB_TOKEN: ${{ github.token }} + with: + repository_full_name: ${{ github.repository }} + commit_sha: ${{ inputs.commit_sha }} + context: ${{ inputs.context_name }} + description: ${{ needs.report.outputs.commit_status_description }} + status: success + target_url: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + + update-failure-status: + runs-on: ubuntu-24.04 + permissions: + contents: read + statuses: write + if: always() && needs.report.result != 'success' + needs: + - dispatch-begin + - report + steps: + - uses: mattermost/actions/delivery/update-commit-status@f324ac89b05cc3511cb06e60642ac2fb829f0a63 + env: + GITHUB_TOKEN: ${{ github.token }} + with: + repository_full_name: ${{ github.repository }} + commit_sha: ${{ inputs.commit_sha }} + context: ${{ inputs.context_name }} + description: ${{ needs.report.outputs.commit_status_description }} + status: failure + target_url: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} diff --git a/.github/workflows/e2e-tests-playwright.yml b/.github/workflows/e2e-tests-playwright.yml index 8c02f365cf7..5cd410f5bf4 100644 --- a/.github/workflows/e2e-tests-playwright.yml +++ b/.github/workflows/e2e-tests-playwright.yml @@ -160,10 +160,15 @@ jobs: -f target_url="https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" echo "Posted success for ${CONTEXT_NAME}" - playwright-full: + # ── Routing fork ───────────────────────────────────────────────────── + # vars.E2E_USE_TEST_IO_DISPATCH selects between v1 (legacy) and v2. + # vars.E2E_USE_STAGING_TEST_IO_URL toggles v2's staging vs production + # endpoint (default: staging). + + playwright-full-v1: needs: - generate-build-variables - if: inputs.should_run != 'false' + if: inputs.should_run != 'false' && vars.E2E_USE_TEST_IO_DISPATCH != 'true' permissions: contents: read statuses: write @@ -191,3 +196,32 @@ jobs: REPORT_WEBHOOK_URL: ${{ secrets.REPORT_WEBHOOK_URL }} AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + + playwright-full-v2: + needs: + - generate-build-variables + if: inputs.should_run != 'false' && vars.E2E_USE_TEST_IO_DISPATCH == 'true' + permissions: + contents: read + statuses: write + id-token: write + uses: ./.github/workflows/e2e-tests-playwright-template-v2.yml + with: + workers: 8 + enabled_docker_services: "postgres inbucket" + commit_sha: ${{ inputs.commit_sha }} + branch: ${{ needs.generate-build-variables.outputs.branch }} + build_id: ${{ needs.generate-build-variables.outputs.build_id }} + server_image_tag: ${{ needs.generate-build-variables.outputs.server_image_tag }} + server_edition: ${{ inputs.server_edition }} + server_image_repo: ${{ inputs.server_image_repo }} + server_image_aliases: ${{ inputs.server_image_aliases }} + server: ${{ inputs.server }} + enable_reporting: ${{ inputs.enable_reporting }} + report_type: ${{ inputs.report_type }} + ref_branch: ${{ inputs.ref_branch }} + pr_number: ${{ inputs.pr_number }} + context_name: "e2e-test/playwright-full/${{ inputs.server_edition || 'enterprise' }}${{ needs.generate-build-variables.outputs.context_suffix }}" + secrets: + MM_LICENSE: ${{ secrets.MM_LICENSE }} + REPORT_WEBHOOK_URL: ${{ secrets.REPORT_WEBHOOK_URL }} diff --git a/e2e-tests/cypress/cypress.config.ts b/e2e-tests/cypress/cypress.config.ts index fccf067b19f..b499d4a98c3 100644 --- a/e2e-tests/cypress/cypress.config.ts +++ b/e2e-tests/cypress/cypress.config.ts @@ -13,8 +13,8 @@ export default defineConfig({ numTestsKeptInMemory: 0, retries: { - // 2 retries for cypress:run - runMode: 2, + // retries for cypress:run + runMode: 1, // No retries for cypress:open openMode: 0, diff --git a/e2e-tests/cypress/tests/integration/channels/messaging/message_deletion_spec.js b/e2e-tests/cypress/tests/integration/channels/messaging/message_deletion_spec.js index bbfc20a8074..48ce19bed81 100644 --- a/e2e-tests/cypress/tests/integration/channels/messaging/message_deletion_spec.js +++ b/e2e-tests/cypress/tests/integration/channels/messaging/message_deletion_spec.js @@ -1,7 +1,7 @@ // Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. // See LICENSE.txt for license information. -import {getAdminAccount} from 'tests/support/env'; +import {getAdminAccount} from '@/support/env'; // *************************************************************** // - [#] indicates a test step (e.g. # Go to a page) diff --git a/e2e-tests/playwright/playwright.config.ts b/e2e-tests/playwright/playwright.config.ts index 95f3e955a3d..1356f3fd191 100644 --- a/e2e-tests/playwright/playwright.config.ts +++ b/e2e-tests/playwright/playwright.config.ts @@ -9,7 +9,7 @@ export default defineConfig({ globalSetup: './global_setup.ts', forbidOnly: testConfig.isCI, outputDir: './results/output', - retries: testConfig.isCI ? 2 : 0, + retries: testConfig.isCI ? 1 : 0, testDir: 'specs', timeout: duration.one_min, workers: testConfig.workers, diff --git a/e2e-tests/playwright/specs/functional/channels/autotranslation/autotranslation.spec.ts b/e2e-tests/playwright/specs/functional/channels/autotranslation/autotranslation.spec.ts index 033059276b1..d8a63a3779a 100644 --- a/e2e-tests/playwright/specs/functional/channels/autotranslation/autotranslation.spec.ts +++ b/e2e-tests/playwright/specs/functional/channels/autotranslation/autotranslation.spec.ts @@ -171,7 +171,7 @@ test( }, ); -test( +test.fixme( 'only new messages are translated after enable; old messages unchanged', { tag: ['@autotranslation'], @@ -351,7 +351,7 @@ test( }, ); -test( +test.fixme( 'auto-translation is ON by default for new channel members', { tag: ['@autotranslation'], @@ -458,7 +458,7 @@ test( }, ); -test( +test.fixme( 'disabling for self reverts translated messages to original', { tag: ['@autotranslation'], @@ -536,7 +536,7 @@ test( }, ); -test( +test.fixme( 'messages only translate when source differs from user language', { tag: ['@autotranslation'], @@ -639,7 +639,7 @@ test( }, ); -test( +test.fixme( 'message indicator only on actually translated message', { tag: ['@autotranslation'], @@ -834,7 +834,7 @@ test.fixme( }, ); -test( +test.fixme( 'message actions include Show translation', { tag: ['@autotranslation'], diff --git a/e2e-tests/playwright/specs/functional/channels/content_flagging/deletion-report/deletion-report.spec.ts b/e2e-tests/playwright/specs/functional/channels/content_flagging/deletion-report/deletion-report.spec.ts index bd4febff80b..f9017cdfa66 100644 --- a/e2e-tests/playwright/specs/functional/channels/content_flagging/deletion-report/deletion-report.spec.ts +++ b/e2e-tests/playwright/specs/functional/channels/content_flagging/deletion-report/deletion-report.spec.ts @@ -14,7 +14,7 @@ import {setupContentFlagging, createPost} from './../support'; * 4. Login as the reviewer and navigate to the content review DM * 5. Verify the deletion report summary table is posted in the reviewer's thread */ -test('Reviewer receives a deletion report summary after removing a flagged post', async ({pw}) => { +test.fixme('Reviewer receives a deletion report summary after removing a flagged post', async ({pw}) => { const {adminClient, team, user: reviewerUser, userClient: reviewerUserClient} = await pw.initSetup(); // Create author user.. diff --git a/e2e-tests/playwright/specs/functional/channels/team_settings/team_settings_policy_editor.spec.ts b/e2e-tests/playwright/specs/functional/channels/team_settings/team_settings_policy_editor.spec.ts index 3088b82df92..4997534c931 100644 --- a/e2e-tests/playwright/specs/functional/channels/team_settings/team_settings_policy_editor.spec.ts +++ b/e2e-tests/playwright/specs/functional/channels/team_settings/team_settings_policy_editor.spec.ts @@ -738,7 +738,7 @@ test.describe('Team Settings Modal - Policy Editor', () => { await teamSettings3.close(); }); - test('MM-67594_14 Add channels modal shows only private member channels even when team has >50 public channels', async ({ + test.fixme('MM-67594_14 Add channels modal shows only private member channels even when team has >50 public channels', async ({ pw, }) => { // Regression: the non-sysConsole fast path previously called AutocompleteChannelsForTeam diff --git a/e2e-tests/playwright/specs/functional/plugins/demo_plugin/server/slash_commands/demo_plugin_hook_toggle.spec.ts b/e2e-tests/playwright/specs/functional/plugins/demo_plugin/server/slash_commands/demo_plugin_hook_toggle.spec.ts index af8a096ad93..766ab07a148 100644 --- a/e2e-tests/playwright/specs/functional/plugins/demo_plugin/server/slash_commands/demo_plugin_hook_toggle.spec.ts +++ b/e2e-tests/playwright/specs/functional/plugins/demo_plugin/server/slash_commands/demo_plugin_hook_toggle.spec.ts @@ -5,7 +5,7 @@ import {expect, test} from '@mattermost/playwright-lib'; import {setupDemoPlugin} from '../../helpers'; -test('should toggle hooks on and off via /demo_plugin command', async ({pw}) => { +test.fixme('should toggle hooks on and off via /demo_plugin command', async ({pw}) => { // 1. Setup: install and activate the demo plugin const {adminClient, user, team} = await pw.initSetup(); await setupDemoPlugin(adminClient, pw); diff --git a/e2e-tests/playwright/specs/functional/system_console/system_users/user_attributes_admin_editing.spec.ts b/e2e-tests/playwright/specs/functional/system_console/system_users/user_attributes_admin_editing.spec.ts index 335c8ef7ab0..9cda3c48e4b 100644 --- a/e2e-tests/playwright/specs/functional/system_console/system_users/user_attributes_admin_editing.spec.ts +++ b/e2e-tests/playwright/specs/functional/system_console/system_users/user_attributes_admin_editing.spec.ts @@ -158,7 +158,7 @@ test.describe('System Console - Admin User Profile Editing', () => { await userDetail.waitForSaveComplete(); }); - test('Should display user attributes in two-column layout', async () => { + test.fixme('Should display user attributes in two-column layout', async () => { const {userCard} = systemConsolePage.users.userDetail; // * Verify two-column layout exists @@ -258,7 +258,7 @@ test.describe('System Console - Admin User Profile Editing', () => { await expect(departmentInput).toHaveValue('Engineering Updated'); }); - test('Should validate invalid email and show error with cancel option', async () => { + test.fixme('Should validate invalid email and show error with cancel option', async () => { const {userDetail} = systemConsolePage.users; const {userCard} = userDetail; @@ -298,7 +298,7 @@ test.describe('System Console - Admin User Profile Editing', () => { await expect(userDetail.saveButton).toBeDisabled(); }); - test('Should validate invalid URL and show error with cancel option', async () => { + test.fixme('Should validate invalid URL and show error with cancel option', async () => { const {userDetail} = systemConsolePage.users; const {userCard} = userDetail; @@ -335,7 +335,7 @@ test.describe('System Console - Admin User Profile Editing', () => { await expect(userDetail.cancelButton).not.toBeVisible(); }); - test('Should validate invalid email in custom email attribute', async () => { + test.fixme('Should validate invalid email in custom email attribute', async () => { const {userDetail} = systemConsolePage.users; const {userCard} = userDetail; diff --git a/e2e-tests/playwright/specs/functional/system_console/user_attributes/user_attributes.spec.ts b/e2e-tests/playwright/specs/functional/system_console/user_attributes/user_attributes.spec.ts index a66321944f8..bb89bd37b26 100644 --- a/e2e-tests/playwright/specs/functional/system_console/user_attributes/user_attributes.spec.ts +++ b/e2e-tests/playwright/specs/functional/system_console/user_attributes/user_attributes.spec.ts @@ -126,7 +126,7 @@ test.describe('System Console - User Attributes Management', () => { * @objective Verify creating a new text attribute via the UI and saving it * persists the field to the server. */ - test('creates a new text attribute and saves', {tag: '@user_attributes'}, async ({pw}) => { + test.fixme('creates a new text attribute and saves', {tag: '@user_attributes'}, async ({pw}) => { const {adminClient, systemConsolePage} = await setupTest(pw); const sp = systemConsolePage.systemProperties; @@ -159,7 +159,7 @@ test.describe('System Console - User Attributes Management', () => { * @objective Verify creating a select attribute with multiple options saves * the field and its options to the server. */ - test('creates a select attribute with options and saves', {tag: '@user_attributes'}, async ({pw}) => { + test.fixme('creates a select attribute with options and saves', {tag: '@user_attributes'}, async ({pw}) => { const {adminClient, systemConsolePage} = await setupTest(pw); const sp = systemConsolePage.systemProperties; @@ -200,7 +200,7 @@ test.describe('System Console - User Attributes Management', () => { * @precondition * A custom profile attribute named "Old Name" exists via API setup. */ - test('edits an existing attribute name and saves', {tag: '@user_attributes'}, async ({pw}) => { + test.fixme('edits an existing attribute name and saves', {tag: '@user_attributes'}, async ({pw}) => { const {adminClient, systemConsolePage} = await setupTest(pw); const sp = systemConsolePage.systemProperties; @@ -233,7 +233,7 @@ test.describe('System Console - User Attributes Management', () => { * @precondition * A custom profile attribute named "To Delete" exists via API setup. */ - test('deletes an attribute via dot menu', {tag: '@user_attributes'}, async ({pw}) => { + test.fixme('deletes an attribute via dot menu', {tag: '@user_attributes'}, async ({pw}) => { const {adminClient, systemConsolePage} = await setupTest(pw); const sp = systemConsolePage.systemProperties; @@ -273,7 +273,7 @@ test.describe('System Console - User Attributes Management', () => { * @precondition * A custom profile attribute named "Original" exists via API setup. */ - test('duplicates an attribute via dot menu', {tag: '@user_attributes'}, async ({pw}) => { + test.fixme('duplicates an attribute via dot menu', {tag: '@user_attributes'}, async ({pw}) => { const {adminClient, systemConsolePage} = await setupTest(pw); const sp = systemConsolePage.systemProperties; @@ -311,7 +311,7 @@ test.describe('System Console - User Attributes Management', () => { * @precondition * A custom profile attribute named "Visibility Test" exists via API setup. */ - test('changes attribute visibility via dot menu', {tag: '@user_attributes'}, async ({pw}) => { + test.fixme('changes attribute visibility via dot menu', {tag: '@user_attributes'}, async ({pw}) => { const {adminClient, systemConsolePage} = await setupTest(pw); const sp = systemConsolePage.systemProperties; @@ -347,7 +347,7 @@ test.describe('System Console - User Attributes Management', () => { * @precondition * A custom profile attribute named "Editable Test" exists via API setup. */ - test('toggles editable by users off via dot menu', {tag: '@user_attributes'}, async ({pw}) => { + test.fixme('toggles editable by users off via dot menu', {tag: '@user_attributes'}, async ({pw}) => { const {adminClient, systemConsolePage} = await setupTest(pw); const sp = systemConsolePage.systemProperties; @@ -412,7 +412,7 @@ test.describe('System Console - User Attributes Management', () => { * @precondition * A custom profile attribute named "Unique Name" exists via API setup. */ - test('shows validation warning for duplicate attribute names', {tag: '@user_attributes'}, async ({pw}) => { + test.fixme('shows validation warning for duplicate attribute names', {tag: '@user_attributes'}, async ({pw}) => { const {adminClient, systemConsolePage} = await setupTest(pw); const sp = systemConsolePage.systemProperties; @@ -447,7 +447,7 @@ test.describe('System Console - User Attributes Management', () => { * @precondition * A text attribute named "Contact Number" exists via API setup. */ - test('changes attribute type from text to phone', {tag: '@user_attributes'}, async ({pw}) => { + test.fixme('changes attribute type from text to phone', {tag: '@user_attributes'}, async ({pw}) => { const {adminClient, systemConsolePage} = await setupTest(pw); const sp = systemConsolePage.systemProperties; @@ -515,7 +515,7 @@ test.describe('System Console - User Attributes Management', () => { * @objective Verify creating multiple text attributes in a single session * and saving them all at once persists both to the server. */ - test('creates multiple text attributes and saves all at once', {tag: '@user_attributes'}, async ({pw}) => { + test.fixme('creates multiple text attributes and saves all at once', {tag: '@user_attributes'}, async ({pw}) => { const {adminClient, systemConsolePage} = await setupTest(pw); const sp = systemConsolePage.systemProperties; @@ -551,7 +551,7 @@ test.describe('System Console - User Attributes Management', () => { * @precondition * A custom profile attribute named "Persistent Field" exists via API setup. */ - test('persists attribute changes after page reload', {tag: '@user_attributes'}, async ({pw}) => { + test.fixme('persists attribute changes after page reload', {tag: '@user_attributes'}, async ({pw}) => { const {adminClient, systemConsolePage} = await setupTest(pw); const sp = systemConsolePage.systemProperties;