diff --git a/.github/actions/restore-e2e-npm-cache/action.yml b/.github/actions/restore-e2e-npm-cache/action.yml new file mode 100644 index 00000000000..df1b7b70403 --- /dev/null +++ b/.github/actions/restore-e2e-npm-cache/action.yml @@ -0,0 +1,13 @@ +name: "Restore E2E npm registry cache" +description: "Restore ~/.npm for cypress/playwright/api jobs (keyed on all E2E lockfiles)" + +runs: + using: "composite" + steps: + - name: ci/restore-npm-cache + uses: actions/cache/restore@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3 + with: + path: ~/.npm + key: node-cache-${{ runner.os }}-${{ runner.arch }}-npm-e2e-${{ hashFiles('e2e-tests/cypress/package-lock.json', 'e2e-tests/playwright/package-lock.json', 'api/package-lock.json') }} + restore-keys: | + node-cache-${{ runner.os }}-${{ runner.arch }}-npm-e2e- diff --git a/.github/actions/webapp-setup/action.yml b/.github/actions/webapp-setup/action.yml index 3d32e3172af..92d7d701931 100644 --- a/.github/actions/webapp-setup/action.yml +++ b/.github/actions/webapp-setup/action.yml @@ -27,6 +27,9 @@ runs: webapp/platform/shared/node_modules webapp/platform/types/node_modules key: node-modules-${{ runner.os }}-${{ hashFiles('webapp/package-lock.json') }} + # fall back to most recent cache for this OS if package-lock.json changed + restore-keys: | + node-modules-${{ runner.os }}- - name: ci/restore-node-modules if: inputs.read-only == 'true' uses: actions/cache/restore@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3 @@ -40,6 +43,9 @@ runs: webapp/platform/shared/node_modules webapp/platform/types/node_modules key: node-modules-${{ runner.os }}-${{ hashFiles('webapp/package-lock.json') }} + # fall back to most recent cache for this OS if package-lock.json changed + restore-keys: | + node-modules-${{ runner.os }}- - name: ci/resolve-cache-hit id: cache-hit shell: bash diff --git a/.github/workflows/api.yml b/.github/workflows/api.yml index ae2205bc73d..52d9d509394 100644 --- a/.github/workflows/api.yml +++ b/.github/workflows/api.yml @@ -22,11 +22,12 @@ jobs: with: persist-credentials: false + - name: ci/restore-npm-cache + uses: ./.github/actions/restore-e2e-npm-cache + - uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0 with: node-version-file: .nvmrc - cache: "npm" - cache-dependency-path: api/package-lock.json - name: Run build run: make build diff --git a/.github/workflows/e2e-ci-cache-warm.yml b/.github/workflows/e2e-ci-cache-warm.yml new file mode 100644 index 00000000000..bd77695f867 --- /dev/null +++ b/.github/workflows/e2e-ci-cache-warm.yml @@ -0,0 +1,78 @@ +# Warms the ~/.npm registry cache used by the e2e suites (cypress, playwright, +# api). Runs once per day on master; all e2e/api CI jobs restore this cache +# read-only via .github/actions/restore-e2e-npm-cache and never write back, so +# this job is its sole writer. +# +# The cache is keyed on the OS/arch and a hash of all e2e lockfiles. Keep the key +# below in sync with .github/actions/restore-e2e-npm-cache, which is the canonical +# definition of the bucket's lockfile set. PRs that change a lockfile miss the +# exact key and fall back to the most recent warm entry via restore-keys. +# +# The webapp node_modules cache is warmed separately by webapp-ci-cache-warm.yml. +name: E2E CI Cache Warm + +on: + schedule: + - cron: "0 1 * * *" # Daily 1am UTC — 1 h before e2e CI typically runs + workflow_dispatch: + +# Prevent an overlapping manual dispatch and scheduled run from both missing the +# lookup and then racing to reserve the same cache key. +concurrency: + group: e2e-ci-cache-warm + cancel-in-progress: false + +permissions: + contents: read + actions: write # required to save the Actions cache + +jobs: + warm: + name: Warm E2E npm Registry Cache + runs-on: ubuntu-24.04 + # Bound a hung run so it can't hold the concurrency group for the default + # 6-hour timeout and block subsequent warm runs. + timeout-minutes: 60 + steps: + - name: Checkout mattermost project + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + + - name: Check if npm registry cache already warm + id: e2e-npm-check + uses: actions/cache/restore@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3 + with: + path: ~/.npm + key: node-cache-${{ runner.os }}-${{ runner.arch }}-npm-e2e-${{ hashFiles('e2e-tests/cypress/package-lock.json', 'e2e-tests/playwright/package-lock.json', 'api/package-lock.json') }} + lookup-only: true + + - name: ci/setup-node + if: steps.e2e-npm-check.outputs.cache-hit != 'true' + uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0 + with: + node-version-file: ".nvmrc" + + # ~/.npm is content-addressable and grows additively across installs, so we + # populate it by running each suite's install, then save the single bucket. + - name: Install playwright npm packages + if: steps.e2e-npm-check.outputs.cache-hit != 'true' + working-directory: e2e-tests/playwright + run: npm ci + + - name: Install cypress npm packages + if: steps.e2e-npm-check.outputs.cache-hit != 'true' + working-directory: e2e-tests/cypress + run: npm ci + + - name: Install api npm packages + if: steps.e2e-npm-check.outputs.cache-hit != 'true' + working-directory: api + run: npm ci + + - name: Save npm registry cache + if: steps.e2e-npm-check.outputs.cache-hit != 'true' + uses: actions/cache/save@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3 + with: + path: ~/.npm + key: node-cache-${{ runner.os }}-${{ runner.arch }}-npm-e2e-${{ hashFiles('e2e-tests/cypress/package-lock.json', 'e2e-tests/playwright/package-lock.json', 'api/package-lock.json') }} diff --git a/.github/workflows/e2e-tests-check.yml b/.github/workflows/e2e-tests-check.yml index a10f0b0f4f6..51f6eae1821 100644 --- a/.github/workflows/e2e-tests-check.yml +++ b/.github/workflows/e2e-tests-check.yml @@ -24,25 +24,25 @@ jobs: persist-credentials: false fetch-depth: 0 + - name: ci/restore-npm-cache + uses: ./.github/actions/restore-e2e-npm-cache - name: ci/setup-node uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0 with: node-version-file: ".nvmrc" - cache: npm - cache-dependency-path: | - e2e-tests/cypress/package-lock.json - e2e-tests/playwright/package-lock.json - webapp/package-lock.json - name: ci/npm-cache-verify # Heal any partial/dangling entries left in the restored ~/.npm cache # before running `npm ci`. Avoids the intermittent EEXIST/ENOENT # failures in npm's cacache writer. run: npm cache verify - # Set up web app subpackages and eslint plugin - - name: ci/get-webapp-node-modules - working-directory: webapp - run: make node_modules + # Set up web app subpackages and eslint plugin, restoring the warmed + # node_modules cache read-only and falling back to `make node_modules` on a + # miss. + - name: ci/webapp-setup + uses: ./.github/actions/webapp-setup + with: + read-only: "true" # Cypress check - name: ci/cypress/npm-install diff --git a/.github/workflows/e2e-tests-ci-template.yml b/.github/workflows/e2e-tests-ci-template.yml index 10de9cf0af1..7de6ca41097 100644 --- a/.github/workflows/e2e-tests-ci-template.yml +++ b/.github/workflows/e2e-tests-ci-template.yml @@ -126,7 +126,6 @@ jobs: shell: bash outputs: workers: "${{ steps.generate.outputs.workers }}" - node-cache-dependency-path: "${{ steps.generate.outputs.node-cache-dependency-path }}" steps: - name: ci/checkout-repo uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 @@ -138,11 +137,9 @@ jobs: id: generate env: WORKERS: ${{ inputs.workers_number }} - TEST: ${{ inputs.TEST }} run: | [ "$WORKERS" -gt "0" ] # Assert that the workers number is an integer greater than 0 echo "workers="$(jq --slurp --compact-output '[range('"$WORKERS"')] | map(tostring)' /dev/null) >> $GITHUB_OUTPUT - echo "node-cache-dependency-path=e2e-tests/${TEST}/package-lock.json" >> $GITHUB_OUTPUT generate-test-cycle: permissions: @@ -163,13 +160,13 @@ jobs: persist-credentials: false ref: ${{ inputs.commit_sha }} fetch-depth: 0 + - name: ci/restore-npm-cache + uses: ./.github/actions/restore-e2e-npm-cache - name: ci/setup-node uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0 id: setup_node with: node-version-file: ".nvmrc" - cache: npm - cache-dependency-path: "e2e-tests/cypress/package-lock.json" # NB: the generate-cycle script is cypress-specific operation for now - name: ci/e2e-test-gencycle id: e2e-test-gencycle env: @@ -265,13 +262,13 @@ jobs: mkdir -p ~/.docker/cli-plugins ln -sfn /usr/local/opt/docker-compose/bin/docker-compose ~/.docker/cli-plugins/docker-compose sudo ln -sf $HOME/.colima/default/docker.sock /var/run/docker.sock + - name: ci/restore-npm-cache + uses: ./.github/actions/restore-e2e-npm-cache - name: ci/setup-node uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0 id: setup_node with: node-version-file: ".nvmrc" - cache: npm - cache-dependency-path: ${{ needs.generate-build-variables.outputs.node-cache-dependency-path }} - name: ci/e2e-test run: | make cloud-init @@ -379,14 +376,15 @@ jobs: path: | e2e-tests/${{ inputs.TEST }}/logs/ e2e-tests/${{ inputs.TEST }}/results/ + - name: ci/restore-npm-cache + if: "${{ inputs.enable_reporting }}" + uses: ./.github/actions/restore-e2e-npm-cache - name: ci/setup-node if: "${{ inputs.enable_reporting }}" uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0 id: setup_node with: node-version-file: ".nvmrc" - cache: npm - cache-dependency-path: ${{ needs.generate-build-variables.outputs.node-cache-dependency-path }} - name: ci/publish-report if: "${{ inputs.enable_reporting }}" env: diff --git a/.github/workflows/e2e-tests-cypress-template.yml b/.github/workflows/e2e-tests-cypress-template.yml index 77d29c0f02e..3dfd88d5f45 100644 --- a/.github/workflows/e2e-tests-cypress-template.yml +++ b/.github/workflows/e2e-tests-cypress-template.yml @@ -151,12 +151,12 @@ jobs: persist-credentials: false ref: ${{ inputs.commit_sha }} fetch-depth: 0 + - name: ci/restore-npm-cache + uses: ./.github/actions/restore-e2e-npm-cache - name: ci/setup-node uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0 with: node-version-file: ".nvmrc" - cache: npm - cache-dependency-path: "e2e-tests/cypress/package-lock.json" - name: ci/generate-test-cycle id: generate-cycle @@ -216,12 +216,12 @@ jobs: persist-credentials: false ref: ${{ inputs.commit_sha }} fetch-depth: 0 + - name: ci/restore-npm-cache + uses: ./.github/actions/restore-e2e-npm-cache - name: ci/setup-node uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0 with: node-version-file: ".nvmrc" - cache: npm - cache-dependency-path: "e2e-tests/cypress/package-lock.json" - name: ci/npm-cache-verify # Heal any partial/dangling entries left in the restored ~/.npm cache # before running `npm ci`. Avoids the intermittent EEXIST/ENOENT @@ -321,12 +321,12 @@ jobs: persist-credentials: false ref: ${{ inputs.commit_sha }} fetch-depth: 0 + - name: ci/restore-npm-cache + uses: ./.github/actions/restore-e2e-npm-cache - name: ci/setup-node uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0 with: node-version-file: ".nvmrc" - cache: npm - cache-dependency-path: "e2e-tests/cypress/package-lock.json" - name: ci/run-failed-specs env: SPEC_FILES: ${{ needs.calculate-results.outputs.failed_specs }} @@ -370,12 +370,12 @@ jobs: uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false + - name: ci/restore-npm-cache + uses: ./.github/actions/restore-e2e-npm-cache - name: ci/setup-node uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0 with: node-version-file: ".nvmrc" - cache: npm - cache-dependency-path: "e2e-tests/cypress/package-lock.json" # PATH A: run-failed-tests was skipped (no failures to retest) - name: ci/download-results-path-a diff --git a/.github/workflows/e2e-tests-playwright-template.yml b/.github/workflows/e2e-tests-playwright-template.yml index 3eb8dd16540..b8c1ae7c17d 100644 --- a/.github/workflows/e2e-tests-playwright-template.yml +++ b/.github/workflows/e2e-tests-playwright-template.yml @@ -178,14 +178,12 @@ jobs: persist-credentials: false ref: ${{ inputs.commit_sha }} fetch-depth: 0 + - name: ci/restore-npm-cache + uses: ./.github/actions/restore-e2e-npm-cache - name: ci/setup-node uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0 with: node-version-file: ".nvmrc" - cache: npm - cache-dependency-path: | - e2e-tests/playwright/package-lock.json - webapp/package-lock.json - name: ci/npm-cache-verify # Heal any partial/dangling entries left in the restored ~/.npm cache # before running `npm ci`. Avoids the intermittent EEXIST/ENOENT @@ -295,12 +293,12 @@ jobs: uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false + - name: ci/restore-npm-cache + uses: ./.github/actions/restore-e2e-npm-cache - name: ci/setup-node uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0 with: node-version-file: ".nvmrc" - cache: npm - cache-dependency-path: "e2e-tests/playwright/package-lock.json" - name: ci/download-shard-results uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0 with: @@ -362,12 +360,12 @@ jobs: uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false + - name: ci/restore-npm-cache + uses: ./.github/actions/restore-e2e-npm-cache - name: ci/setup-node uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0 with: node-version-file: ".nvmrc" - cache: npm - cache-dependency-path: "e2e-tests/playwright/package-lock.json" # Download merged results (uploaded by calculate-results). These blob # reports already include the inline per-shard retry results, so no diff --git a/.github/workflows/webapp-ci-cache-warm.yml b/.github/workflows/webapp-ci-cache-warm.yml new file mode 100644 index 00000000000..d1525e959c8 --- /dev/null +++ b/.github/workflows/webapp-ci-cache-warm.yml @@ -0,0 +1,77 @@ +# Warms the webapp node_modules cache used by webapp CI and the E2E v2 templates +# (via .github/actions/webapp-setup). Runs once per day on master so the cache is +# fresh before CI runs; webapp CI restores it read-only, though the E2E v2 +# templates also populate it within their own runs. +# +# The cache is keyed on the OS and a hash of webapp/package-lock.json. PRs that +# change the lockfile miss the exact key and fall back to the most recent warm +# entry via restore-keys, then reconcile the delta with `make node_modules`. +# +# The E2E ~/.npm registry cache is warmed separately by e2e-ci-cache-warm.yml. +name: Webapp CI Cache Warm + +on: + schedule: + - cron: "0 1 * * *" # Daily 1am UTC — 1 h before webapp CI typically runs + workflow_dispatch: + +# Prevent an overlapping manual dispatch and scheduled run from both missing the +# lookup and then racing to reserve the same cache key. +concurrency: + group: webapp-ci-cache-warm + cancel-in-progress: false + +permissions: + contents: read + actions: write # required to save the Actions cache + +jobs: + warm: + name: Warm Webapp node_modules Cache + runs-on: ubuntu-24.04 + # Bound a hung run so it can't hold the concurrency group for the default + # 6-hour timeout and block subsequent warm runs. + timeout-minutes: 60 + steps: + - name: Checkout mattermost project + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + + - name: Check if node_modules cache already warm + id: node-modules-check + uses: actions/cache/restore@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3 + with: + path: | + webapp/node_modules + webapp/channels/node_modules + webapp/platform/client/node_modules + webapp/platform/components/node_modules + webapp/platform/shared/node_modules + webapp/platform/types/node_modules + key: node-modules-${{ runner.os }}-${{ hashFiles('webapp/package-lock.json') }} + lookup-only: true + + - name: ci/setup-node + if: steps.node-modules-check.outputs.cache-hit != 'true' + uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0 + with: + node-version-file: ".nvmrc" + + - name: Install node modules + if: steps.node-modules-check.outputs.cache-hit != 'true' + working-directory: webapp + run: make node_modules + + - name: Save node_modules cache + if: steps.node-modules-check.outputs.cache-hit != 'true' + uses: actions/cache/save@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3 + with: + path: | + webapp/node_modules + webapp/channels/node_modules + webapp/platform/client/node_modules + webapp/platform/components/node_modules + webapp/platform/shared/node_modules + webapp/platform/types/node_modules + key: node-modules-${{ runner.os }}-${{ hashFiles('webapp/package-lock.json') }} diff --git a/.github/workflows/webapp-ci.yml b/.github/workflows/webapp-ci.yml index 5f17495cbf5..dd456e1ac1e 100644 --- a/.github/workflows/webapp-ci.yml +++ b/.github/workflows/webapp-ci.yml @@ -31,6 +31,8 @@ jobs: persist-credentials: false - name: ci/setup uses: ./.github/actions/webapp-setup + with: + read-only: "true" - name: ci/lint run: | npm run check @@ -50,6 +52,8 @@ jobs: persist-credentials: false - name: ci/setup uses: ./.github/actions/webapp-setup + with: + read-only: "true" - name: ci/i18n-extract working-directory: webapp/channels run: | @@ -71,6 +75,8 @@ jobs: persist-credentials: false - name: ci/setup uses: ./.github/actions/webapp-setup + with: + read-only: "true" - name: ci/check-external-links run: | set -o pipefail @@ -91,6 +97,8 @@ jobs: persist-credentials: false - name: ci/setup uses: ./.github/actions/webapp-setup + with: + read-only: "true" - name: ci/lint run: | npm run check-types @@ -115,6 +123,8 @@ jobs: persist-credentials: false - name: ci/setup uses: ./.github/actions/webapp-setup + with: + read-only: "true" - name: ci/test env: NODE_OPTIONS: --max_old_space_size=5120 @@ -150,6 +160,8 @@ jobs: persist-credentials: false - name: ci/setup uses: ./.github/actions/webapp-setup + with: + read-only: "true" - name: ci/test env: NODE_OPTIONS: --max_old_space_size=5120 @@ -186,6 +198,8 @@ jobs: persist-credentials: false - name: ci/setup uses: ./.github/actions/webapp-setup + with: + read-only: "true" - name: ci/test env: MATRIX_SHARD: ${{ matrix.shard }} @@ -216,6 +230,8 @@ jobs: persist-credentials: false - name: ci/setup uses: ./.github/actions/webapp-setup + with: + read-only: "true" - name: ci/download-coverage-artifacts uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0 with: @@ -276,6 +292,8 @@ jobs: persist-credentials: false - name: ci/setup uses: ./.github/actions/webapp-setup + with: + read-only: "true" - name: ci/build run: | npm run build