diff --git a/.github/workflows/scenario.yml b/.github/workflows/scenario.yml deleted file mode 100644 index b6999c6e4d..0000000000 --- a/.github/workflows/scenario.yml +++ /dev/null @@ -1,140 +0,0 @@ -name: Scenario Tests - -on: - push: - branches: - - main - pull_request: - types: [opened, reopened, synchronize, ready_for_review] - workflow_dispatch: - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - -jobs: - matrix_prep: - runs-on: ubuntu-latest - permissions: - id-token: write - contents: read - outputs: - matrix: ${{ steps.set-matrix.outputs.matrix }} - steps: - - id: set-matrix - run: | - echo 'matrix=[{"runner":"ubuntu"},{"runner":"windows"},{"runner":"macos"}]' >> $GITHUB_OUTPUT - - validate-scenario: - runs-on: ubuntu-latest - permissions: - contents: read - steps: - - uses: actions/checkout@v4 - - name: Validate PR-specific scenario metadata - run: | - if [ "${{ github.event_name }}" != "pull_request" ]; then - echo "Not a pull_request event; skipping scenario validation." - exit 0 - fi - bash scripts/validate-scenario.sh "${{ github.event.number }}" - - scenarios: - needs: [matrix_prep, validate-scenario] - strategy: - fail-fast: false - matrix: - include: ${{ fromJson(needs.matrix_prep.outputs.matrix) }} - runs-on: ${{ matrix.runner }}-latest - timeout-minutes: 20 - permissions: - id-token: write - contents: read - steps: - - uses: actions/checkout@v4 - - - name: Setup Node.js environment - uses: actions/setup-node@v4 - with: - node-version: 22 - - # Cache root dependencies - only reuse if package-lock.json exactly matches - - name: Cache root dependencies - uses: actions/cache@v4 - id: root-cache - with: - path: node_modules - key: ${{ runner.os }}-npm-${{ hashFiles('package-lock.json') }} - - # Cache webview-ui dependencies - only reuse if package-lock.json exactly matches - - name: Cache webview-ui dependencies - uses: actions/cache@v4 - id: webview-cache - with: - path: webview-ui/node_modules - key: ${{ runner.os }}-npm-webview-${{ hashFiles('webview-ui/package-lock.json') }} - - # Cache VS Code installation - - name: Cache VS Code - uses: actions/cache@v4 - id: vscode-cache - with: - path: .vscode-test - key: vscode-${{ runner.os }}-stable-${{ hashFiles('.vscode-test.mjs', 'package.json') }} - restore-keys: | - vscode-${{ runner.os }}-stable- - - # Cache Playwright browsers - - name: Cache Playwright browsers - uses: actions/cache@v4 - id: playwright-cache - with: - path: | - ~/.cache/ms-playwright - ~/Library/Caches/ms-playwright - ~/AppData/Local/ms-playwright - key: playwright-browsers-${{ runner.os }}-${{ hashFiles('package-lock.json') }} - restore-keys: | - playwright-browsers-${{ runner.os }}- - - - name: Install root dependencies - if: steps.root-cache.outputs.cache-hit != 'true' - run: npm ci - - - name: Install webview-ui dependencies - if: steps.webview-cache.outputs.cache-hit != 'true' - run: cd webview-ui && npm ci - - - name: Install xvfb on Linux - if: matrix.runner == 'ubuntu' - run: sudo apt-get update && sudo apt-get install -y xvfb - - # Run optimized Scenario tests (reuses build steps like e2e) - - name: Run Scenario tests - Linux - if: matrix.runner == 'ubuntu' - run: xvfb-run -a npm run test:scenarios:optimal - - - name: Run Scenario tests - Non-Linux - if: matrix.runner != 'ubuntu' - run: npm run test:scenarios:optimal - - - uses: actions/upload-artifact@v4 - if: ${{ failure() }} - with: - name: playwright-recordings-${{ matrix.runner }} - path: | - test-results/playwright/ - - scenario-summary: - needs: scenarios - permissions: {} - runs-on: ubuntu-latest - if: always() - steps: - - name: Check all scenarios passed - run: | - if [ "${{ needs.scenarios.result }}" != "success" ]; then - echo "Some scenario tests failed" - exit 1 - fi - echo "All scenario tests passed successfully" diff --git a/package.json b/package.json index a54a8fab83..ee0221faf2 100644 --- a/package.json +++ b/package.json @@ -365,8 +365,6 @@ "e2e": "playwright test -c playwright.config.ts", "test:e2e": "playwright install && vsce package --no-dependencies --allow-package-secrets sendgrid --out dist/e2e.vsix && node src/test/e2e/utils/build.mjs && playwright test", "test:e2e:optimal": "vsce package --no-dependencies --allow-package-secrets sendgrid --out dist/e2e.vsix && node src/test/e2e/utils/build.mjs && playwright test", - "test:scenarios": "playwright install && vsce package --no-dependencies --allow-package-secrets sendgrid --out dist/e2e.vsix && node src/test/e2e/utils/build.mjs && playwright test -c playwright.scenarios.config.ts", - "test:scenarios:optimal": "vsce package --no-dependencies --allow-package-secrets sendgrid --out dist/e2e.vsix && node src/test/e2e/utils/build.mjs && playwright test -c playwright.scenarios.config.ts", "install:all": "npm install && cd webview-ui && npm install", "dev:webview": "cd webview-ui && npm run dev", "build:webview": "cd webview-ui && npm run build", diff --git a/playwright.scenarios.config.ts b/playwright.scenarios.config.ts deleted file mode 100644 index 1376b74bee..0000000000 --- a/playwright.scenarios.config.ts +++ /dev/null @@ -1,35 +0,0 @@ -import { defineConfig } from "@playwright/test" - -const isCI = !!process?.env?.CI -const isWindow = process?.platform?.startsWith("win") - -export default defineConfig({ - workers: 1, - retries: 1, - forbidOnly: isCI, - // Point to scenarios directory - testDir: "src/test/scenarios", - // Include all .ts in scenarios (we add a local global.setup.ts here) - testMatch: /.*\.ts/, - timeout: isCI || isWindow ? 40000 : 20000, - expect: { - timeout: isCI || isWindow ? 5000 : 2000, - }, - fullyParallel: true, - reporter: isCI ? [["github"], ["list"]] : [["list"]], - use: { - video: "retain-on-failure", - }, - projects: [ - { - name: "setup test environment", - // Reuse E2E setup file to avoid duplication - testDir: "src/test/e2e/utils", - testMatch: /global\.setup\.ts/, - }, - { - name: "scenario tests", - dependencies: ["setup test environment"], - }, - ], -}) diff --git a/scripts/validate-scenario.sh b/scripts/validate-scenario.sh deleted file mode 100644 index dd1d42d057..0000000000 --- a/scripts/validate-scenario.sh +++ /dev/null @@ -1,151 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -# Validate a PR-specific scenario exists in src/test/scenarios -# Rules: -# - example.ts is exempt -# - Exactly one scenario must match the current PR number -# - All non-exempt scenario files must include a valid "GitHub PR" metadata line -# -# Usage: ./scripts/validate-scenario.sh <PR_NUMBER> - -PR_NUMBER="${1:-}" - -SCENARIOS_DIR="src/test/scenarios" -EXEMPT_BASENAMES=("example.ts") - -red() { printf "\033[31m%s\033[0m\n" "$*"; } -green() { printf "\033[32m%s\033[0m\n" "$*"; } -yellow(){ printf "\033[33m%s\033[0m\n" "$*"; } - -is_exempt() { - local base="$1" - for ex in "${EXEMPT_BASENAMES[@]}"; do - if [[ "$base" == "$ex" ]]; then - return 0 - fi - done - return 1 -} - -usage() { - echo "Usage: $0 <PR_NUMBER>" >&2 - exit 2 -} - -if [[ -z "$PR_NUMBER" ]]; then - red "Error: PR number is required." - usage -fi - -if ! [[ "$PR_NUMBER" =~ ^[0-9]+$ ]]; then - red "Error: PR number must be numeric. Got: '$PR_NUMBER'" - exit 2 -fi - -if [[ ! -d "$SCENARIOS_DIR" ]]; then - red "Error: Scenarios directory not found: $SCENARIOS_DIR" - exit 1 -fi - -declare -a ALL_FILES=() -declare -a MISSING_METADATA=() -declare -a MALFORMED_METADATA=() -declare -a FOUND_FILES=() -declare -a FOUND_PRS=() - -# Find top-level .ts files in scenarios dir -while IFS= read -r -d '' file; do - ALL_FILES+=("$file") -done < <(find "$SCENARIOS_DIR" -maxdepth 1 -type f -name "*.ts" -print0 | sort -z) - -if [[ ${#ALL_FILES[@]} -eq 0 ]]; then - red "Error: No scenario files found in $SCENARIOS_DIR" - exit 1 -fi - -for file in "${ALL_FILES[@]}"; do - base="$(basename "$file")" - if is_exempt "$base"; then - continue - fi - - # Extract metadata line, pattern: // GitHub PR - 123 - meta_line="$(grep -E -m1 '^[[:space:]]*//[[:space:]]*GitHub[[:space:]]+PR[[:space:]]*-[[:space:]]*[0-9]+[[:space:]]*$' "$file" || true)" - - if [[ -z "$meta_line" ]]; then - MISSING_METADATA+=("$file") - continue - fi - - pr_in_file="$(sed -E 's@.*GitHub[[:space:]]+PR[[:space:]]*-[[:space:]]*([0-9]+).*@\1@' <<< "$meta_line" | tr -d '[:space:]')" - - if ! [[ "$pr_in_file" =~ ^[0-9]+$ ]]; then - MALFORMED_METADATA+=("$file") - continue - fi - - FOUND_FILES+=("$file") - FOUND_PRS+=("$pr_in_file") -done - -# Fail if any non-exempt file is missing metadata -if [[ ${#MISSING_METADATA[@]} -gt 0 ]]; then - red "Error: The following scenario files are missing the required metadata line '// GitHub PR - <number>':" - for f in "${MISSING_METADATA[@]}"; do - echo " - $f" - done - echo - echo "Please add a metadata line like:" - echo " // GitHub PR - $PR_NUMBER" - exit 1 -fi - -# Fail if any metadata is malformed -if [[ ${#MALFORMED_METADATA[@]} -gt 0 ]]; then - red "Error: The following scenario files have a malformed 'GitHub PR' metadata line:" - for f in "${MALFORMED_METADATA[@]}"; do - echo " - $f" - done - echo - echo "Expected format:" - echo " // GitHub PR - $PR_NUMBER" - exit 1 -fi - -# Count matches for this PR -matches=0 -declare -a MATCHED_FILES=() -for i in "${!FOUND_FILES[@]}"; do - if [[ "${FOUND_PRS[$i]}" == "$PR_NUMBER" ]]; then - matches=$((matches + 1)) - MATCHED_FILES+=("${FOUND_FILES[$i]}") - fi -done - -if [[ $matches -eq 0 ]]; then - red "Error: No scenario file found for PR #$PR_NUMBER." - echo "Scenarios with declared PRs:" - if [[ ${#FOUND_FILES[@]} -eq 0 ]]; then - echo " (none)" - else - for i in "${!FOUND_FILES[@]}"; do - echo " - ${FOUND_FILES[$i]} (PR: ${FOUND_PRS[$i]})" - done - fi - echo - echo "Please add a scenario in $SCENARIOS_DIR with a metadata line:" - echo " // GitHub PR - $PR_NUMBER" - exit 1 -fi - -if [[ $matches -gt 1 ]]; then - red "Error: Multiple scenario files found for PR #$PR_NUMBER (exactly one required):" - for f in "${MATCHED_FILES[@]}"; do - echo " - $f" - done - exit 1 -fi - -green "Success: Exactly one scenario found for PR #$PR_NUMBER:" -echo " - ${MATCHED_FILES[0]}" diff --git a/src/test/scenarios/README.md b/src/test/scenarios/README.md deleted file mode 100644 index 49649dc334..0000000000 --- a/src/test/scenarios/README.md +++ /dev/null @@ -1,147 +0,0 @@ -# Scenario Tests - -This directory contains PR-specific scenario tests that run on top of the existing Playwright E2E framework. Each PR must provide exactly one scenario test file with a PR metadata comment for validation and CI gating. - -## Overview - -- Framework: Playwright, reusing the same E2E fixtures/utilities at `src/test/e2e/utils/` -- Config: Scenario runs are configured via `playwright.scenarios.config.ts` -- CI Workflow: `.github/workflows/scenario.yml` -- Validation: A script enforces “exactly one scenario per PR” with a metadata comment - -## Architecture - -Scenario tests reuse the E2E fixture stack: -- `e2e` fixture from `src/test/e2e/utils/helpers.ts` bootstraps VS Code, installs the extension VSIX, opens the Cline sidebar, and provides utilities (`helper`, `sidebar`, etc.). -- Scenario tests live here in `src/test/scenarios`, using the same Playwright helpers and patterns as E2E but focused on validating PR-specific behavior. -- Scenario tests are intended to be short, focused validations of the change introduced in the PR. - -Key pieces: -- `playwright.scenarios.config.ts` points Playwright to this directory and reuses the shared E2E global setup to keep behavior identical. -- `scripts/validate-scenario.sh` checks that a PR includes exactly one scenario test with a metadata line identifying the PR number. -- `.github/workflows/scenario.yml` runs validation, then runs the Playwright scenario matrix on Ubuntu/Windows/macOS, then aggregates results in a single summary job. - -## File Layout - -- `example.ts` — A template/example scenario (exempt from validation) -- `*.ts` — Your PR-specific scenario tests (top-level only; not recursive) - -Only top-level `.ts` files in this directory are scanned by the validator. Subdirectories are not scanned. - -## Authoring a Scenario - -Add a new file in this directory for your PR, and include the PR metadata comment. Exactly one scenario per PR is required. - -Template: - -```ts -import { expect } from "@playwright/test" -import { e2e } from "../e2e/utils/helpers" - -// Title – Short, descriptive name. -// Description – Purpose of the scenario and any relevant background. -// Preconditions – State the environment, data, or setup required. -// Steps – Numbered, detailed instructions for execution. -// Expected Results – The specific outcome that constitutes a pass. -// Priority – High/Medium/Low, depending on risk. -// GitHub PR - 123 <-- REQUIRED: replace 123 with your PR number - -e2e("Scenario - PR 123 - brief description", async ({ helper, sidebar }) => { - await helper.signin(sidebar) - // ... your steps ... - await expect(sidebar.getByTestId("chat-input")).toBeVisible() -}) -``` - -Notes: -- `example.ts` is exempt and will be ignored by the validator. -- The PR metadata comment must match the pattern: `// GitHub PR - `. -- Exactly one scenario with the current PR number must exist. - -## Running Locally - -### Quick run (recommended) -Runs packaging, VS Code/Chromium setup, then the scenario tests: - -```bash -npm run test:scenarios:optimal -``` - -This will: -1) Package the extension VSIX used for testing -2) Ensure Playwright Chromium and VS Code test binary are installed -3) Execute Playwright using `playwright.scenarios.config.ts` - -### Validate your scenario metadata locally -The validator enforces “one scenario per PR” by scanning the metadata line: - -```bash -# Fails if no scenario exists for PR 123 (or if multiple exist, or metadata is malformed) -bash scripts/validate-scenario.sh 123 -``` - -Tip: To simulate success quickly: - -```bash -printf '%s\n%s\n' '// GitHub PR - 123' 'export {}' > src/test/scenarios/pr-123.ts -bash scripts/validate-scenario.sh 123 -rm src/test/scenarios/pr-123.ts -``` - -### Using act to run the workflow locally (Linux-only) -`act` cannot run macOS/Windows runners. Map Ubuntu runner to a Docker image and run: - -```bash -# Validate then run the Ubuntu scenario job -act -W .github/workflows/scenario.yml -P ubuntu-latest=catthehacker/ubuntu:act-latest -``` - -Note: macOS/Windows jobs will be skipped under `act`. The GitHub-hosted Actions will run all three OS jobs. - -## CI / GitHub Actions Setup - -Workflow file: `.github/workflows/scenario.yml` - -Jobs: -1) `matrix_prep` — Builds the 3-OS matrix -2) `validate-scenario` — Ensures PR has exactly one scenario with `// GitHub PR - ` (skips validation for non-PR events) -3) `scenarios` — Runs Playwright across Ubuntu/Windows/macOS -4) `scenario-summary` — Single summary status check that fails if any OS job fails - -### Enforcing “must run before merging” -Use Branch Protection Rules: -1) Repo → Settings → Branches → Add/Edit rule for `main` -2) Enable “Require status checks to pass before merging” -3) Select `scenario-summary` as a required check -4) Optionally enable “Require branches to be up to date before merging” - -This blocks merges until: -- The validator passes (exactly one scenario per PR with correct metadata) -- All three OS scenario runs pass - -## Troubleshooting - -- “No scenario file found for PR #” - - Ensure exactly one `.ts` file in this directory (excluding `example.ts`) includes the line: - `// GitHub PR - ` - - Ensure the file is top-level (not in a subdirectory). - -- “Malformed ‘GitHub PR’ metadata line” - - The line must be a single-line comment containing a numeric PR: - `// GitHub PR - 123` - -- “Multiple scenario files found for PR #” - - Reduce to exactly one scenario file for the PR. - -- `act` skips macOS/Windows - - This is expected. On GitHub Actions, those OS jobs will run. - -- Slow first runs locally/CI - - The E2E/Scenario flows download VS Code and Playwright Chromium. Caches in CI should speed up subsequent runs. - -## See Also - -- E2E fixtures and helpers: `src/test/e2e/utils/` -- Scenario Playwright config: `playwright.scenarios.config.ts` -- Scenario workflow: `.github/workflows/scenario.yml` -- Validator: `scripts/validate-scenario.sh` diff --git a/src/test/scenarios/example.ts b/src/test/scenarios/example.ts deleted file mode 100644 index 49955b7dad..0000000000 --- a/src/test/scenarios/example.ts +++ /dev/null @@ -1,21 +0,0 @@ -import { expect } from "@playwright/test" -import { e2e } from "../e2e/utils/helpers" - -// Title – Short, descriptive name. -// Description – Purpose of the scenario and any relevant background. -// Preconditions – State the environment, data, or setup required. -// Steps – Numbered, detailed instructions for execution. -// Expected Results – The specific outcome that constitutes a pass. -// Priority – High/Medium/Low, depending on risk. -// GitHub PR - The GitHub PR number for which this scenario is written. - -// Minimal scenario test using the same E2E fixture stack -// This verifies we can open the Cline sidebar and interact with the chat input. -e2e("Scenario - basic smoke: sidebar opens and chat input visible", async ({ helper, sidebar }) => { - // Complete initial setup (mock BYOK API key) - await helper.signin(sidebar) - - // Verify chat input is available - const input = sidebar.getByTestId("chat-input") - await expect(input).toBeVisible() -})