diff --git a/.github/workflows/scenario.yml b/.github/workflows/scenario.yml new file mode 100644 index 0000000000..b6999c6e4d --- /dev/null +++ b/.github/workflows/scenario.yml @@ -0,0 +1,140 @@ +name: Scenario Tests + +on: + push: + branches: + - main + pull_request: + types: [opened, reopened, synchronize, ready_for_review] + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + matrix_prep: + runs-on: ubuntu-latest + permissions: + id-token: write + contents: read + outputs: + matrix: ${{ steps.set-matrix.outputs.matrix }} + steps: + - id: set-matrix + run: | + echo 'matrix=[{"runner":"ubuntu"},{"runner":"windows"},{"runner":"macos"}]' >> $GITHUB_OUTPUT + + validate-scenario: + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - uses: actions/checkout@v4 + - name: Validate PR-specific scenario metadata + run: | + if [ "${{ github.event_name }}" != "pull_request" ]; then + echo "Not a pull_request event; skipping scenario validation." + exit 0 + fi + bash scripts/validate-scenario.sh "${{ github.event.number }}" + + scenarios: + needs: [matrix_prep, validate-scenario] + strategy: + fail-fast: false + matrix: + include: ${{ fromJson(needs.matrix_prep.outputs.matrix) }} + runs-on: ${{ matrix.runner }}-latest + timeout-minutes: 20 + permissions: + id-token: write + contents: read + steps: + - uses: actions/checkout@v4 + + - name: Setup Node.js environment + uses: actions/setup-node@v4 + with: + node-version: 22 + + # Cache root dependencies - only reuse if package-lock.json exactly matches + - name: Cache root dependencies + uses: actions/cache@v4 + id: root-cache + with: + path: node_modules + key: ${{ runner.os }}-npm-${{ hashFiles('package-lock.json') }} + + # Cache webview-ui dependencies - only reuse if package-lock.json exactly matches + - name: Cache webview-ui dependencies + uses: actions/cache@v4 + id: webview-cache + with: + path: webview-ui/node_modules + key: ${{ runner.os }}-npm-webview-${{ hashFiles('webview-ui/package-lock.json') }} + + # Cache VS Code installation + - name: Cache VS Code + uses: actions/cache@v4 + id: vscode-cache + with: + path: .vscode-test + key: vscode-${{ runner.os }}-stable-${{ hashFiles('.vscode-test.mjs', 'package.json') }} + restore-keys: | + vscode-${{ runner.os }}-stable- + + # Cache Playwright browsers + - name: Cache Playwright browsers + uses: actions/cache@v4 + id: playwright-cache + with: + path: | + ~/.cache/ms-playwright + ~/Library/Caches/ms-playwright + ~/AppData/Local/ms-playwright + key: playwright-browsers-${{ runner.os }}-${{ hashFiles('package-lock.json') }} + restore-keys: | + playwright-browsers-${{ runner.os }}- + + - name: Install root dependencies + if: steps.root-cache.outputs.cache-hit != 'true' + run: npm ci + + - name: Install webview-ui dependencies + if: steps.webview-cache.outputs.cache-hit != 'true' + run: cd webview-ui && npm ci + + - name: Install xvfb on Linux + if: matrix.runner == 'ubuntu' + run: sudo apt-get update && sudo apt-get install -y xvfb + + # Run optimized Scenario tests (reuses build steps like e2e) + - name: Run Scenario tests - Linux + if: matrix.runner == 'ubuntu' + run: xvfb-run -a npm run test:scenarios:optimal + + - name: Run Scenario tests - Non-Linux + if: matrix.runner != 'ubuntu' + run: npm run test:scenarios:optimal + + - uses: actions/upload-artifact@v4 + if: ${{ failure() }} + with: + name: playwright-recordings-${{ matrix.runner }} + path: | + test-results/playwright/ + + scenario-summary: + needs: scenarios + permissions: {} + runs-on: ubuntu-latest + if: always() + steps: + - name: Check all scenarios passed + run: | + if [ "${{ needs.scenarios.result }}" != "success" ]; then + echo "Some scenario tests failed" + exit 1 + fi + echo "All scenario tests passed successfully" diff --git a/package.json b/package.json index a612e56377..97763826be 100644 --- a/package.json +++ b/package.json @@ -365,6 +365,8 @@ "e2e": "playwright test -c playwright.config.ts", "test:e2e": "playwright install && vsce package --no-dependencies --allow-package-secrets sendgrid --out dist/e2e.vsix && node src/test/e2e/utils/build.mjs && playwright test", "test:e2e:optimal": "vsce package --no-dependencies --allow-package-secrets sendgrid --out dist/e2e.vsix && node src/test/e2e/utils/build.mjs && playwright test", + "test:scenarios": "playwright install && vsce package --no-dependencies --allow-package-secrets sendgrid --out dist/e2e.vsix && node src/test/e2e/utils/build.mjs && playwright test -c playwright.scenarios.config.ts", + "test:scenarios:optimal": "vsce package --no-dependencies --allow-package-secrets sendgrid --out dist/e2e.vsix && node src/test/e2e/utils/build.mjs && playwright test -c playwright.scenarios.config.ts", "install:all": "npm install && cd webview-ui && npm install", "dev:webview": "cd webview-ui && npm run dev", "build:webview": "cd webview-ui && npm run build", diff --git a/playwright.scenarios.config.ts b/playwright.scenarios.config.ts new file mode 100644 index 0000000000..1376b74bee --- /dev/null +++ b/playwright.scenarios.config.ts @@ -0,0 +1,35 @@ +import { defineConfig } from "@playwright/test" + +const isCI = !!process?.env?.CI +const isWindow = process?.platform?.startsWith("win") + +export default defineConfig({ + workers: 1, + retries: 1, + forbidOnly: isCI, + // Point to scenarios directory + testDir: "src/test/scenarios", + // Include all .ts in scenarios (we add a local global.setup.ts here) + testMatch: /.*\.ts/, + timeout: isCI || isWindow ? 40000 : 20000, + expect: { + timeout: isCI || isWindow ? 5000 : 2000, + }, + fullyParallel: true, + reporter: isCI ? [["github"], ["list"]] : [["list"]], + use: { + video: "retain-on-failure", + }, + projects: [ + { + name: "setup test environment", + // Reuse E2E setup file to avoid duplication + testDir: "src/test/e2e/utils", + testMatch: /global\.setup\.ts/, + }, + { + name: "scenario tests", + dependencies: ["setup test environment"], + }, + ], +}) diff --git a/scripts/validate-scenario.sh b/scripts/validate-scenario.sh new file mode 100644 index 0000000000..dd1d42d057 --- /dev/null +++ b/scripts/validate-scenario.sh @@ -0,0 +1,151 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Validate a PR-specific scenario exists in src/test/scenarios +# Rules: +# - example.ts is exempt +# - Exactly one scenario must match the current PR number +# - All non-exempt scenario files must include a valid "GitHub PR" metadata line +# +# Usage: ./scripts/validate-scenario.sh <PR_NUMBER> + +PR_NUMBER="${1:-}" + +SCENARIOS_DIR="src/test/scenarios" +EXEMPT_BASENAMES=("example.ts") + +red() { printf "\033[31m%s\033[0m\n" "$*"; } +green() { printf "\033[32m%s\033[0m\n" "$*"; } +yellow(){ printf "\033[33m%s\033[0m\n" "$*"; } + +is_exempt() { + local base="$1" + for ex in "${EXEMPT_BASENAMES[@]}"; do + if [[ "$base" == "$ex" ]]; then + return 0 + fi + done + return 1 +} + +usage() { + echo "Usage: $0 <PR_NUMBER>" >&2 + exit 2 +} + +if [[ -z "$PR_NUMBER" ]]; then + red "Error: PR number is required." + usage +fi + +if ! [[ "$PR_NUMBER" =~ ^[0-9]+$ ]]; then + red "Error: PR number must be numeric. Got: '$PR_NUMBER'" + exit 2 +fi + +if [[ ! -d "$SCENARIOS_DIR" ]]; then + red "Error: Scenarios directory not found: $SCENARIOS_DIR" + exit 1 +fi + +declare -a ALL_FILES=() +declare -a MISSING_METADATA=() +declare -a MALFORMED_METADATA=() +declare -a FOUND_FILES=() +declare -a FOUND_PRS=() + +# Find top-level .ts files in scenarios dir +while IFS= read -r -d '' file; do + ALL_FILES+=("$file") +done < <(find "$SCENARIOS_DIR" -maxdepth 1 -type f -name "*.ts" -print0 | sort -z) + +if [[ ${#ALL_FILES[@]} -eq 0 ]]; then + red "Error: No scenario files found in $SCENARIOS_DIR" + exit 1 +fi + +for file in "${ALL_FILES[@]}"; do + base="$(basename "$file")" + if is_exempt "$base"; then + continue + fi + + # Extract metadata line, pattern: // GitHub PR - 123 + meta_line="$(grep -E -m1 '^[[:space:]]*//[[:space:]]*GitHub[[:space:]]+PR[[:space:]]*-[[:space:]]*[0-9]+[[:space:]]*$' "$file" || true)" + + if [[ -z "$meta_line" ]]; then + MISSING_METADATA+=("$file") + continue + fi + + pr_in_file="$(sed -E 's@.*GitHub[[:space:]]+PR[[:space:]]*-[[:space:]]*([0-9]+).*@\1@' <<< "$meta_line" | tr -d '[:space:]')" + + if ! [[ "$pr_in_file" =~ ^[0-9]+$ ]]; then + MALFORMED_METADATA+=("$file") + continue + fi + + FOUND_FILES+=("$file") + FOUND_PRS+=("$pr_in_file") +done + +# Fail if any non-exempt file is missing metadata +if [[ ${#MISSING_METADATA[@]} -gt 0 ]]; then + red "Error: The following scenario files are missing the required metadata line '// GitHub PR - <number>':" + for f in "${MISSING_METADATA[@]}"; do + echo " - $f" + done + echo + echo "Please add a metadata line like:" + echo " // GitHub PR - $PR_NUMBER" + exit 1 +fi + +# Fail if any metadata is malformed +if [[ ${#MALFORMED_METADATA[@]} -gt 0 ]]; then + red "Error: The following scenario files have a malformed 'GitHub PR' metadata line:" + for f in "${MALFORMED_METADATA[@]}"; do + echo " - $f" + done + echo + echo "Expected format:" + echo " // GitHub PR - $PR_NUMBER" + exit 1 +fi + +# Count matches for this PR +matches=0 +declare -a MATCHED_FILES=() +for i in "${!FOUND_FILES[@]}"; do + if [[ "${FOUND_PRS[$i]}" == "$PR_NUMBER" ]]; then + matches=$((matches + 1)) + MATCHED_FILES+=("${FOUND_FILES[$i]}") + fi +done + +if [[ $matches -eq 0 ]]; then + red "Error: No scenario file found for PR #$PR_NUMBER." + echo "Scenarios with declared PRs:" + if [[ ${#FOUND_FILES[@]} -eq 0 ]]; then + echo " (none)" + else + for i in "${!FOUND_FILES[@]}"; do + echo " - ${FOUND_FILES[$i]} (PR: ${FOUND_PRS[$i]})" + done + fi + echo + echo "Please add a scenario in $SCENARIOS_DIR with a metadata line:" + echo " // GitHub PR - $PR_NUMBER" + exit 1 +fi + +if [[ $matches -gt 1 ]]; then + red "Error: Multiple scenario files found for PR #$PR_NUMBER (exactly one required):" + for f in "${MATCHED_FILES[@]}"; do + echo " - $f" + done + exit 1 +fi + +green "Success: Exactly one scenario found for PR #$PR_NUMBER:" +echo " - ${MATCHED_FILES[0]}" diff --git a/src/test/scenarios/README.md b/src/test/scenarios/README.md new file mode 100644 index 0000000000..49649dc334 --- /dev/null +++ b/src/test/scenarios/README.md @@ -0,0 +1,147 @@ +# Scenario Tests + +This directory contains PR-specific scenario tests that run on top of the existing Playwright E2E framework. Each PR must provide exactly one scenario test file with a PR metadata comment for validation and CI gating. + +## Overview + +- Framework: Playwright, reusing the same E2E fixtures/utilities at `src/test/e2e/utils/` +- Config: Scenario runs are configured via `playwright.scenarios.config.ts` +- CI Workflow: `.github/workflows/scenario.yml` +- Validation: A script enforces “exactly one scenario per PR” with a metadata comment + +## Architecture + +Scenario tests reuse the E2E fixture stack: +- `e2e` fixture from `src/test/e2e/utils/helpers.ts` bootstraps VS Code, installs the extension VSIX, opens the Cline sidebar, and provides utilities (`helper`, `sidebar`, etc.). +- Scenario tests live here in `src/test/scenarios`, using the same Playwright helpers and patterns as E2E but focused on validating PR-specific behavior. +- Scenario tests are intended to be short, focused validations of the change introduced in the PR. + +Key pieces: +- `playwright.scenarios.config.ts` points Playwright to this directory and reuses the shared E2E global setup to keep behavior identical. +- `scripts/validate-scenario.sh` checks that a PR includes exactly one scenario test with a metadata line identifying the PR number. +- `.github/workflows/scenario.yml` runs validation, then runs the Playwright scenario matrix on Ubuntu/Windows/macOS, then aggregates results in a single summary job. + +## File Layout + +- `example.ts` — A template/example scenario (exempt from validation) +- `*.ts` — Your PR-specific scenario tests (top-level only; not recursive) + +Only top-level `.ts` files in this directory are scanned by the validator. Subdirectories are not scanned. + +## Authoring a Scenario + +Add a new file in this directory for your PR, and include the PR metadata comment. Exactly one scenario per PR is required. + +Template: + +```ts +import { expect } from "@playwright/test" +import { e2e } from "../e2e/utils/helpers" + +// Title – Short, descriptive name. +// Description – Purpose of the scenario and any relevant background. +// Preconditions – State the environment, data, or setup required. +// Steps – Numbered, detailed instructions for execution. +// Expected Results – The specific outcome that constitutes a pass. +// Priority – High/Medium/Low, depending on risk. +// GitHub PR - 123 <-- REQUIRED: replace 123 with your PR number + +e2e("Scenario - PR 123 - brief description", async ({ helper, sidebar }) => { + await helper.signin(sidebar) + // ... your steps ... + await expect(sidebar.getByTestId("chat-input")).toBeVisible() +}) +``` + +Notes: +- `example.ts` is exempt and will be ignored by the validator. +- The PR metadata comment must match the pattern: `// GitHub PR - `. +- Exactly one scenario with the current PR number must exist. + +## Running Locally + +### Quick run (recommended) +Runs packaging, VS Code/Chromium setup, then the scenario tests: + +```bash +npm run test:scenarios:optimal +``` + +This will: +1) Package the extension VSIX used for testing +2) Ensure Playwright Chromium and VS Code test binary are installed +3) Execute Playwright using `playwright.scenarios.config.ts` + +### Validate your scenario metadata locally +The validator enforces “one scenario per PR” by scanning the metadata line: + +```bash +# Fails if no scenario exists for PR 123 (or if multiple exist, or metadata is malformed) +bash scripts/validate-scenario.sh 123 +``` + +Tip: To simulate success quickly: + +```bash +printf '%s\n%s\n' '// GitHub PR - 123' 'export {}' > src/test/scenarios/pr-123.ts +bash scripts/validate-scenario.sh 123 +rm src/test/scenarios/pr-123.ts +``` + +### Using act to run the workflow locally (Linux-only) +`act` cannot run macOS/Windows runners. Map Ubuntu runner to a Docker image and run: + +```bash +# Validate then run the Ubuntu scenario job +act -W .github/workflows/scenario.yml -P ubuntu-latest=catthehacker/ubuntu:act-latest +``` + +Note: macOS/Windows jobs will be skipped under `act`. The GitHub-hosted Actions will run all three OS jobs. + +## CI / GitHub Actions Setup + +Workflow file: `.github/workflows/scenario.yml` + +Jobs: +1) `matrix_prep` — Builds the 3-OS matrix +2) `validate-scenario` — Ensures PR has exactly one scenario with `// GitHub PR - ` (skips validation for non-PR events) +3) `scenarios` — Runs Playwright across Ubuntu/Windows/macOS +4) `scenario-summary` — Single summary status check that fails if any OS job fails + +### Enforcing “must run before merging” +Use Branch Protection Rules: +1) Repo → Settings → Branches → Add/Edit rule for `main` +2) Enable “Require status checks to pass before merging” +3) Select `scenario-summary` as a required check +4) Optionally enable “Require branches to be up to date before merging” + +This blocks merges until: +- The validator passes (exactly one scenario per PR with correct metadata) +- All three OS scenario runs pass + +## Troubleshooting + +- “No scenario file found for PR #” + - Ensure exactly one `.ts` file in this directory (excluding `example.ts`) includes the line: + `// GitHub PR - ` + - Ensure the file is top-level (not in a subdirectory). + +- “Malformed ‘GitHub PR’ metadata line” + - The line must be a single-line comment containing a numeric PR: + `// GitHub PR - 123` + +- “Multiple scenario files found for PR #” + - Reduce to exactly one scenario file for the PR. + +- `act` skips macOS/Windows + - This is expected. On GitHub Actions, those OS jobs will run. + +- Slow first runs locally/CI + - The E2E/Scenario flows download VS Code and Playwright Chromium. Caches in CI should speed up subsequent runs. + +## See Also + +- E2E fixtures and helpers: `src/test/e2e/utils/` +- Scenario Playwright config: `playwright.scenarios.config.ts` +- Scenario workflow: `.github/workflows/scenario.yml` +- Validator: `scripts/validate-scenario.sh` diff --git a/src/test/scenarios/example.ts b/src/test/scenarios/example.ts new file mode 100644 index 0000000000..49955b7dad --- /dev/null +++ b/src/test/scenarios/example.ts @@ -0,0 +1,21 @@ +import { expect } from "@playwright/test" +import { e2e } from "../e2e/utils/helpers" + +// Title – Short, descriptive name. +// Description – Purpose of the scenario and any relevant background. +// Preconditions – State the environment, data, or setup required. +// Steps – Numbered, detailed instructions for execution. +// Expected Results – The specific outcome that constitutes a pass. +// Priority – High/Medium/Low, depending on risk. +// GitHub PR - The GitHub PR number for which this scenario is written. + +// Minimal scenario test using the same E2E fixture stack +// This verifies we can open the Cline sidebar and interact with the chat input. +e2e("Scenario - basic smoke: sidebar opens and chat input visible", async ({ helper, sidebar }) => { + // Complete initial setup (mock BYOK API key) + await helper.signin(sidebar) + + // Verify chat input is available + const input = sidebar.getByTestId("chat-input") + await expect(input).toBeVisible() +})