From f8105861833aca4bd4e3403d428a9b7be9b5622c Mon Sep 17 00:00:00 2001 From: "kiloconnect[bot]" <240665456+kiloconnect[bot]@users.noreply.github.com> Date: Thu, 12 Mar 2026 10:31:48 +0000 Subject: [PATCH 1/2] fix: prevent infinite loop in visual regression CI when screenshots are non-deterministic --- .github/workflows/visual-regression.yml | 84 ++++++++++++++++++------- 1 file changed, 60 insertions(+), 24 deletions(-) diff --git a/.github/workflows/visual-regression.yml b/.github/workflows/visual-regression.yml index 49c02335fe5..9c3801df291 100644 --- a/.github/workflows/visual-regression.yml +++ b/.github/workflows/visual-regression.yml @@ -120,27 +120,45 @@ jobs: exit 1 fi - - name: Commit and push new baselines (if any) + - name: Verify updated baselines are stable if: needs.check-paths.outputs.is_fork != 'true' - id: commit-baselines + id: verify-baselines + run: | + git add packages/kilo-ui/tests/visual-regression.spec.ts-snapshots/ + if git diff --cached --quiet; then + echo "No new baselines — nothing to verify." + echo "changed=false" >> "$GITHUB_OUTPUT" + else + echo "Baselines changed — re-running tests to verify stability..." + git reset HEAD -- packages/kilo-ui/tests/visual-regression.spec.ts-snapshots/ + if (cd packages/kilo-ui && bun run test:visual); then + echo "Baselines are stable." + echo "changed=true" >> "$GITHUB_OUTPUT" + else + echo "::error::Visual regression baselines are non-deterministic. Tests failed against newly generated baselines." + echo "::error::This means screenshots keep changing between runs, which would cause an infinite update loop." + echo "::error::Please investigate the flaky screenshots and add non-deterministic stories to the SKIP set." + exit 1 + fi + fi + env: + CI: true + PLAYWRIGHT_WORKERS: "4" + + - name: Commit and push new baselines (if any) + if: needs.check-paths.outputs.is_fork != 'true' && steps.verify-baselines.outputs.changed == 'true' env: GH_TOKEN: ${{ secrets.BOT_PAT }} run: | git config user.name "github-actions[bot]" git config user.email "github-actions[bot]@users.noreply.github.com" git add packages/kilo-ui/tests/visual-regression.spec.ts-snapshots/ - if git diff --cached --quiet; then - echo "No new baselines — nothing to commit." - echo "changed=false" >> "$GITHUB_OUTPUT" - else - git commit -m "chore: update visual regression baselines" - git lfs push --all origin - git push - echo "changed=true" >> "$GITHUB_OUTPUT" - fi + git commit -m "chore: update visual regression baselines" + git lfs push --all origin + git push - name: Fail if baselines changed - if: needs.check-paths.outputs.is_fork != 'true' && steps.commit-baselines.outputs.changed == 'true' + if: needs.check-paths.outputs.is_fork != 'true' && steps.verify-baselines.outputs.changed == 'true' run: | echo "::error::Visual regression baselines changed. New baselines have been committed to the branch. Please pull and review." exit 1 @@ -238,27 +256,45 @@ jobs: exit 1 fi - - name: Commit and push new baselines (if any) + - name: Verify updated baselines are stable if: needs.check-paths.outputs.is_fork != 'true' - id: commit-baselines-vscode + id: verify-baselines-vscode + run: | + git add packages/kilo-vscode/tests/visual-regression.spec.ts-snapshots/ + if git diff --cached --quiet; then + echo "No new baselines — nothing to verify." + echo "changed=false" >> "$GITHUB_OUTPUT" + else + echo "Baselines changed — re-running tests to verify stability..." + git reset HEAD -- packages/kilo-vscode/tests/visual-regression.spec.ts-snapshots/ + if (cd packages/kilo-vscode && bun run test:visual); then + echo "Baselines are stable." + echo "changed=true" >> "$GITHUB_OUTPUT" + else + echo "::error::Visual regression baselines are non-deterministic. Tests failed against newly generated baselines." + echo "::error::This means screenshots keep changing between runs, which would cause an infinite update loop." + echo "::error::Please investigate the flaky screenshots and add non-deterministic stories to the SKIP set." + exit 1 + fi + fi + env: + CI: true + PLAYWRIGHT_WORKERS: "4" + + - name: Commit and push new baselines (if any) + if: needs.check-paths.outputs.is_fork != 'true' && steps.verify-baselines-vscode.outputs.changed == 'true' env: GH_TOKEN: ${{ secrets.BOT_PAT }} run: | git config user.name "github-actions[bot]" git config user.email "github-actions[bot]@users.noreply.github.com" git add packages/kilo-vscode/tests/visual-regression.spec.ts-snapshots/ - if git diff --cached --quiet; then - echo "No new baselines — nothing to commit." - echo "changed=false" >> "$GITHUB_OUTPUT" - else - git commit -m "chore: update kilo-vscode visual regression baselines" - git lfs push --all origin - git push - echo "changed=true" >> "$GITHUB_OUTPUT" - fi + git commit -m "chore: update kilo-vscode visual regression baselines" + git lfs push --all origin + git push - name: Fail if baselines changed - if: needs.check-paths.outputs.is_fork != 'true' && steps.commit-baselines-vscode.outputs.changed == 'true' + if: needs.check-paths.outputs.is_fork != 'true' && steps.verify-baselines-vscode.outputs.changed == 'true' run: | echo "::error::Visual regression baselines changed. New baselines have been committed to the branch. Please pull and review." exit 1 From ad11479ff396cc75d112d8f865d71334d58fdab4 Mon Sep 17 00:00:00 2001 From: "kiloconnect[bot]" <240665456+kiloconnect[bot]@users.noreply.github.com> Date: Thu, 12 Mar 2026 10:38:48 +0000 Subject: [PATCH 2/2] fix: use commit message check instead of re-running tests for loop prevention --- .github/workflows/visual-regression.yml | 112 ++++++++++++++---------- 1 file changed, 64 insertions(+), 48 deletions(-) diff --git a/.github/workflows/visual-regression.yml b/.github/workflows/visual-regression.yml index 9c3801df291..c32b187ae0e 100644 --- a/.github/workflows/visual-regression.yml +++ b/.github/workflows/visual-regression.yml @@ -57,6 +57,18 @@ jobs: with: lfs: true + - name: Check if HEAD is a baseline update commit + if: needs.check-paths.outputs.is_fork != 'true' + id: check-baseline-commit + run: | + COMMIT_MSG=$(git log -1 --format=%s) + if [ "$COMMIT_MSG" = "chore: update visual regression baselines" ] || [ "$COMMIT_MSG" = "chore: update kilo-vscode visual regression baselines" ]; then + echo "is_baseline_update=true" >> "$GITHUB_OUTPUT" + echo "HEAD commit is a baseline update commit — will not auto-commit again." + else + echo "is_baseline_update=false" >> "$GITHUB_OUTPUT" + fi + - name: Setup Bun uses: oven-sh/setup-bun@v2 with: @@ -120,45 +132,41 @@ jobs: exit 1 fi - - name: Verify updated baselines are stable - if: needs.check-paths.outputs.is_fork != 'true' - id: verify-baselines + - name: Fail if baselines still changing after auto-update + if: needs.check-paths.outputs.is_fork != 'true' && steps.check-baseline-commit.outputs.is_baseline_update == 'true' run: | git add packages/kilo-ui/tests/visual-regression.spec.ts-snapshots/ if git diff --cached --quiet; then - echo "No new baselines — nothing to verify." - echo "changed=false" >> "$GITHUB_OUTPUT" + echo "Baselines are stable after auto-update." else - echo "Baselines changed — re-running tests to verify stability..." - git reset HEAD -- packages/kilo-ui/tests/visual-regression.spec.ts-snapshots/ - if (cd packages/kilo-ui && bun run test:visual); then - echo "Baselines are stable." - echo "changed=true" >> "$GITHUB_OUTPUT" - else - echo "::error::Visual regression baselines are non-deterministic. Tests failed against newly generated baselines." - echo "::error::This means screenshots keep changing between runs, which would cause an infinite update loop." - echo "::error::Please investigate the flaky screenshots and add non-deterministic stories to the SKIP set." - exit 1 - fi + echo "::error::Visual regression baselines changed again after a previous auto-update commit." + echo "::error::This indicates non-deterministic screenshots that would cause an infinite update loop." + echo "::error::Please investigate the flaky screenshots and add non-deterministic stories to the SKIP set." + git diff --cached --stat + exit 1 fi - env: - CI: true - PLAYWRIGHT_WORKERS: "4" - name: Commit and push new baselines (if any) - if: needs.check-paths.outputs.is_fork != 'true' && steps.verify-baselines.outputs.changed == 'true' + if: needs.check-paths.outputs.is_fork != 'true' && steps.check-baseline-commit.outputs.is_baseline_update != 'true' + id: commit-baselines env: GH_TOKEN: ${{ secrets.BOT_PAT }} run: | git config user.name "github-actions[bot]" git config user.email "github-actions[bot]@users.noreply.github.com" git add packages/kilo-ui/tests/visual-regression.spec.ts-snapshots/ - git commit -m "chore: update visual regression baselines" - git lfs push --all origin - git push + if git diff --cached --quiet; then + echo "No new baselines — nothing to commit." + echo "changed=false" >> "$GITHUB_OUTPUT" + else + git commit -m "chore: update visual regression baselines" + git lfs push --all origin + git push + echo "changed=true" >> "$GITHUB_OUTPUT" + fi - name: Fail if baselines changed - if: needs.check-paths.outputs.is_fork != 'true' && steps.verify-baselines.outputs.changed == 'true' + if: needs.check-paths.outputs.is_fork != 'true' && steps.commit-baselines.outputs.changed == 'true' run: | echo "::error::Visual regression baselines changed. New baselines have been committed to the branch. Please pull and review." exit 1 @@ -193,6 +201,18 @@ jobs: with: lfs: true + - name: Check if HEAD is a baseline update commit + if: needs.check-paths.outputs.is_fork != 'true' + id: check-baseline-commit-vscode + run: | + COMMIT_MSG=$(git log -1 --format=%s) + if [ "$COMMIT_MSG" = "chore: update visual regression baselines" ] || [ "$COMMIT_MSG" = "chore: update kilo-vscode visual regression baselines" ]; then + echo "is_baseline_update=true" >> "$GITHUB_OUTPUT" + echo "HEAD commit is a baseline update commit — will not auto-commit again." + else + echo "is_baseline_update=false" >> "$GITHUB_OUTPUT" + fi + - name: Setup Bun uses: oven-sh/setup-bun@v2 with: @@ -256,45 +276,41 @@ jobs: exit 1 fi - - name: Verify updated baselines are stable - if: needs.check-paths.outputs.is_fork != 'true' - id: verify-baselines-vscode + - name: Fail if baselines still changing after auto-update + if: needs.check-paths.outputs.is_fork != 'true' && steps.check-baseline-commit-vscode.outputs.is_baseline_update == 'true' run: | git add packages/kilo-vscode/tests/visual-regression.spec.ts-snapshots/ if git diff --cached --quiet; then - echo "No new baselines — nothing to verify." - echo "changed=false" >> "$GITHUB_OUTPUT" + echo "Baselines are stable after auto-update." else - echo "Baselines changed — re-running tests to verify stability..." - git reset HEAD -- packages/kilo-vscode/tests/visual-regression.spec.ts-snapshots/ - if (cd packages/kilo-vscode && bun run test:visual); then - echo "Baselines are stable." - echo "changed=true" >> "$GITHUB_OUTPUT" - else - echo "::error::Visual regression baselines are non-deterministic. Tests failed against newly generated baselines." - echo "::error::This means screenshots keep changing between runs, which would cause an infinite update loop." - echo "::error::Please investigate the flaky screenshots and add non-deterministic stories to the SKIP set." - exit 1 - fi + echo "::error::Visual regression baselines changed again after a previous auto-update commit." + echo "::error::This indicates non-deterministic screenshots that would cause an infinite update loop." + echo "::error::Please investigate the flaky screenshots and add non-deterministic stories to the SKIP set." + git diff --cached --stat + exit 1 fi - env: - CI: true - PLAYWRIGHT_WORKERS: "4" - name: Commit and push new baselines (if any) - if: needs.check-paths.outputs.is_fork != 'true' && steps.verify-baselines-vscode.outputs.changed == 'true' + if: needs.check-paths.outputs.is_fork != 'true' && steps.check-baseline-commit-vscode.outputs.is_baseline_update != 'true' + id: commit-baselines-vscode env: GH_TOKEN: ${{ secrets.BOT_PAT }} run: | git config user.name "github-actions[bot]" git config user.email "github-actions[bot]@users.noreply.github.com" git add packages/kilo-vscode/tests/visual-regression.spec.ts-snapshots/ - git commit -m "chore: update kilo-vscode visual regression baselines" - git lfs push --all origin - git push + if git diff --cached --quiet; then + echo "No new baselines — nothing to commit." + echo "changed=false" >> "$GITHUB_OUTPUT" + else + git commit -m "chore: update kilo-vscode visual regression baselines" + git lfs push --all origin + git push + echo "changed=true" >> "$GITHUB_OUTPUT" + fi - name: Fail if baselines changed - if: needs.check-paths.outputs.is_fork != 'true' && steps.verify-baselines-vscode.outputs.changed == 'true' + if: needs.check-paths.outputs.is_fork != 'true' && steps.commit-baselines-vscode.outputs.changed == 'true' run: | echo "::error::Visual regression baselines changed. New baselines have been committed to the branch. Please pull and review." exit 1