From 587272125687da9982c9d86508c9ba805c75ecc0 Mon Sep 17 00:00:00 2001 From: Josh Lambert Date: Mon, 9 Mar 2026 00:36:18 -0400 Subject: [PATCH 1/2] Draft of triggering smoketest after draft release and before public publish --- .github/workflows/publish.yml | 67 +++++++++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 178b6ba582..c485b91548 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -240,11 +240,78 @@ jobs: # APPLE_API_KEY: ${{ secrets.APPLE_API_KEY }} # APPLE_API_KEY_PATH: ${{ runner.temp }}/apple-api-key.p8 + # kilocode_change start + smoke-test: + name: Smoke Test (pre-publish gate) + needs: + - version + - build-cli + if: github.repository == 'Kilo-Org/kilocode' + runs-on: ubuntu-24.04 + steps: + - name: Trigger kilo-bench smoke test + id: trigger + env: + GH_TOKEN: ${{ secrets.BENCH_GITHUB_TOKEN }} + run: | + # Dispatch to kilo-bench with the CLI version so it downloads the + # draft release binary rather than the npm-published version. + gh api repos/Kilo-Org/kilo-bench/dispatches \ + --method POST \ + -f event_type=smoke-test \ + -f 'client_payload[cli_version]=${{ needs.version.outputs.version }}' + + # Give GitHub a moment to register the run before we start polling. + sleep 10 + + # Find the run that was just created. + RUN_ID=$(gh run list \ + --repo Kilo-Org/kilo-bench \ + --workflow "smoke-test.yml" \ + --limit 1 \ + --json databaseId \ + --jq '.[0].databaseId') + + echo "run_id=$RUN_ID" >> "$GITHUB_OUTPUT" + echo "::notice::Smoke test run: https://github.com/Kilo-Org/kilo-bench/actions/runs/$RUN_ID" + + - name: Wait for smoke test to complete + env: + GH_TOKEN: ${{ secrets.BENCH_GITHUB_TOKEN }} + run: | + RUN_ID="${{ steps.trigger.outputs.run_id }}" + echo "Waiting for run $RUN_ID..." + + for i in $(seq 1 60); do + CONCLUSION=$(gh run view "$RUN_ID" \ + --repo Kilo-Org/kilo-bench \ + --json conclusion \ + --jq '.conclusion') + + echo " attempt $i: $CONCLUSION" + + if [[ "$CONCLUSION" == "success" ]]; then + echo "::notice::Smoke test passed." + exit 0 + elif [[ "$CONCLUSION" != "null" && "$CONCLUSION" != "" ]]; then + echo "::error::Smoke test failed with conclusion: $CONCLUSION" + echo "See: https://github.com/Kilo-Org/kilo-bench/actions/runs/$RUN_ID" + exit 1 + fi + + sleep 30 + done + + echo "::error::Smoke test did not complete within 30 minutes." + exit 1 + # kilocode_change end + publish: needs: - version - build-cli - build-vscode + - smoke-test # - build-tauri runs-on: ubuntu-24.04 steps: From 5ca582cafbba62c4667412731a0e82239dd2af60 Mon Sep 17 00:00:00 2001 From: josh Date: Tue, 10 Mar 2026 01:53:30 -0400 Subject: [PATCH 2/2] fix: robust smoke-test run identification and release targeting - Replace gh run list --limit 1 with timestamp-based API polling to avoid picking up an older or unrelated smoke-test run - Pass release_tag instead of cli_version so kilo-bench downloads from the exact draft release, not just the first one found for a version - Add 60s clock skew buffer for runner/GitHub time drift --- .github/workflows/publish.yml | 38 +++++++++++++++++++++-------------- 1 file changed, 23 insertions(+), 15 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index c485b91548..73f7b2fc33 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -254,26 +254,35 @@ jobs: env: GH_TOKEN: ${{ secrets.BENCH_GITHUB_TOKEN }} run: | - # Dispatch to kilo-bench with the CLI version so it downloads the - # draft release binary rather than the npm-published version. + BEFORE=$(date -u -d '60 seconds ago' +%Y-%m-%dT%H:%M:%SZ) + gh api repos/Kilo-Org/kilo-bench/dispatches \ --method POST \ -f event_type=smoke-test \ - -f 'client_payload[cli_version]=${{ needs.version.outputs.version }}' + -f 'client_payload[release_tag]=${{ needs.version.outputs.tag }}' \ + -f 'client_payload[source_run_id]=${{ github.run_id }}' - # Give GitHub a moment to register the run before we start polling. - sleep 10 + # Poll for the run created after our dispatch timestamp. + # The dispatch API returns no run ID, so we query by created time + # and pick the oldest match to avoid grabbing an unrelated run. + echo "Waiting for smoke-test run to appear (dispatched after $BEFORE)..." + for attempt in $(seq 1 30); do + RUN_ID=$(gh api \ + "repos/Kilo-Org/kilo-bench/actions/workflows/smoke-test.yml/runs?event=repository_dispatch&created=>=$BEFORE" \ + --jq '.workflow_runs | sort_by(.created_at) | .[0].id // empty') - # Find the run that was just created. - RUN_ID=$(gh run list \ - --repo Kilo-Org/kilo-bench \ - --workflow "smoke-test.yml" \ - --limit 1 \ - --json databaseId \ - --jq '.[0].databaseId') + if [[ -n "$RUN_ID" && "$RUN_ID" != "null" ]]; then + echo "run_id=$RUN_ID" >> "$GITHUB_OUTPUT" + echo "::notice::Smoke test run: https://github.com/Kilo-Org/kilo-bench/actions/runs/$RUN_ID" + exit 0 + fi - echo "run_id=$RUN_ID" >> "$GITHUB_OUTPUT" - echo "::notice::Smoke test run: https://github.com/Kilo-Org/kilo-bench/actions/runs/$RUN_ID" + echo " attempt $attempt: run not yet registered, retrying in 10s..." + sleep 10 + done + + echo "::error::Smoke test run did not appear within 5 minutes after dispatch." + exit 1 - name: Wait for smoke test to complete env: @@ -305,7 +314,6 @@ jobs: echo "::error::Smoke test did not complete within 30 minutes." exit 1 # kilocode_change end - publish: needs: - version