kilocode/.github/workflows/smoke-test.yml

# kilocode_change - new file
# Standalone smoke test — checks out kilo-bench and runs a small set of
# Harbor eval tasks against the latest published CLI (or a specific version).
#
# Tasks:  hello-world (~$0.01), log-summary-date-ranges (~$0.13)
# Expected total: < $0.50, < 15 min wall clock
#
# Triggers:
#   - workflow_dispatch: manually from Actions tab (optionally pass a CLI version)
#   - workflow_call: from publish.yml after draft release assets are uploaded
#
# Required secrets:
#   KILO_API_KEY         — Kilo Gateway key
#   KILO_ORG_ID          — Kilo org ID
#   BENCH_GITHUB_TOKEN   — PAT with contents:read on Kilo-Org/kilo-bench (private repo)

name: smoke-test

on:
  workflow_dispatch:
    inputs:
      cli_version:
        description: "CLI version to test (e.g. 7.0.36). Leave blank for latest npm release."
        required: false
        type: string
  workflow_call:
    inputs:
      cli_version:
        description: "CLI version to test from draft release assets."
        required: false
        type: string

concurrency:
  group: smoke-test
  cancel-in-progress: false

permissions:
  contents: write # needed to read draft release assets when called from publish.yml

jobs:
  smoke-test:
    name: Smoke Test
    runs-on: ubuntu-24.04
    timeout-minutes: 30
    env:
      KILO_API_KEY: ${{ secrets.KILO_API_KEY }}
      KILO_ORG_ID: ${{ secrets.KILO_ORG_ID }}
    steps:
      - name: Free disk space
        continue-on-error: true
        run: |
          sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc \
                      /usr/local/share/boost /opt/hostedtoolcache
          sudo docker image prune --all --force

      - name: Checkout kilo-bench
        uses: actions/checkout@v4
        with:
          repository: Kilo-Org/kilo-bench
          token: ${{ secrets.BENCH_GITHUB_TOKEN }}

      - name: Install uv
        uses: astral-sh/setup-uv@v6
        with:
          enable-cache: true

      - name: Set up Python
        run: uv python install 3.13

      - name: Install dependencies
        run: uv sync --no-dev

      - name: Validate API key
        run: |
          if [[ -z "$KILO_API_KEY" ]]; then
            echo "::error::KILO_API_KEY secret is not set."
            exit 1
          fi

      - name: Resolve CLI asset URL
        id: cli
        env:
          GH_TOKEN: ${{ github.token }}
        run: |
          VERSION="${{ inputs.cli_version }}"
          if [[ -z "$VERSION" ]]; then
            echo "cli_url=" >> "$GITHUB_OUTPUT"
            echo "::notice::Testing latest CLI from npm"
            exit 0
          fi
          # Resolve the release asset via the API so this works for draft
          # releases too (browser /releases/download/... URLs 404 on drafts).
          # The installer passes Accept: application/octet-stream + bearer
          # auth, which is what the API asset endpoint expects.
          # The tag_name filter yields at most one release and one asset,
          # so --jq returns a single URL without needing head/pipefail games.
          URL=$(gh api "repos/${{ github.repository }}/releases" \
            --jq ".[] | select(.tag_name == \"v${VERSION}\") | .assets[] | select(.name == \"kilo-linux-x64.tar.gz\") | .url")
          if [[ -z "$URL" ]]; then
            echo "::error::asset kilo-linux-x64.tar.gz not found for v${VERSION}"
            exit 1
          fi
          echo "cli_url=$URL" >> "$GITHUB_OUTPUT"
          echo "::notice::Testing CLI v${VERSION} via asset API: $URL"

      - name: Run smoke test — hello-world
        env:
          KILO_CLI_URL: ${{ steps.cli.outputs.cli_url }}
          KILO_CLI_GITHUB_TOKEN: ${{ github.token }}
        run: |
          ./scripts/run_eval.sh \
            -m kilo/anthropic/claude-sonnet-4.6 \
            -d hello-world \
            --job-name smoke-test-hello-world

      - name: Run smoke test — log-summary-date-ranges
        env:
          KILO_CLI_URL: ${{ steps.cli.outputs.cli_url }}
          KILO_CLI_GITHUB_TOKEN: ${{ github.token }}
        run: |
          ./scripts/run_eval.sh \
            -m kilo/anthropic/claude-sonnet-4.6 \
            -d terminal-bench-sample \
            -t "log-summary-date-ranges" \
            --job-name smoke-test-log-summary

      - name: Validate results
        run: python3 scripts/validate_smoke_test.py jobs/smoke-test-*/

      - name: Upload results
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: smoke-test-results
          path: |
            jobs/smoke-test-*/**/result.json
            jobs/smoke-test-*/**/trajectory.json
          retention-days: 30
          if-no-files-found: warn