smoke-codex.yml

# Post-deploy end-to-end smoke test for the Codex plugin. It installs the
# deployed marketplace, runs a real Codex session through the daemon-capable
# `bt` CLI, and verifies that the Rust daemon sends span rows to a local mock
# Braintrust backend. Credentials and backend URLs are supplied to `bt`; the
# shared daemon config contains behavior settings only.

name: Smoke (codex)

on:
  workflow_call:
    inputs:
      dist_repo:
        required: true
        type: string
    secrets:
      PUBLISH_TOKEN:
        required: true
      OPENAI_API_KEY:
        required: false
  workflow_dispatch:
    inputs:
      dist_repo:
        description: "Distribution repo to install from (owner/name)."
        required: true
        type: string
        default: braintrustdata/test-coding-agent-dist

permissions:
  contents: read

env:
  MARKETPLACE: braintrust-codex-plugins

jobs:
  guard:
    runs-on: ubuntu-24.04
    outputs:
      has_key: ${{ steps.check.outputs.has_key }}
    steps:
      - id: check
        env:
          KEY: ${{ secrets.OPENAI_API_KEY }}
        run: |
          if [ -n "$KEY" ]; then
            echo "has_key=true" >> "$GITHUB_OUTPUT"
          else
            echo "has_key=false" >> "$GITHUB_OUTPUT"
            echo "::warning::OPENAI_API_KEY not set; skipping codex smoke test."
          fi

  smoke:
    needs: guard
    if: needs.guard.outputs.has_key == 'true'
    strategy:
      fail-fast: false
      matrix:
        include:
          - { runner: ubuntu-24.04, label: linux-x64 }
          - { runner: macos-15, label: darwin-arm64 }
          - { runner: macos-15-intel, label: darwin-x64 }
    runs-on: ${{ matrix.runner }}
    timeout-minutes: 15
    steps:
      - name: Checkout monorepo
        uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1

      - name: Set up Node
        uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4
        with:
          node-version: 22

      - name: Install Codex and bt CLIs
        run: |
          npm install -g @openai/codex
          curl -fsSL https://bt.dev/cli/install.sh | sh
          echo "$HOME/.local/bin" >> "$GITHUB_PATH"

      - name: Require daemon-capable bt
        run: |
          "$HOME/.local/bin/bt" agents hook --help

      - name: Install plugin from ${{ inputs.dist_repo }}
        env:
          GH_TOKEN: ${{ secrets.PUBLISH_TOKEN }}
        run: |
          git config --global url."https://x-access-token:${GH_TOKEN}@github.com/".insteadOf "https://github.com/"
          codex plugin marketplace add "${{ inputs.dist_repo }}"
          codex plugin add "trace-codex@${MARKETPLACE}"

      - name: Run real traced Codex session (${{ matrix.label }})
        env:
          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
          CODEX_API_KEY: ${{ secrets.OPENAI_API_KEY }}
          BRAINTRUST_API_KEY: smoke-key
          BRAINTRUST_API_URL: http://127.0.0.1:53999
          BRAINTRUST_APP_URL: http://127.0.0.1:53999
          BT_DAEMON_CONFIG: ${{ runner.temp }}/bt-daemon-config.json
          MOCK_COLLECTOR_OUT: ${{ runner.temp }}/mock-summary.json
        run: |
          printf '%s\n' '{"traceToBraintrust":true,"project":"trace-codex-smoke","flushOnTurnEnd":true}' > "$BT_DAEMON_CONFIG"
          python3 .github/scripts/mock-braintrust.py > "${{ runner.temp }}/mock-collector.log" 2>&1 &
          collector_pid=$!
          trap 'kill "$collector_pid" 2>/dev/null || true' EXIT
          for attempt in $(seq 1 50); do
            curl -fsS http://127.0.0.1:53999/version >/dev/null && break
            sleep 0.2
          done
          curl -fsS http://127.0.0.1:53999/version >/dev/null
          codex exec \
            --skip-git-repo-check \
            --dangerously-bypass-hook-trust \
            --sandbox read-only \
            "say hi"
          python3 -c 'import json, os; s=json.load(open(os.environ["MOCK_COLLECTOR_OUT"])); assert s["totalRows"] >= 1, s; print(s)'