.github/workflows/eval.yml

name: eval

# The Trust & Safety eval harness in CI. The deterministic safety gate runs free on every push
# (no API key, no model) and BLOCKS the build if a policy-gate invariant or citation check fails.
# The LLM-as-judge layer runs only when an ANTHROPIC_API_KEY secret is configured, on a small
# subset, so the live model never gates the build and never surprises us with cost.

on:
  push:
    branches: ["**"]
  pull_request:
    branches: [main]

jobs:
  eval:
    runs-on: ubuntu-latest
    env:
      # Empty unless the repo has the secret set (Settings -> Secrets -> Actions). The judge step
      # below keys off this being non-empty, so forks / unconfigured repos still get a green gate.
      ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
      LLM_PROVIDER: anthropic
    steps:
      - uses: actions/checkout@v4

      - uses: actions/setup-python@v5
        with:
          python-version: "3.11"
          cache: pip

      - name: Install dependencies
        run: pip install -r requirements.txt

      - name: Test set is committed and up to date
        run: python -m eval.build_testset --check

      - name: Deterministic safety gate (free, blocking)
        run: python -m eval.run_eval --deterministic-only

      # Cache the local embedding model so the judge step's build_index doesn't re-download it from
      # the HF Hub on every run (that download has rate-limited to HTTP 429 and is the only thing
      # that has ever reddened this workflow). Cold cache still downloads; a warm cache skips it.
      - name: Cache embedding model (all-MiniLM-L6-v2)
        if: ${{ env.ANTHROPIC_API_KEY != '' }}
        uses: actions/cache@v4
        with:
          path: ~/.cache/huggingface
          key: hf-${{ runner.os }}-all-MiniLM-L6-v2

      # Live layers run only if the secret is present. Builds the policy/precedent index first
      # (local embeddings, no key needed), then triages a 5-case subset with the LLM-as-judge.
      # ADVISORY, not blocking (continue-on-error): per this repo's design the *deterministic* gate
      # above is the only thing that may fail the build. The live model is nondeterministic and its
      # external deps (HF Hub, Anthropic API) can flake, so it must never gate. Results are still
      # uploaded below; a real judge regression surfaces there, not as a red required check.
      - name: LLM-judge on a subset (only if ANTHROPIC_API_KEY is set)
        if: ${{ env.ANTHROPIC_API_KEY != '' }}
        continue-on-error: true
        run: |
          python -m scripts.build_index
          python -m eval.run_eval --judge --limit 5 --strict

      - name: Upload eval results
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: eval-results
          path: eval/results.json
          if-no-files-found: ignore

  # The React SPA is built into the deploy image, so a broken build would only surface on the Space.
  # This job catches it in CI (type errors, bad imports, Tailwind config) before it reaches deploy.
  frontend-build:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-node@v4
        with:
          node-version: "20"
          cache: npm
          cache-dependency-path: frontend/package-lock.json
      - name: Install & build the SPA
        working-directory: frontend
        run: |
          npm ci
          npm run build