.github/workflows/eval.yml

name: Eval

# The measurement loop, one button:
#   suite=retrieval+judge (default) — recall@k + MRR over the 100-question set
#     against the LIVE index (plus the rank diagnostic), AND the LLM-as-judge
#     answer-quality anchor. The default is the full one-click measurement
#     because the GitHub mobile app cannot set workflow inputs.
#   suite=retrieval — recall/MRR only. No LLM needed.
#   suite=agentic — the agent-loop benchmark (loop vs single-shot coverage
#     delta). Needs LLM keys; slow on free tiers.
#   suite=judge — LLM-as-judge answer quality (faithfulness / relevance /
#     citation-correctness) over the grounded path. Needs LLM keys.
#   suite=all — everything.
# Results are committed back to main for before/after diffing.
on:
  workflow_dispatch:
    inputs:
      suite:
        description: "what to measure. The default runs retrieval AND judge —
          deliberately, because the GitHub mobile app can't set inputs, so the
          default must be the measurement we actually want one click to give:
          recall/MRR plus the answer-quality anchor. agentic stays opt-in."
        type: choice
        options: [retrieval+judge, retrieval, agentic, judge, all]
        default: retrieval+judge
      set:
        description: "retrieval eval set (v1 = 100 questions, v0 = original 13)"
        type: string
        default: v1
      model:
        description: "embedding model to query with — MUST match the model the
          live index was built with, or the query vector width won't fit."
        type: string
        default: "BAAI/bge-small-en-v1.5"
      k:
        description: "top-k sections retrieved per question (the app's default
          is 8; try 4 to measure precision-over-volume)"
        type: string
        default: "8"
      provider:
        description: "agentic/judge: LLM provider. Default openai-compat = PAID
          hy3 (account credit) — reliable, no daily cap. gemini free is only
          20 req/day per model, so it 429s (RESOURCE_EXHAUSTED) partway through
          the judge set; failures still fall through to the other provider."
        type: choice
        options: [openai-compat, gemini]
        default: openai-compat
      llm:
        description: "agentic/judge: OpenRouter slugs (fallback chain). Primary
          is PAID tencent/hy3 (spends account credit) — no free-tier queue; the
          :free llama stays as an emergency fallback only."
        type: string
        default: "tencent/hy3,meta-llama/llama-3.3-70b-instruct:free"
      limit:
        description: "agentic only: first N questions (0 = all 20)"
        type: string
        default: "6"
      judge_limit:
        description: "judge only: first N questions of the set (0 = all)"
        type: string
        default: "10"

permissions:
  contents: write

jobs:
  retrieval:
    if: ${{ inputs.suite == 'retrieval' || inputs.suite == 'retrieval+judge' || inputs.suite == 'all' }}
    runs-on: ubuntu-latest
    timeout-minutes: 15
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with:
          python-version: "3.11"
          cache: pip
      - name: Restore embedding model
        uses: actions/cache/restore@v4
        with:
          path: ~/.cache/huggingface
          key: hf-${{ inputs.model }}
      - run: pip install -e .
      - name: Measure recall/MRR
        env:
          PYTHONUNBUFFERED: "1"
          NEON_URL: ${{ secrets.NEON }}
          TORCHDOCS_EVAL_SET: ${{ inputs.set }}
          TORCHDOCS_EMBED_MODEL: ${{ inputs.model }}
          TORCHDOCS_RETRIEVAL_K: ${{ inputs.k }}
        run: python -m eval.run_retrieval
      - name: Rank diagnostic (why do the known misses miss)
        env:
          PYTHONUNBUFFERED: "1"
          NEON_URL: ${{ secrets.NEON }}
          TORCHDOCS_EMBED_MODEL: ${{ inputs.model }}
        run: python -m eval.diagnose_retrieval
      - name: Commit results back to the branch
        run: |
          git config user.name "github-actions[bot]"
          git config user.email "github-actions[bot]@users.noreply.github.com"
          git add -f eval/results/retrieval_*.jsonl
          git diff --cached --quiet || git commit -m "eval: retrieval results from Actions run [skip ci]"
          git pull --rebase origin main  # main may have moved during the run
          git push

  agentic:
    if: ${{ inputs.suite == 'agentic' || inputs.suite == 'all' }}
    runs-on: ubuntu-latest
    timeout-minutes: 120
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with:
          python-version: "3.11"
          cache: pip
      - name: Restore embedding model
        uses: actions/cache/restore@v4
        with:
          path: ~/.cache/huggingface
          key: hf-${{ inputs.model }}
      - run: pip install -e .
      - name: Run the agentic benchmark
        env:
          PYTHONUNBUFFERED: "1"
          NEON_URL: ${{ secrets.NEON }}
          TORCHDOCS_PROVIDER: ${{ inputs.provider }}
          GEMINI_API_KEY: ${{ secrets.GOOGLE_API }}
          TORCHDOCS_EMBED_MODEL: ${{ inputs.model }}
          OPENAI_COMPAT_BASE_URL: https://openrouter.ai/api/v1
          OPENAI_COMPAT_API_KEY: ${{ secrets.OPENROUTER }}
          TORCHDOCS_OPENAI_COMPAT_MODEL: ${{ inputs.llm }}
          TORCHDOCS_AGENTIC_LIMIT: ${{ inputs.limit }}
        run: python -m eval.run_agentic
      - name: Commit results back to the branch
        run: |
          git config user.name "github-actions[bot]"
          git config user.email "github-actions[bot]@users.noreply.github.com"
          git add -f eval/results/agentic_v1*.jsonl
          git diff --cached --quiet || git commit -m "eval: agentic results from Actions run [skip ci]"
          git pull --rebase origin main  # main may have moved during the run
          git push

  judge:
    if: ${{ inputs.suite == 'judge' || inputs.suite == 'retrieval+judge' || inputs.suite == 'all' }}
    runs-on: ubuntu-latest
    timeout-minutes: 120
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with:
          python-version: "3.11"
          cache: pip
      - name: Restore embedding model
        uses: actions/cache/restore@v4
        with:
          path: ~/.cache/huggingface
          key: hf-${{ inputs.model }}
      - run: pip install -e .
      - name: Judge answer quality (faithfulness / relevance / citations)
        env:
          PYTHONUNBUFFERED: "1"
          NEON_URL: ${{ secrets.NEON }}
          TORCHDOCS_PROVIDER: ${{ inputs.provider }}
          GEMINI_API_KEY: ${{ secrets.GOOGLE_API }}
          TORCHDOCS_EMBED_MODEL: ${{ inputs.model }}
          OPENAI_COMPAT_BASE_URL: https://openrouter.ai/api/v1
          OPENAI_COMPAT_API_KEY: ${{ secrets.OPENROUTER }}
          TORCHDOCS_OPENAI_COMPAT_MODEL: ${{ inputs.llm }}
          TORCHDOCS_EVAL_SET: ${{ inputs.set }}
          TORCHDOCS_JUDGE_LIMIT: ${{ inputs.judge_limit }}
          TORCHDOCS_RETRIEVAL_K: ${{ inputs.k }}
        run: python -m eval.run_judge
      - name: Commit results back to the branch
        run: |
          git config user.name "github-actions[bot]"
          git config user.email "github-actions[bot]@users.noreply.github.com"
          git add -f eval/results/judge_*.jsonl
          git diff --cached --quiet || git commit -m "eval: judge results from Actions run [skip ci]"
          git pull --rebase origin main  # main may have moved during the run
          git push