# Eval profile catalog — match your model claim to a profile, then run with:
#   slm-lm-eval --list-profiles
#   slm-lm-eval --profile reasoning --preset minicpm5-1b --experiment-name my-run
#
# See research/evals/docs/eval_profiles.md for full guide.

profiles:
  smoke:
    tool: slm-lm-eval
    claim: Quick sanity check
    description: Fast validation before a long run (2 tasks, capped samples).
    config: lm_eval_smoke.yaml
    tasks:
      - arc_easy
      - hellaswag

  reasoning:
    tool: slm-lm-eval
    claim: Better reasoning
    description: Math + commonsense + science QA (generation + multiple-choice).
    config: lm_eval_reasoning.yaml
    tasks:
      - gsm8k
      - arc_easy
      - arc_challenge
      - hellaswag

  math:
    tool: slm-lm-eval
    claim: Better math reasoning
    description: Grade-school math word problems (GSM8K) + abstract reasoning QA.
    config: lm_eval_math.yaml
    tasks:
      - gsm8k
      - arc_challenge

  science:
    tool: slm-lm-eval
    claim: Better science knowledge
    description: Science fact recall (SciQ, OpenBookQA) + science reasoning QA.
    config: lm_eval_science.yaml
    tasks:
      - sciq
      - openbookqa
      - arc_challenge

  understanding:
    tool: slm-lm-eval
    claim: Better language understanding
    description: NLU / reading comprehension (SuperGLUE-style multiple-choice).
    config: lm_eval_understanding.yaml
    tasks:
      - boolq
      - piqa
      - copa
      - rte

  code:
    tool: slm-lm-eval
    claim: Better code generation
    description: Python function synthesis (HumanEval + MBPP via lm-eval).
    config: lm_eval_code.yaml
    tasks:
      - humaneval
      - mbpp

  instructions:
    tool: slm-lm-eval
    claim: Better instruction following
    description: Verifiable instruction constraints (IFEval).
    config: lm_eval_instructions.yaml
    tasks:
      - ifeval

  medical:
    tool: slm-lm-eval
    claim: Better medical knowledge
    description: Clinical Q&A — PubMedQA + MedMCQA + MedQA (USMLE) with arc guard.
    config: lm_eval_medical.yaml
    tasks:
      - pubmedqa
      - medmcqa
      - medqa_4options
      - arc_challenge

  multilingual:
    tool: slm-lm-eval
    claim: Better multilingual understanding
    description: Cross-lingual NLI / commonsense / coreference (XNLI, XCOPA, XWinograd).
    config: lm_eval_multilingual.yaml
    tasks:
      - xnli
      - xcopa
      - xwinograd

  commonsense:
    tool: slm-lm-eval
    claim: Better commonsense reasoning
    description: Everyday-knowledge MCQ + coreference + physical commonsense.
    config: lm_eval_commonsense.yaml
    tasks:
      - commonsense_qa
      - winogrande
      - piqa
      - hellaswag

  safety:
    tool: slm-lm-eval
    claim: More truthful, fewer imitative falsehoods
    description: TruthfulQA MC2/MC1 (eval-only; do not train on the test set).
    config: lm_eval_safety.yaml
    tasks:
      - truthfulqa_mc2
      - truthfulqa_mc1
      - arc_easy

  french:
    tool: slm-lm-eval
    claim: Better French understanding and translation
    description: Official FrenchBench MC tasks + WMT14 EN→FR (CroissantLLM benchmark suite).
    config: lm_eval_french.yaml
    tasks:
      - french_bench_xnli
      - belebele_fra_Latn
      - french_bench_boolqa
      - wmt14-en-fr

  general_slm:
    tool: slm-lm-eval
    claim: General ~1B SLM baseline
    description: Balanced academic mix for before/after fine-tune on chat data.
    config: lm_eval_minicpm5.yaml
    tasks:
      - arc_easy
      - arc_challenge
      - hellaswag
      - piqa
      - boolq
      - gsm8k

  compare_study:
    tool: slm-lm-eval
    claim: Baseline vs finetune comparison
    description: Same tasks as general_slm with limit 100 for paired studies.
    config: lm_eval_compare_study.yaml
    tasks:
      - arc_easy
      - arc_challenge
      - hellaswag
      - piqa
      - boolq
      - gsm8k

suites:
  agentic_tool_use:
    tool: slm-benchmark
    claim: Tool use and function calling
    description: BFCL + tau-bench for agents that call tools.
    command: >-
      uv run --package slm-evals slm-benchmark
      --model <model> --benchmarks bfcl tau_bench --max-samples 50
    benchmarks:
      - bfcl
      - tau_bench

  agentic_gaia:
    tool: slm-benchmark
    claim: End-to-end assistant tasks
    description: GAIA — multi-step reasoning with optional tools.
    command: >-
      uv run --package slm-evals slm-benchmark
      --model <model> --benchmarks gaia --max-samples 20
    benchmarks:
      - gaia

  agentic_code:
    tool: slm-benchmark
    claim: Real-world code repair
    description: SWE-bench Verified patch generation (lightweight mode by default).
    command: >-
      uv run --package slm-evals slm-benchmark
      --model <model> --benchmarks swe_bench --max-samples 10
    benchmarks:
      - swe_bench

  agentic_all:
    tool: slm-benchmark
    claim: Full agentic suite
    description: All four slm-benchmark benchmarks.
    command: >-
      uv run --package slm-evals slm-benchmark
      --model <model> --benchmarks all --max-samples 50
    benchmarks:
      - bfcl
      - tau_bench
      - gaia
      - swe_bench

external:
  embeddings_mteb:
    tool: mteb
    claim: Better embeddings
    description: MTEB, BEIR, STS — not wired in this repo; use embeddings-benchmark/mteb.
    tasks_note: 56+ embedding tasks; requires a dedicated encoder model.

  chat_judge:
    tool: mt_bench_alpacaeval
    claim: Chat / instruction quality (judge-based)
    description: MT-Bench, AlpacaEval — require LLM-as-judge APIs; not in slm-lm-eval.

  rl_envs:
    tool: babyai_minigrid
    claim: Embodied planning
    description: BabyAI, MiniGrid — separate RL env evals; not integrated here.