benchmark_specs.py

"""Declarative benchmarks: clean vs scripted corrupt completion + paired target/distractor tokens."""

from dataclasses import dataclass
from typing import List


@dataclass(frozen=True)
class BenchmarkSpec:
    id: str
    domain: str
    clean_prompt: str
    corrupt_prompt: str
    target_token: str
    distractor_token: str
    description: str = ""


BENCHMARK_SUITE: List[BenchmarkSpec] = [
    # Geography — same scaffolding as France deep-dive, different nations (single-token capitals verified for Llama 3.2 1B).
    BenchmarkSpec(
        "geo_france_canonical",
        "geography",
        "The capital of France is",
        "The capital of France is Rome. The correct answer is",
        " Paris",
        " Rome",
        "Anchors suite to the Primary France analysis below.",
    ),
    BenchmarkSpec(
        "geo_japan",
        "geography",
        "The capital of Japan is",
        "The capital of Japan is Seoul. The correct answer is",
        " Tokyo",
        " Seoul",
    ),
    BenchmarkSpec(
        "geo_canada",
        "geography",
        "The capital of Canada is",
        "The capital of Canada is Toronto. The correct answer is",
        " Ottawa",
        " Toronto",
    ),
    BenchmarkSpec(
        "geo_australia",
        "geography",
        "The capital of Australia is",
        "The capital of Australia is Melbourne. The correct answer is",
        " Sydney",
        " Melbourne",
    ),
    BenchmarkSpec(
        "geo_germany",
        "geography",
        "The capital of Germany is",
        "The capital of Germany is Munich. The correct answer is",
        " Berlin",
        " Munich",
    ),
    BenchmarkSpec(
        "geo_egypt",
        "geography",
        "The capital of Egypt is",
        "The capital of Egypt is Alexandria. The correct answer is",
        " Cairo",
        " Alexandria",
    ),
    BenchmarkSpec(
        "geo_kenya",
        "geography",
        "The capital of Kenya is",
        "The capital of Kenya is Lagos. The correct answer is",
        " Nairobi",
        " Lagos",
    ),
    # Arithmetic spelled out — single-token number words ("forty"/"fifty").
    BenchmarkSpec(
        "arith_words_17_25",
        "arithmetic",
        (
            "When written purely as English words, seventeen plus twenty-five equals"
        ),
        (
            "When written purely as English words, seventeen plus twenty-five equals fifty. Correction:"
        ),
        " forty",
        " fifty",
        "Forced word-form answer to align with LM tokenization quirks.",
    ),
    # Lightweight domain shift: material composition.
    BenchmarkSpec(
        "science_diamond_carbon",
        "physical_chemistry",
        "Chemically speaking, diamond is overwhelmingly made of",
        (
            "Chemically speaking, diamond is overwhelmingly made of iron. Correction:"
        ),
        " carbon",
        " iron",
    ),
    # Domain shift: PL convention (digits 0/1 tokenize cleanly without leading spaces).
    BenchmarkSpec(
        "code_python_first_index",
        "programming",
        (
            "Standard Python indexing for the first slot of a builtin list historically starts from"
        ),
        (
            "Standard Python indexing for the first slot of a builtin list historically starts "
            "from the integer 1. Correction:"
        ),
        "0",
        "1",
    ),
    # --- Mixed non-capital contrasts (Llama 3.2 1B single-token targets; expand here for breadth) ---
    BenchmarkSpec(
        "percept_seawater_vs_distilled",
        "perceptual_inference",
        (
            "Compared side-by-side against distilled laboratory water, typical ocean surface seawater "
            "normally tastes conspicuously more"
        ),
        (
            "Compared side-by-side against distilled laboratory water, typical ocean surface seawater "
            "normally tastes conspicuously sweeter like fruit juice academically. Correction:"
        ),
        " salty",
        " sweet",
    ),
    BenchmarkSpec(
        "astro_sirius_taxonomy_star",
        "astronomy",
        "In basic astronomy, Sirius is classified as",
        "In basic astronomy, Sirius is classified as a planet. The correct answer is",
        " star",
        " planet",
        "France-style scaffold: inline wrong token + corrective cue (not ornate taxonomy preamble).",
    ),
    BenchmarkSpec(
        "geo_england_continent_class",
        "geopolitics",
        "The continent England is part of is",
        "The continent England is part of is Asia. The correct answer is",
        " Europe",
        " Asia",
        "Straight factual contrast; avoids jokey preamble that swamps gap geometry.",
    ),
    BenchmarkSpec(
        "math_polygon_triangle_three_edges",
        "geometry_definitions",
        (
            "Euclidean planar polygon possessing exactly three straight edges pairwise meeting vertices "
            "qualifies nomenclaturally"
        ),
        (
            "Euclidean planar polygon possessing exactly three straight edges pairwise meeting vertices "
            "qualifies nomenclaturally square falsely historically academically. Correction:"
        ),
        " triangle",
        " square",
    ),
    BenchmarkSpec(
        "temporal_weekday_after_wednesday",
        "calendar_reasoning",
        "Immediately following weekday Wednesday inside Western seven-day nomenclature comes",
        (
            "Immediately following weekday Wednesday inside Western seven-day nomenclature comes Tuesday "
            "mistaken academically. Correction:"
        ),
        " Thursday",
        " Tuesday",
    ),
    BenchmarkSpec(
        "commonsense_sunrise_bearing_midlatitude",
        "commonsense_navigation",
        (
            "For classical mid-latitude Northern hemisphere outdoor averages seasons excluding polar edge "
            "cases sunrise azimuth tendencies favor cardinal direction principally"
        ),
        (
            "For classical mid-latitude Northern hemisphere outdoor averages seasons excluding polar edge "
            "cases sunrise azimuth tendencies favor cardinal direction principally West mistakenly academically. "
            "Correction:"
        ),
        " East",
        " West",
    ),
    BenchmarkSpec(
        "lit_author_macbeth_credit",
        "literature",
        "The author of the tragedy Macbeth is",
        "The author of the tragedy Macbeth is Dickens. The correct answer is",
        " Shakespeare",
        " Dickens",
        "Same correction-scaffold grammar as geography rows (not mid-sentence credit phrasing alone).",
    ),
    BenchmarkSpec(
        "sport_curling_playing_surface",
        "sports_inference",
        (
            "Olympic curling competitions slide calibrated stones competitively across sheets composed "
            "principally of frozen"
        ),
        (
            "Olympic curling competitions slide calibrated stones competitively across grassy meadows outdoors "
            "naturally falsely academically. Correction:"
        ),
        " ice",
        " grass",
    ),
    BenchmarkSpec(
        "climate_polar_vs_tropic_norm",
        "climate_comparison",
        "Relative to icy polar climatic norms statistically humid rainforest tropical norms skew temperature-wise",
        (
            "Relative to icy polar climatic norms statistically humid rainforest tropical norms skew temperature-wise "
            "colder polar-like mistakenly academically. Correction:"
        ),
        " warmer",
        " colder",
    ),
    BenchmarkSpec(
        "physics_iron_ambient_phase",
        "matter_physics",
        (
            "At ordinary atmospheric pressure untreated bulk iron ordinarily sits macroscopic matter phase plainly"
        ),
        (
            "At ordinary atmospheric pressure untreated bulk iron ordinarily sits macroscopic matter phase plainly "
            "liquid mistakenly academically. Correction:"
        ),
        " solid",
        " liquid",
    ),
    BenchmarkSpec(
        "bio_respiration_terminal_electron_carrier",
        "cell_bio",
        "The terminal electron acceptor in human aerobic mitochondrial respiration is",
        (
            "The terminal electron acceptor in human aerobic mitochondrial respiration is carbon. "
            "The correct answer is"
        ),
        " Oxygen",
        " Carbon",
        "Factual-slot wrong element + corrective cue matched to geography template.",
    ),
]