"""Declarative benchmarks: clean vs scripted corrupt completion + paired target/distractor tokens.""" from dataclasses import dataclass from typing import List @dataclass(frozen=True) class BenchmarkSpec: id: str domain: str clean_prompt: str corrupt_prompt: str target_token: str distractor_token: str description: str = "" BENCHMARK_SUITE: List[BenchmarkSpec] = [ # Geography — same scaffolding as France deep-dive, different nations (single-token capitals verified for Llama 3.2 1B). BenchmarkSpec( "geo_france_canonical", "geography", "The capital of France is", "The capital of France is Rome. The correct answer is", " Paris", " Rome", "Anchors suite to the Primary France analysis below.", ), BenchmarkSpec( "geo_japan", "geography", "The capital of Japan is", "The capital of Japan is Seoul. The correct answer is", " Tokyo", " Seoul", ), BenchmarkSpec( "geo_canada", "geography", "The capital of Canada is", "The capital of Canada is Toronto. The correct answer is", " Ottawa", " Toronto", ), BenchmarkSpec( "geo_australia", "geography", "The capital of Australia is", "The capital of Australia is Melbourne. The correct answer is", " Sydney", " Melbourne", ), BenchmarkSpec( "geo_germany", "geography", "The capital of Germany is", "The capital of Germany is Munich. The correct answer is", " Berlin", " Munich", ), BenchmarkSpec( "geo_egypt", "geography", "The capital of Egypt is", "The capital of Egypt is Alexandria. The correct answer is", " Cairo", " Alexandria", ), BenchmarkSpec( "geo_kenya", "geography", "The capital of Kenya is", "The capital of Kenya is Lagos. The correct answer is", " Nairobi", " Lagos", ), # Arithmetic spelled out — single-token number words ("forty"/"fifty"). BenchmarkSpec( "arith_words_17_25", "arithmetic", ( "When written purely as English words, seventeen plus twenty-five equals" ), ( "When written purely as English words, seventeen plus twenty-five equals fifty. Correction:" ), " forty", " fifty", "Forced word-form answer to align with LM tokenization quirks.", ), # Lightweight domain shift: material composition. BenchmarkSpec( "science_diamond_carbon", "physical_chemistry", "Chemically speaking, diamond is overwhelmingly made of", ( "Chemically speaking, diamond is overwhelmingly made of iron. Correction:" ), " carbon", " iron", ), # Domain shift: PL convention (digits 0/1 tokenize cleanly without leading spaces). BenchmarkSpec( "code_python_first_index", "programming", ( "Standard Python indexing for the first slot of a builtin list historically starts from" ), ( "Standard Python indexing for the first slot of a builtin list historically starts " "from the integer 1. Correction:" ), "0", "1", ), # --- Mixed non-capital contrasts (Llama 3.2 1B single-token targets; expand here for breadth) --- BenchmarkSpec( "percept_seawater_vs_distilled", "perceptual_inference", ( "Compared side-by-side against distilled laboratory water, typical ocean surface seawater " "normally tastes conspicuously more" ), ( "Compared side-by-side against distilled laboratory water, typical ocean surface seawater " "normally tastes conspicuously sweeter like fruit juice academically. Correction:" ), " salty", " sweet", ), BenchmarkSpec( "astro_sirius_taxonomy_star", "astronomy", "In basic astronomy, Sirius is classified as", "In basic astronomy, Sirius is classified as a planet. The correct answer is", " star", " planet", "France-style scaffold: inline wrong token + corrective cue (not ornate taxonomy preamble).", ), BenchmarkSpec( "geo_england_continent_class", "geopolitics", "The continent England is part of is", "The continent England is part of is Asia. The correct answer is", " Europe", " Asia", "Straight factual contrast; avoids jokey preamble that swamps gap geometry.", ), BenchmarkSpec( "math_polygon_triangle_three_edges", "geometry_definitions", ( "Euclidean planar polygon possessing exactly three straight edges pairwise meeting vertices " "qualifies nomenclaturally" ), ( "Euclidean planar polygon possessing exactly three straight edges pairwise meeting vertices " "qualifies nomenclaturally square falsely historically academically. Correction:" ), " triangle", " square", ), BenchmarkSpec( "temporal_weekday_after_wednesday", "calendar_reasoning", "Immediately following weekday Wednesday inside Western seven-day nomenclature comes", ( "Immediately following weekday Wednesday inside Western seven-day nomenclature comes Tuesday " "mistaken academically. Correction:" ), " Thursday", " Tuesday", ), BenchmarkSpec( "commonsense_sunrise_bearing_midlatitude", "commonsense_navigation", ( "For classical mid-latitude Northern hemisphere outdoor averages seasons excluding polar edge " "cases sunrise azimuth tendencies favor cardinal direction principally" ), ( "For classical mid-latitude Northern hemisphere outdoor averages seasons excluding polar edge " "cases sunrise azimuth tendencies favor cardinal direction principally West mistakenly academically. " "Correction:" ), " East", " West", ), BenchmarkSpec( "lit_author_macbeth_credit", "literature", "The author of the tragedy Macbeth is", "The author of the tragedy Macbeth is Dickens. The correct answer is", " Shakespeare", " Dickens", "Same correction-scaffold grammar as geography rows (not mid-sentence credit phrasing alone).", ), BenchmarkSpec( "sport_curling_playing_surface", "sports_inference", ( "Olympic curling competitions slide calibrated stones competitively across sheets composed " "principally of frozen" ), ( "Olympic curling competitions slide calibrated stones competitively across grassy meadows outdoors " "naturally falsely academically. Correction:" ), " ice", " grass", ), BenchmarkSpec( "climate_polar_vs_tropic_norm", "climate_comparison", "Relative to icy polar climatic norms statistically humid rainforest tropical norms skew temperature-wise", ( "Relative to icy polar climatic norms statistically humid rainforest tropical norms skew temperature-wise " "colder polar-like mistakenly academically. Correction:" ), " warmer", " colder", ), BenchmarkSpec( "physics_iron_ambient_phase", "matter_physics", ( "At ordinary atmospheric pressure untreated bulk iron ordinarily sits macroscopic matter phase plainly" ), ( "At ordinary atmospheric pressure untreated bulk iron ordinarily sits macroscopic matter phase plainly " "liquid mistakenly academically. Correction:" ), " solid", " liquid", ), BenchmarkSpec( "bio_respiration_terminal_electron_carrier", "cell_bio", "The terminal electron acceptor in human aerobic mitochondrial respiration is", ( "The terminal electron acceptor in human aerobic mitochondrial respiration is carbon. " "The correct answer is" ), " Oxygen", " Carbon", "Factual-slot wrong element + corrective cue matched to geography template.", ), ]