{
  "date": "2026-08-25",
  "type": "controlled regression samples; not a full official leaderboard run",
  "seed": 20260819,
  "decoding": {
    "method": "greedy",
    "budgets": {
      "gsm8k": 256,
      "math500": 384,
      "arc_challenge": 128,
      "folio": 160
    }
  },
  "models": {
    "budgie": {
      "repo": "Asilarkness/Budgie-500m",
      "checkpoint": "verified-math-a025",
      "source_repo": "Asilarkness/testgeniy",
      "source_path": "candidates/budgie-alignment-v2/verified-math-a025",
      "source_revision": "79aec5157c10e3f7e082823a6e9150482d740a91"
    },
    "previous_public": {
      "repo": "Asilarkness/Budgie-500m",
      "revision": "39c877f07ddc80aa975809a4d87c848372316caf"
    },
    "qwen3": {
      "repo": "Qwen/Qwen3-0.6B",
      "revision": "c1899de289a04d12100db370d81485cdf75e47ca",
      "thinking": false
    }
  },
  "datasets": {
    "GSM8K": "openai/gsm8k main test, fixed n=30",
    "MATH-500": "HuggingFaceH4/MATH-500 test, fixed n=15",
    "ARC-Challenge": "allenai/ai2_arc ARC-Challenge test, fixed n=30",
    "FOLIO": "tasksource/folio validation, fixed n=30",
    "HelpSteer2-pairwise": "nvidia/HelpSteer2 validation, fixed n=200"
  },
  "previous_root_results": {
    "GSM8K": {
      "correct": 4,
      "total": 30,
      "score": 13.333333333333334
    },
    "MATH-500": {
      "correct": 2,
      "total": 15,
      "score": 13.333333333333334
    },
    "ARC-Challenge": {
      "correct": 8,
      "total": 30,
      "score": 26.666666666666668
    },
    "FOLIO": {
      "correct": 11,
      "total": 30,
      "score": 36.666666666666664
    },
    "HelpSteer2-pairwise": {
      "correct": 96,
      "total": 200,
      "score": 48.0
    },
    "Composite": {
      "score": 27.6
    }
  },
  "results": {
    "budgie": {
      "GSM8K": {
        "correct": 5,
        "total": 30,
        "score": 16.666666666666664
      },
      "MATH-500": {
        "correct": 3,
        "total": 15,
        "score": 20.0
      },
      "ARC-Challenge": {
        "correct": 11,
        "total": 30,
        "score": 36.666666666666664
      },
      "FOLIO": {
        "correct": 13,
        "total": 30,
        "score": 43.333333333333336
      },
      "HelpSteer2-pairwise": {
        "correct": 97,
        "total": 200,
        "score": 48.5
      },
      "Composite": {
        "score": 33.03333333333333
      }
    },
    "qwen3": {
      "GSM8K": {
        "correct": 14,
        "total": 30,
        "score": 46.666666666666664
      },
      "MATH-500": {
        "correct": 2,
        "total": 15,
        "score": 13.333333333333334
      },
      "ARC-Challenge": {
        "correct": 19,
        "total": 30,
        "score": 63.333333333333336
      },
      "FOLIO": {
        "correct": 13,
        "total": 30,
        "score": 43.333333333333336
      },
      "HelpSteer2-pairwise": {
        "correct": 100,
        "total": 200,
        "score": 50.0
      },
      "Composite": {
        "score": 43.33333333333333
      }
    }
  },
  "expanded_disjoint_gate": {
    "seed": 20260908,
    "split": "disjoint expanded gate; excludes every row in the original controlled gate",
    "results": {
      "GSM8K": {
        "total": 80,
        "previous_correct": 9,
        "previous_score": 11.25,
        "current_correct": 24,
        "current_score": 30.0
      },
      "MATH-500": {
        "total": 50,
        "previous_correct": 10,
        "previous_score": 20.0,
        "current_correct": 10,
        "current_score": 20.0
      },
      "ARC-Challenge": {
        "total": 80,
        "previous_correct": 16,
        "previous_score": 20.0,
        "current_correct": 25,
        "current_score": 31.25
      },
      "FOLIO": {
        "total": 100,
        "previous_correct": 31,
        "previous_score": 31.0,
        "current_correct": 36,
        "current_score": 36.0
      },
      "HelpSteer2-pairwise": {
        "total": 373,
        "previous_correct": 166,
        "previous_score": 44.50402144772118,
        "current_correct": 169,
        "current_score": 45.30831099195711
      },
      "Composite": {
        "previous_score": 25.35080428954424,
        "current_score": 32.51166219839142,
        "delta": 7.160857908847185
      }
    },
    "paired_bootstrap": {
      "samples": 30000,
      "current_composite_95": [
        28.34289544235925,
        36.82319034852547
      ],
      "delta_95": [
        2.5861930294906132,
        11.725335120643432
      ],
      "probability_delta_positive": 99.86333333333333
    }
  },
  "training_disclosure": {
    "no_fixed_test_or_validation_row_used_for_training": true,
    "decontamination": "normalized exact matching, long containment checks, deduplication, and shared 12-token-window filtering against fixed development/final gates",
    "legacy_training_split_sources": [
      "GSM8K train",
      "ARC-Challenge train",
      "Hendrycks MATH train",
      "ProofWriter train"
    ],
    "other_sources": [
      "NuminaMath-CoT",
      "Bespoke-Stratos-17k",
      "SmolTalk / Smol-SmolTalk",
      "Qwen3.8-27B synthetic and distilled answers",
      "random verified mathematics"
    ],
    "note": "This retained checkpoint predates the stricter benchmark-evaluation-only policy now used for future work. Training-split use is disclosed here; no reported held-out row was used for training."
  },
  "full_split_comparison": {
    "date": "2026-08-26",
    "type": "complete-split controlled comparison; not an official leaderboard submission",
    "models": {
      "old_public": {
        "repo": "Asilarkness/Budgie-500m",
        "revision": "39c877f07ddc80aa975809a4d87c848372316caf",
        "weight_sha256": "10845d227ea7ea50d14757df426e5265820c6fe1d48741b0c38b4fc23bd2381e"
      },
      "current_leader": {
        "repo": "Asilarkness/Budgie-500m",
        "weights_revision": "34f0d09538dcca578ea672ba717e8e9f2672021f",
        "checkpoint": "ba2_vmath_a025 / verified-math-a025",
        "weight_sha256": "7404de2c08df968f41800d1362704a0ad4106ea0116cd0cd4b1709612a2a66be"
      }
    },
    "datasets": {
      "gsm8k": {
        "repo": "openai/gsm8k",
        "config": "main",
        "split": "test",
        "n": 1319
      },
      "math500": {
        "repo": "HuggingFaceH4/MATH-500",
        "split": "test",
        "n": 500
      },
      "arc_challenge": {
        "repo": "allenai/ai2_arc",
        "config": "ARC-Challenge",
        "split": "test",
        "n": 1172
      },
      "folio": {
        "repo": "tasksource/folio",
        "split": "validation",
        "n": 203
      },
      "helpsteer2": {
        "repo": "nvidia/HelpSteer2",
        "split": "validation",
        "paired_non_tied_rows": 373,
        "note": "Reused complete 373-pair evaluation already computed for these exact revisions."
      }
    },
    "protocol": {
      "prompts": "same user-only task prompts for both models",
      "decoding": "greedy",
      "budgets": {
        "gsm8k": 256,
        "math500": 384,
        "arc_challenge": 128,
        "folio": 160
      },
      "scoring": {
        "gsm8k": "final numeric answer exact match",
        "math500": "math_verify symbolic equivalence against reference solution, main thread",
        "arc_challenge": "final option label; handles letter and numeric label variants",
        "folio": "final True/False/Uncertain exact label",
        "helpsteer2": "lower assistant-answer NLL on human-preferred response"
      },
      "batching": "Only prompts with identical tokenized input length were batched. No input padding. Each row was truncated manually at its first EOS. A two-prompt audit exactly matched sequential generation.",
      "invalid_attempt": "An earlier unequal-length padded batch was detected as corrupt and discarded; none of those outputs enter this report."
    },
    "runtime": {
      "torch": "2.11.0+cu130",
      "transformers": "5.15.1",
      "datasets": "5.0.0"
    },
    "results": {
      "tasks": {
        "gsm8k": {
          "n": 1319,
          "old_correct": 191,
          "current_correct": 245,
          "old_score": 0.14480667172100076,
          "current_score": 0.18574677786201668,
          "delta": 0.04094010614101592,
          "improved": 143,
          "worsened": 89,
          "paired_binomial_p": 0.00047624975354938266
        },
        "math500": {
          "n": 500,
          "old_correct": 62,
          "current_correct": 60,
          "old_score": 0.124,
          "current_score": 0.12,
          "delta": -0.004,
          "improved": 24,
          "worsened": 26,
          "paired_binomial_p": 0.887724827340783
        },
        "arc_challenge": {
          "n": 1172,
          "old_correct": 291,
          "current_correct": 323,
          "old_score": 0.24829351535836178,
          "current_score": 0.27559726962457337,
          "delta": 0.027303754266211604,
          "improved": 246,
          "worsened": 214,
          "paired_binomial_p": 0.14827050225768382
        },
        "folio": {
          "n": 203,
          "old_correct": 63,
          "current_correct": 72,
          "old_score": 0.3103448275862069,
          "current_score": 0.35467980295566504,
          "delta": 0.04433497536945813,
          "improved": 25,
          "worsened": 16,
          "paired_binomial_p": 0.21102359760880063
        },
        "helpsteer2": {
          "n": 373,
          "old_correct": 166,
          "current_correct": 169,
          "old_score": 0.4450402144772118,
          "current_score": 0.45308310991957107,
          "delta": 0.00804289544235925,
          "improved": 9,
          "worsened": 6,
          "paired_binomial_p": 0.6072387695312499
        }
      },
      "reasoning_macro_old": 0.20686125366639235,
      "reasoning_macro_current": 0.23400596261056378,
      "reasoning_delta": 0.027144708944171425,
      "five_metric_composite_old": 0.25449704582855626,
      "five_metric_composite_current": 0.2778213920723652,
      "five_metric_delta": 0.02332434624380897,
      "paired_bootstrap_delta_95": [
        0.00678765357806182,
        0.03969201450574924
      ],
      "probability_delta_positive": 0.9978666666666667,
      "bootstrap_samples": 30000
    },
    "artifact": "https://huggingface.co/Asilarkness/testgeniy/tree/main/candidates/budgie-alignment-v2/fullbench-old-vs-vmath25-v1"
  }
}